data-analyst-mcp 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -4
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +3 -0
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.js +2 -13
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +44 -12
- package/dist/server.js.map +1 -1
- package/dist/tools.js +27 -6
- package/dist/tools.js.map +1 -1
- package/guide/assets/download_data.ps1 +2 -0
- package/guide/assets/download_data.sh +8 -2
- package/guide/references/routing.md +30 -3
- package/package.json +1 -1
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/college-scorecard/source.md +1 -1
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/source.md +1 -1
- package/packs/fbi/source.md +3 -3
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fsa/source.md +1 -1
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/source.md +1 -1
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/port.json +65 -0
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/treasury/source.md +1 -1
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/uspto/source.md +1 -1
package/README.md
CHANGED
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
An MCP server for official U.S. government data. It asks the user the questions that change a headline
|
|
4
4
|
number before fetching anything, gets the data from the agency at call time, packages it (CSV, data
|
|
5
|
-
dictionary, provenance) and cites every result. It covers
|
|
5
|
+
dictionary, provenance) and cites every result. It covers 76 sources.
|
|
6
6
|
|
|
7
|
-
> **Status: 0.3.** All
|
|
7
|
+
> **Status: 0.3.** All 76 sources work, along with the analysis and audit tools, hosted mode (streamable HTTP, a
|
|
8
8
|
> claude.ai connector) and per-user keys. The test suite runs offline on recorded fixtures.
|
|
9
9
|
> One gap remains: usda-fna's three PDF endpoints need a PDF text reader, and return `not_supported`.
|
|
10
|
+
> Four sources are manual only, because their sites refuse scripts: cdc-wonder, hud-ahar, foreign-assistance and
|
|
11
|
+
> cpsc's NEISS endpoint (`manual_bundle` gives the links, `ingest` reads the saved file).
|
|
10
12
|
> The package is not published to npm yet.
|
|
11
13
|
|
|
12
14
|
## Install
|
|
@@ -234,8 +236,30 @@ its `source_brief`.
|
|
|
234
236
|
|
|
235
237
|
The files are read again on each lookup, so a key stored during a session works without a restart.
|
|
236
238
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
+
What the server guarantees, and tests check:
|
|
240
|
+
|
|
241
|
+
- **No key in any result.** Every tool result, error, prompt and resource is redacted for every key the server can
|
|
242
|
+
find, in every form it can be echoed in (as is, URL-encoded, JSON-escaped).
|
|
243
|
+
- **No tool reads a key file.** A path the model names (`ingest`, a `for_each` table, `report.json` paths,
|
|
244
|
+
`read_file`) is refused when it is one of the files above, a `.env` file, `~/.ssh`, `~/.aws` or a similar place,
|
|
245
|
+
or when the file holds a key.
|
|
246
|
+
- **No key in a written file.** A key an agency echoes is replaced with `[REDACTED]` in every file written to the
|
|
247
|
+
workspace (the result says which), and never reaches the download cache or a replication package.
|
|
248
|
+
- **A key goes only to its agency:** never along a next-page link to another site, never from https to http. On a
|
|
249
|
+
redirect to another host, a key in the URL or a header is dropped; a key in the request body (BLS) follows a
|
|
250
|
+
307/308 redirect wherever it points.
|
|
251
|
+
|
|
252
|
+
**What the server cannot stop:** an assistant that has its own file or shell tools (Claude Code does) can open the
|
|
253
|
+
key files itself. `setup` prints the Claude Code rule that blocks this; add it to the `permissions.deny` list of your
|
|
254
|
+
`~/.claude/settings.json`, for example:
|
|
255
|
+
|
|
256
|
+
```json
|
|
257
|
+
{ "permissions": { "deny": ["Read(~/.config/data-analyst/**)", "Edit(~/.config/data-analyst/**)"] } }
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
Prefer `setup` (or a secret manager through `<VAR>_FILE`) over the `env` block of `~/.claude/settings.json`: that
|
|
261
|
+
file is one Claude Code reads anyway. The hosted server reads keys only from its environment and from each user's
|
|
262
|
+
encrypted store (see [docs/hosting.md](docs/hosting.md)).
|
|
239
263
|
|
|
240
264
|
## Hosted mode (claude.ai connector)
|
|
241
265
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bureau of Justice Statistics. NCVS Select records from api.ojp.gov (Socrata-style SODA) and report table packages
|
|
3
|
+
* (zips of CSVs) from bjs.ojp.gov, both read by core; this adapter keeps requests off the site's bot check.
|
|
4
|
+
*
|
|
5
|
+
* Verified 2026-11-07: plain filtered queries ($where, $select of columns, $limit/$offset) answer JSON; any
|
|
6
|
+
* aggregate ($select with sum(), count(*), with or without $group) gets a Cloudflare "Just a moment..." challenge
|
|
7
|
+
* page (HTTP 403, which core reports as "blocked", or 200 HTML) instead. checkSpec refuses aggregates before a
|
|
8
|
+
* request is sent, and checkError names an HTTP 200 challenge page. The challenge is never answered: aggregate the
|
|
9
|
+
* rows with compute.
|
|
10
|
+
*/
|
|
11
|
+
import type { Adapter } from "../core/packs.js";
|
|
12
|
+
declare const adapter: Adapter;
|
|
13
|
+
export default adapter;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const AGGREGATE = /\(|\*/;
|
|
3
|
+
function checkSpec(spec) {
|
|
4
|
+
const bad = [];
|
|
5
|
+
for (const r of spec.requests || []) {
|
|
6
|
+
if ((r.endpoint ?? "query") !== "query")
|
|
7
|
+
continue;
|
|
8
|
+
const p = r.params || {};
|
|
9
|
+
if ("$group" in p || "$having" in p || AGGREGATE.test(pyStr(p.$select ?? ""))) {
|
|
10
|
+
bad.push(`request ${JSON.stringify(r.name)}: BJS's API answers aggregate queries (sum, count, $group) with a bot ` +
|
|
11
|
+
"check, not data. Select plain columns (e.g. \"year, newcrime, newoff, newwgt\") with a $where filter and sum the " +
|
|
12
|
+
"weights with compute.");
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return bad;
|
|
16
|
+
}
|
|
17
|
+
function checkError(status, body) {
|
|
18
|
+
const head = body.subarray(0, 4096).toString("latin1");
|
|
19
|
+
if (/<title>Just a moment|challenges\.cloudflare\.com|cf-chl/i.test(head)) {
|
|
20
|
+
return `BJS's site answered with a bot-check page (HTTP ${status}) instead of data. Aggregate queries always get ` +
|
|
21
|
+
"it; for a plain query, wait a minute and fetch again with fewer requests, or use manual_bundle to download the " +
|
|
22
|
+
"file in a browser.";
|
|
23
|
+
}
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const adapter = { checkSpec, checkError };
|
|
27
|
+
export default adapter;
|
|
28
|
+
//# sourceMappingURL=bjs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bjs.js","sourceRoot":"","sources":["../../src/adapters/bjs.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,SAAS,GAAG,OAAO,CAAC;AAE1B,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,OAAO;YAAE,SAAS;QAClD,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,QAAQ,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YAC9E,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,wEAAwE;gBAChH,mHAAmH;gBACnH,uBAAuB,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,MAAc,EAAE,IAAY;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,0DAA0D,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1E,OAAO,mDAAmD,MAAM,kDAAkD;YAChH,iHAAiH;YACjH,oBAAoB,CAAC;IACzB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;AACnD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U.S. Customs and Border Protection nationwide encounter files. Verified 2026-11-07: the Nationwide Encounters
|
|
3
|
+
* document page (www.cbp.gov/document/stats/nationwide-encounters) links monthly-replaced CSVs under
|
|
4
|
+
* /sites/default/files/<yyyy-mm>/nationwide-encounters-fy..-..-<month>-aor.csv (by area of responsibility) and
|
|
5
|
+
* ...-state.csv (by state). list_files reads that page; encounters reads one file, filtered here.
|
|
6
|
+
*/
|
|
7
|
+
import * as gr from "../core/generic-rest.js";
|
|
8
|
+
import { DataError } from "../core/http.js";
|
|
9
|
+
import { pyStr } from "../core/py.js";
|
|
10
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
11
|
+
.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
12
|
+
/** The encounter CSVs linked from the document page, newest first. */
|
|
13
|
+
export function filesOnPage(html) {
|
|
14
|
+
const seen = new Set();
|
|
15
|
+
const out = [];
|
|
16
|
+
for (const m of html.matchAll(/href="(?:https:\/\/www\.cbp\.gov)?\/sites\/default\/files\/([^"]*?nationwide-encounters[^"]*?)\.csv"/gi)) {
|
|
17
|
+
const file = m[1];
|
|
18
|
+
if (seen.has(file))
|
|
19
|
+
continue;
|
|
20
|
+
seen.add(file);
|
|
21
|
+
const by = /-aor$/i.test(file) ? "area of responsibility" : /-state$/i.test(file) ? "state" : "";
|
|
22
|
+
const fy = /fy(\d{2})-fy(\d{2})/i.exec(file);
|
|
23
|
+
out.push({ file, by, fiscal_years: fy ? `FY20${fy[1]}-FY20${fy[2]}` : "", url: `https://www.cbp.gov/sites/default/files/${file}.csv` });
|
|
24
|
+
}
|
|
25
|
+
// newest first: by the last fiscal year covered, then by path
|
|
26
|
+
const key = (r) => `${pyStr(r.fiscal_years).slice(-4)} ${pyStr(r.file).split("/").pop()}`;
|
|
27
|
+
return out.sort((a, b) => (key(a) < key(b) ? 1 : key(a) > key(b) ? -1 : 0));
|
|
28
|
+
}
|
|
29
|
+
async function fetch(spec, pack) {
|
|
30
|
+
const lists = spec.requests.filter((r) => r.endpoint === "list_files");
|
|
31
|
+
const rest = spec.requests.filter((r) => r.endpoint !== "list_files");
|
|
32
|
+
const tables = {};
|
|
33
|
+
const notes = [];
|
|
34
|
+
if (lists.length) {
|
|
35
|
+
const resp = await gr.clientFor(pack).request("GET", pack.endpoints.document_page, null);
|
|
36
|
+
const rows = filesOnPage(resp.text());
|
|
37
|
+
if (!rows.length)
|
|
38
|
+
throw new DataError("the CBP document page lists no encounter CSV (the page changed?)", "api");
|
|
39
|
+
for (const r of lists)
|
|
40
|
+
tables[r.name] = rows;
|
|
41
|
+
}
|
|
42
|
+
if (rest.length) {
|
|
43
|
+
const [t, n] = await gr.fetchSpec(pack, { ...spec, requests: rest });
|
|
44
|
+
Object.assign(tables, t);
|
|
45
|
+
notes.push(...n);
|
|
46
|
+
}
|
|
47
|
+
return [tables, notes];
|
|
48
|
+
}
|
|
49
|
+
const FILTERS = [["region", "Land Border Region"], ["component", "Component"],
|
|
50
|
+
["fiscal_year", "Fiscal Year"], ["citizenship", "Citizenship"]];
|
|
51
|
+
function normalize(req, _ep, rows) {
|
|
52
|
+
if (req.endpoint !== "encounters")
|
|
53
|
+
return rows;
|
|
54
|
+
const wants = FILTERS.map(([k, c]) => [c, list(req.params?.[k])]).filter(([, v]) => v.length);
|
|
55
|
+
return wants.length ? rows.filter((r) => wants.every(([c, v]) => v.includes(pyStr(r[c] ?? "").trim().toLowerCase()))) : rows;
|
|
56
|
+
}
|
|
57
|
+
function checkSpec(spec) {
|
|
58
|
+
return (spec.requests || []).filter((r) => r.endpoint === "encounters" && r.params?.file !== undefined &&
|
|
59
|
+
!/^[\w./-]+$/.test(pyStr(r.params.file)) || (r.endpoint === "encounters" && /\.\./.test(pyStr(r.params?.file ?? ""))))
|
|
60
|
+
.map((r) => `request ${JSON.stringify(r.name)}: file is a path from list_files, e.g. 2026-11/nationwide-encounters-fy24-fy27-oct-aor`);
|
|
61
|
+
}
|
|
62
|
+
const adapter = { fetch, normalize, checkSpec };
|
|
63
|
+
export default adapter;
|
|
64
|
+
//# sourceMappingURL=cbp.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cbp.js","sourceRoot":"","sources":["../../src/adapters/cbp.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,sEAAsE;AACtE,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,wGAAwG,CAAC,EAAE,CAAC;QACxI,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,EAAE,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,2CAA2C,IAAI,MAAM,EAAE,CAAC,CAAC;IAC1I,CAAC;IACD,8DAA8D;IAC9D,MAAM,GAAG,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/F,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9E,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC;IACvE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC;IACtE,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QACzF,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,kEAAkE,EAAE,KAAK,CAAC,CAAC;QACjH,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC/C,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,OAAO,GAAuB,CAAC,CAAC,QAAQ,EAAE,oBAAoB,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;IAC/F,CAAC,aAAa,EAAE,aAAa,CAAC,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC;AAElE,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpH,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/H,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,YAAY,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS;QACpG,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;SACrH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAC;AAC3I,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AACzD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CDC WONDER query results, uploaded as WONDER exports them. Manual only.
|
|
3
|
+
*
|
|
4
|
+
* wonder.cdc.gov answers this client's requests (the API controller and the help pages) with HTTP 403 (verified
|
|
5
|
+
* 2026-11-07), so refusedHosts sends every request to manual_bundle. The WONDER API would also need each query
|
|
6
|
+
* to accept the data use restrictions; on the web site the user reads and accepts them, which is the user's
|
|
7
|
+
* decision to make.
|
|
8
|
+
*
|
|
9
|
+
* parseUpload reads the export (Export button, tab-delimited .txt). The layout was not checked against a real
|
|
10
|
+
* export here, so the reader is defensive:
|
|
11
|
+
* - the header is the first line; the rows run to the line "---", after which WONDER writes its notes (kept in
|
|
12
|
+
* raw/, not in the table);
|
|
13
|
+
* - "Suppressed", "Unreliable" and "Not Applicable" stay as text;
|
|
14
|
+
* - a file without a header line fails with a clear message.
|
|
15
|
+
*/
|
|
16
|
+
import fs from "node:fs";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { DataError } from "../core/http.js";
|
|
19
|
+
import { pyStr } from "../core/py.js";
|
|
20
|
+
import { columnsOf, decodeText, tsvRecords, withColumns } from "../core/tabular.js";
|
|
21
|
+
const HOME = "https://wonder.cdc.gov/";
|
|
22
|
+
const REFUSED = "wonder.cdc.gov answers this client with HTTP 403 (verified 2026-11-07), and its API needs each " +
|
|
23
|
+
"query to accept CDC WONDER's data use restrictions, which is the user's own decision: use manual_bundle. The " +
|
|
24
|
+
"user runs the query on the WONDER site (accepting the restrictions there), exports the result as a " +
|
|
25
|
+
"tab-delimited .txt and uploads it; then ingest";
|
|
26
|
+
const refusedHosts = (_spec) => REFUSED;
|
|
27
|
+
function checkSpec(spec) {
|
|
28
|
+
return (spec.requests || []).filter((r) => !pyStr(r.params?.database ?? "").trim())
|
|
29
|
+
.map((r) => `request ${JSON.stringify(r.name)}: name the WONDER database the query ran on (database)`);
|
|
30
|
+
}
|
|
31
|
+
function manualItems(spec, _pack) {
|
|
32
|
+
return (spec.requests || []).map((r) => {
|
|
33
|
+
const db = pyStr(r.params?.database ?? ""), q = pyStr(r.params?.query ?? "");
|
|
34
|
+
return { name: `${r.name}#1`, request: r.name, url: HOME, filename: `${r.name}.txt`, also_ext: [".txt", ".tsv"],
|
|
35
|
+
needs_key: false, method: "GET", body: null,
|
|
36
|
+
note: `open in a web browser, choose ${db || "the database"}, read and accept its data use restrictions, ` +
|
|
37
|
+
(q ? `set the query (${q}), ` : "set Group By and the filters, ") +
|
|
38
|
+
"click Send, then Export and save the tab-delimited .txt. Upload it as saved." };
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/** The table part of a WONDER export (the lines before "---"). */
|
|
42
|
+
export function exportRows(text) {
|
|
43
|
+
const lines = text.replace(/^/, "").split(/\r?\n/);
|
|
44
|
+
const end = lines.findIndex((l) => l.replace(/"/g, "").trim() === "---");
|
|
45
|
+
const table = (end >= 0 ? lines.slice(0, end) : lines).join("\n");
|
|
46
|
+
return tsvRecords({ header_row: 1 }, table);
|
|
47
|
+
}
|
|
48
|
+
function parseUpload(_pack, req, files) {
|
|
49
|
+
const rows = [];
|
|
50
|
+
const order = [];
|
|
51
|
+
for (const f of [...files].sort()) {
|
|
52
|
+
const got = exportRows(decodeText(fs.readFileSync(f)));
|
|
53
|
+
const cols = columnsOf(got) || (got[0] ? Object.keys(got[0]) : []);
|
|
54
|
+
if (!got.length || cols.length < 2 || /<html/i.test(cols.join(" "))) {
|
|
55
|
+
throw new DataError(`${path.basename(f)}: not a WONDER export (no tab-delimited header line). On the results ` +
|
|
56
|
+
"page click Export and save the .txt file.", "bad_request");
|
|
57
|
+
}
|
|
58
|
+
for (const c of cols)
|
|
59
|
+
if (!order.includes(c))
|
|
60
|
+
order.push(c);
|
|
61
|
+
rows.push(...got);
|
|
62
|
+
}
|
|
63
|
+
const out = rows.map((r) => Object.fromEntries(Object.entries(r).map(([k, v]) => [k, v === null || v === undefined ? "" : v])));
|
|
64
|
+
return withColumns(out, order);
|
|
65
|
+
}
|
|
66
|
+
const adapter = { refusedHosts, checkSpec, manualItems, parseUpload };
|
|
67
|
+
export default adapter;
|
|
68
|
+
//# sourceMappingURL=cdc-wonder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cdc-wonder.js","sourceRoot":"","sources":["../../src/adapters/cdc-wonder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEpF,MAAM,IAAI,GAAG,yBAAyB,CAAC;AACvC,MAAM,OAAO,GAAG,iGAAiG;IAC/G,+GAA+G;IAC/G,qGAAqG;IACrG,gDAAgD,CAAC;AAEnD,MAAM,YAAY,GAAG,CAAC,KAAW,EAAU,EAAE,CAAC,OAAO,CAAC;AAEtD,SAAS,SAAS,CAAC,IAAU;IAC3B,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,KAAW;IAC1C,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACrC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,MAAM,EAAE,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YAC7G,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;YAC3C,IAAI,EAAE,iCAAiC,EAAE,IAAI,cAAc,+CAA+C;gBACxG,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,gCAAgC,CAAC;gBACjE,8EAA8E,EAAE,CAAC;IACvF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClE,OAAO,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,WAAW,CAAC,KAAW,EAAE,GAAgB,EAAE,KAAe;IACjE,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,uEAAuE;gBAC5G,2CAA2C,EAAE,aAAa,CAAC,CAAC;QAChE,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChI,OAAO,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAC/E,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consumer Product Safety Commission: the SaferProducts.gov Recall API (JSON) and NEISS injury data (manual only).
|
|
3
|
+
*
|
|
4
|
+
* Recall API, verified 2026-11-07:
|
|
5
|
+
* - With no filter it returns an empty list. An unknown parameter is ignored: it returns what the other filters
|
|
6
|
+
* give, which would look like an answer, so checkSpec refuses a request with no API filter.
|
|
7
|
+
* - These filters work (part of the field, case-insensitive): RecallNumber, RecallDateStart/End,
|
|
8
|
+
* LastPublishDateStart, ProductName, ProductType, RecallTitle, RecallDescription, Manufacturer, Retailer,
|
|
9
|
+
* Importer, Injury, Remedy.
|
|
10
|
+
* - Hazard, ManufacturerCountry and ProductModel are accepted but match nothing, so hazard and country are
|
|
11
|
+
* filtered here instead.
|
|
12
|
+
* normalize turns the nested lists (Products, Hazards, Remedies ...) into one row per recall with joined text.
|
|
13
|
+
*
|
|
14
|
+
* NEISS: www.cpsc.gov answers scripts with HTTP 403 (the query system, the archived files and the NEISS pages;
|
|
15
|
+
* verified 2026-11-07). The archived yearly files sit behind a form. So the neiss endpoint is refused before any
|
|
16
|
+
* request (refusedHosts), manualItems points at the query page, and parseUpload reads the saved Tab Delimited Text
|
|
17
|
+
* or Excel file.
|
|
18
|
+
*/
|
|
19
|
+
import fs from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import * as gr from "../core/generic-rest.js";
|
|
22
|
+
import { DataError } from "../core/http.js";
|
|
23
|
+
import { pyStr } from "../core/py.js";
|
|
24
|
+
import { decodeText, sheetRecords, tsvRecords } from "../core/tabular.js";
|
|
25
|
+
const NEISS_PAGE = "https://www.cpsc.gov/cgibin/NEISSQuery/home.aspx";
|
|
26
|
+
const REFUSED = "www.cpsc.gov answers scripts with HTTP 403 (verified 2026-11-07), and NEISS's archived yearly files " +
|
|
27
|
+
"are behind a form, so NEISS data cannot be fetched from here. Use manual_bundle: the user downloads the year " +
|
|
28
|
+
"from the NEISS query page (Archived Annual NEISS Data, file format Tab Delimited Text) and uploads it; then " +
|
|
29
|
+
"ingest. Fetch recall requests in a separate spec.";
|
|
30
|
+
const API_FILTERS = ["RecallNumber", "RecallDateStart", "RecallDateEnd", "LastPublishDateStart", "ProductName",
|
|
31
|
+
"ProductType", "RecallTitle", "RecallDescription", "Manufacturer", "Retailer", "Importer", "Injury", "Remedy"];
|
|
32
|
+
const blank = (v) => v === undefined || v === null || pyStr(v).trim() === "";
|
|
33
|
+
const list = (v) => (Array.isArray(v) ? v : blank(v) ? [] : [v]).map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
34
|
+
function refusedHosts(spec) {
|
|
35
|
+
return (spec.requests || []).some((r) => r.endpoint === "neiss") ? REFUSED : "";
|
|
36
|
+
}
|
|
37
|
+
function checkSpec(spec) {
|
|
38
|
+
const bad = [];
|
|
39
|
+
for (const r of spec.requests || []) {
|
|
40
|
+
const p = r.params || {};
|
|
41
|
+
if (r.endpoint === "recalls" && !API_FILTERS.some((k) => !blank(p[k]))) {
|
|
42
|
+
bad.push(`request ${JSON.stringify(r.name)}: give at least one of ${API_FILTERS.join(", ")} (the API returns ` +
|
|
43
|
+
"nothing without a filter); e.g. RecallDateStart for a period");
|
|
44
|
+
}
|
|
45
|
+
for (const k of ["RecallDateStart", "RecallDateEnd", "LastPublishDateStart"]) {
|
|
46
|
+
if (!blank(p[k]) && !/^\d{4}-\d{2}-\d{2}$/.test(pyStr(p[k])))
|
|
47
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} must be YYYY-MM-DD`);
|
|
48
|
+
}
|
|
49
|
+
if (r.endpoint === "neiss" && !/^(19|20)\d{2}$/.test(pyStr(p.year ?? ""))) {
|
|
50
|
+
bad.push(`request ${JSON.stringify(r.name)}: neiss needs year (e.g. 2024)`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return bad;
|
|
54
|
+
}
|
|
55
|
+
/** The names of a recall's list field (Products, Hazards ...), joined with " | ". */
|
|
56
|
+
function names(v, key = "Name") {
|
|
57
|
+
if (!Array.isArray(v))
|
|
58
|
+
return "";
|
|
59
|
+
return v.map((x) => (x && typeof x === "object" ? pyStr(x[key] ?? "") : pyStr(x)).replace(/\s+/g, " ").trim())
|
|
60
|
+
.filter(Boolean).join(" | ");
|
|
61
|
+
}
|
|
62
|
+
const day = (v) => pyStr(v ?? "").slice(0, 10);
|
|
63
|
+
export function recallRow(r) {
|
|
64
|
+
return {
|
|
65
|
+
recall_number: pyStr(r.RecallNumber ?? ""), recall_date: day(r.RecallDate), title: r.Title ?? "",
|
|
66
|
+
products: names(r.Products), product_types: names(r.Products, "Type"),
|
|
67
|
+
number_of_units: names(r.Products, "NumberOfUnits"), hazards: names(r.Hazards), injuries: names(r.Injuries),
|
|
68
|
+
remedies: names(r.Remedies), remedy_options: names(r.RemedyOptions, "Option"), manufacturers: names(r.Manufacturers),
|
|
69
|
+
importers: names(r.Importers), distributors: names(r.Distributors), retailers: names(r.Retailers),
|
|
70
|
+
manufacturer_countries: names(r.ManufacturerCountries, "Country"), upcs: names(r.ProductUPCs, "UPC"),
|
|
71
|
+
description: pyStr(r.Description ?? "").replace(/\s+/g, " ").trim(), url: r.URL ?? "",
|
|
72
|
+
last_publish_date: day(r.LastPublishDate), recall_id: pyStr(r.RecallID ?? ""),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
function neissRows(req, rows) {
|
|
76
|
+
const want = list(req.params?.product);
|
|
77
|
+
if (!want.length)
|
|
78
|
+
return rows;
|
|
79
|
+
return rows.filter((r) => ["Product_1", "Product_2", "Product_3"].some((k) => want.includes(pyStr(r[k] ?? "").trim())));
|
|
80
|
+
}
|
|
81
|
+
function normalize(req, _ep, rows) {
|
|
82
|
+
if (req.endpoint === "neiss")
|
|
83
|
+
return neissRows(req, rows);
|
|
84
|
+
if (req.endpoint !== "recalls")
|
|
85
|
+
return rows;
|
|
86
|
+
const hz = pyStr(req.params?.hazard ?? "").trim().toLowerCase();
|
|
87
|
+
const countries = list(req.params?.country);
|
|
88
|
+
return rows.map(recallRow).filter((r) => (!hz || pyStr(r.hazards).toLowerCase().includes(hz)) &&
|
|
89
|
+
(!countries.length || pyStr(r.manufacturer_countries).toLowerCase().split(" | ").some((c) => countries.includes(c))));
|
|
90
|
+
}
|
|
91
|
+
function manualItems(spec, pack) {
|
|
92
|
+
const items = [];
|
|
93
|
+
const recalls = (spec.requests || []).filter((r) => r.endpoint !== "neiss");
|
|
94
|
+
if (recalls.length)
|
|
95
|
+
items.push(...gr.manualItems(pack, { ...spec, requests: recalls }));
|
|
96
|
+
for (const r of spec.requests || []) {
|
|
97
|
+
if (r.endpoint !== "neiss")
|
|
98
|
+
continue;
|
|
99
|
+
const y = pyStr(r.params?.year ?? "");
|
|
100
|
+
items.push({ name: `${r.name}#1`, request: r.name, url: NEISS_PAGE, filename: `${r.name}.tsv`,
|
|
101
|
+
also_ext: [".txt", ".tsv", ".xlsx"], needs_key: false, method: "GET", body: null,
|
|
102
|
+
note: `open in a web browser, pick a User Affiliation, then under 'Archived Annual NEISS Data' choose Year ${y} and ` +
|
|
103
|
+
"File Format 'Tab Delimited Text', click Find Data and save the file (Excel also works). Upload it as saved." });
|
|
104
|
+
}
|
|
105
|
+
return items;
|
|
106
|
+
}
|
|
107
|
+
function parseUpload(pack, req, files) {
|
|
108
|
+
if (req.endpoint !== "neiss")
|
|
109
|
+
return gr.parseUpload(pack, req, files);
|
|
110
|
+
const rows = [];
|
|
111
|
+
for (const f of [...files].sort()) {
|
|
112
|
+
const data = fs.readFileSync(f);
|
|
113
|
+
const got = data.subarray(0, 4).equals(Buffer.from("PK\x03\x04", "latin1"))
|
|
114
|
+
? sheetRecords({ sheet: 1, header_row: 1 }, data) : tsvRecords({ header_row: 1 }, decodeText(data));
|
|
115
|
+
if (got.length && !("CPSC_Case_Number" in got[0])) {
|
|
116
|
+
throw new DataError(`${path.basename(f)}: not a NEISS data file (no CPSC_Case_Number column; columns: ` +
|
|
117
|
+
`${Object.keys(got[0]).slice(0, 8).join(", ")}). Save the archived year's Tab Delimited Text file.`, "bad_request");
|
|
118
|
+
}
|
|
119
|
+
rows.push(...got);
|
|
120
|
+
}
|
|
121
|
+
return gr.clean(neissRows(req, rows), {});
|
|
122
|
+
}
|
|
123
|
+
const adapter = { refusedHosts, checkSpec, normalize, manualItems, parseUpload };
|
|
124
|
+
export default adapter;
|
|
125
|
+
//# sourceMappingURL=cpsc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cpsc.js","sourceRoot":"","sources":["../../src/adapters/cpsc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE1E,MAAM,UAAU,GAAG,kDAAkD,CAAC;AACtE,MAAM,OAAO,GAAG,sGAAsG;IACpH,+GAA+G;IAC/G,8GAA8G;IAC9G,mDAAmD,CAAC;AACtD,MAAM,WAAW,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,eAAe,EAAE,sBAAsB,EAAE,aAAa;IAC5G,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAEjH,MAAM,KAAK,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;AACtF,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAEpI,SAAS,YAAY,CAAC,IAAU;IAC9B,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAClF,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB;gBAC5G,8DAA8D,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,eAAe,EAAE,sBAAsB,CAAC,EAAE,CAAC;YAC7E,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACvI,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YAC1E,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qFAAqF;AACrF,SAAS,KAAK,CAAC,CAAU,EAAE,GAAG,GAAG,MAAM;IACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAE,CAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;SACpH,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,GAAG,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAExD,MAAM,UAAU,SAAS,CAAC,CAAM;IAC9B,OAAO;QACL,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;QAChG,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC;QACrE,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC3G,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC;QACpH,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACjG,sBAAsB,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;QACpG,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE;QACrF,iBAAiB,EAAE,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,IAAW;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC1H,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC1D,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1H,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,KAAK,GAAU,EAAE,CAAC;IACxB,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IACxF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,CAAC,QAAQ,KAAK,OAAO;YAAE,SAAS;QACrC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,MAAM;YAC3F,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;YAChF,IAAI,EAAE,uGAAuG,CAAC,OAAO;gBACnH,6GAA6G,EAAE,CAAC,CAAC;IACvH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,GAAgB,EAAE,KAAe;IAChE,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACtE,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;YACzE,CAAC,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;QACtG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,kBAAkB,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,gEAAgE;gBACrG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,sDAAsD,EAAE,aAAa,CAAC,CAAC;QACxH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAC1F,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FHWA Highway Statistics tables: /policyinformation/statistics/<year>/xls/<table>.xlsx, read by core's published-
|
|
3
|
+
* table reader (verified 2026-11-07 for 2023 and 2024: vm2, dl22, mv1, hm20, mf1, fe9; mf2 is not a 2023 file).
|
|
4
|
+
* Several workbooks carry a hidden report-tool sheet (CRYSTAL_PERSIST) first; skip_sheets leaves it out of the
|
|
5
|
+
* sheet numbering. This adapter only checks the spec.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
function checkSpec(spec) {
|
|
3
|
+
const bad = [];
|
|
4
|
+
for (const r of spec.requests || []) {
|
|
5
|
+
const p = r.params || {};
|
|
6
|
+
if (p.year !== undefined && !/^(19|20)\d{2}$/.test(pyStr(p.year)))
|
|
7
|
+
bad.push(`request ${JSON.stringify(r.name)}: year is the edition year, e.g. 2023`);
|
|
8
|
+
if (p.table !== undefined && !/^[a-z]{1,3}\d{1,3}[a-z]?$/.test(pyStr(p.table))) {
|
|
9
|
+
bad.push(`request ${JSON.stringify(r.name)}: table is the id in lower case without the hyphen (VM-2 -> vm2)`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return bad;
|
|
13
|
+
}
|
|
14
|
+
const adapter = { checkSpec };
|
|
15
|
+
export default adapter;
|
|
16
|
+
//# sourceMappingURL=fhwa.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fhwa.js","sourceRoot":"","sources":["../../src/adapters/fhwa.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACtJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,CAAC;AACvC,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ForeignAssistance.gov aggregate data files (the Data page's CSV downloads under /static/, refreshed with the
|
|
3
|
+
* site). Manual only: on 2026-11-07 foreignassistance.gov answered this server's requests with HTTP 403 (its
|
|
4
|
+
* documented Data API at /data-api/ answered 404), so refusedHosts sends fetch to manual_bundle; the user
|
|
5
|
+
* downloads the CSV in a browser and ingest filters it here by country, fiscal year and transaction type.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
4
|
+
const REFUSED = "foreignassistance.gov answers this server with HTTP 403 (verified 2026-11-07), so its files cannot " +
|
|
5
|
+
"be fetched from here: use manual_bundle; the user downloads the CSV from https://foreignassistance.gov/data " +
|
|
6
|
+
"(Data Download) in a web browser and uploads it; then ingest applies the filters";
|
|
7
|
+
const refusedHosts = (_spec) => REFUSED;
|
|
8
|
+
function checkSpec(spec) {
|
|
9
|
+
const bad = [];
|
|
10
|
+
for (const r of spec.requests || []) {
|
|
11
|
+
const p = r.params || {};
|
|
12
|
+
for (const k of ["from", "to"]) {
|
|
13
|
+
if (p[k] !== undefined && p[k] !== "" && !/^\d{4}$/.test(pyStr(p[k])))
|
|
14
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} is a fiscal year, e.g. 2024`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return bad;
|
|
18
|
+
}
|
|
19
|
+
function normalize(req, _ep, rows) {
|
|
20
|
+
const p = req.params || {};
|
|
21
|
+
const countries = list(p.country), years = list(p.fiscal_year), types = list(p.transaction_type);
|
|
22
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
23
|
+
return rows.filter((r) => {
|
|
24
|
+
const fy = Number(pyStr(r["Fiscal Year"] ?? ""));
|
|
25
|
+
if (countries.length && !countries.includes(pyStr(r["Country Code"] ?? "").toLowerCase()) &&
|
|
26
|
+
!countries.includes(pyStr(r["Country Name"] ?? "").toLowerCase()))
|
|
27
|
+
return false;
|
|
28
|
+
if (years.length && !years.includes(pyStr(r["Fiscal Year"] ?? "").toLowerCase()))
|
|
29
|
+
return false;
|
|
30
|
+
if (from !== null && !(fy >= from))
|
|
31
|
+
return false;
|
|
32
|
+
if (to !== null && !(fy <= to))
|
|
33
|
+
return false;
|
|
34
|
+
if (types.length && !types.includes(pyStr(r["Transaction Type Name"] ?? "").toLowerCase()) &&
|
|
35
|
+
!types.includes(pyStr(r["Transaction Type ID"] ?? "").toLowerCase()))
|
|
36
|
+
return false;
|
|
37
|
+
return true;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const adapter = { refusedHosts, checkSpec, normalize };
|
|
41
|
+
export default adapter;
|
|
42
|
+
//# sourceMappingURL=foreign-assistance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"foreign-assistance.js","sourceRoot":"","sources":["../../src/adapters/foreign-assistance.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,MAAM,OAAO,GAAG,qGAAqG;IACnH,8GAA8G;IAC9G,kFAAkF,CAAC;AAErF,MAAM,YAAY,GAAG,CAAC,KAAW,EAAU,EAAE,CAAC,OAAO,CAAC;AAEtD,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACjG,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACvF,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAClF,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/F,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACjD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACxF,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAChE,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HUD AHAR data files: Point-in-Time (PIT) counts and Housing Inventory Counts (HIC), by CoC and by state, from
|
|
3
|
+
* the 2025 AHAR Part 1 page on HUD User. Manual only.
|
|
4
|
+
*
|
|
5
|
+
* www.huduser.gov answers scripts with HTTP 202 and an empty body (a WAF challenge; verified 2026-11-07 on the
|
|
6
|
+
* AHAR pages and the .xlsb/.xlsx files), so refusedHosts sends every request to manual_bundle. The file names come
|
|
7
|
+
* from the 2025 edition's page (verified 2026-11-07). parseUpload reads the saved workbook:
|
|
8
|
+
* - .xlsb (binary, the PIT files) and .xls through SheetJS, .xlsx with the core reader;
|
|
9
|
+
* - the sheet named for the requested year (or `sheet`), header at `header_row` (default 1);
|
|
10
|
+
* - columns as published.
|
|
11
|
+
* The workbook layout was not checked against a downloaded copy (the files cannot be fetched from here): a
|
|
12
|
+
* missing sheet fails with the list of sheets, and source.md says to read the columns before use.
|
|
13
|
+
*/
|
|
14
|
+
import fs from "node:fs";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { DataError } from "../core/http.js";
|
|
17
|
+
import { pyStr } from "../core/py.js";
|
|
18
|
+
import { columnsOf, gridRecords, sheetNames, withColumns, xlsSheets, xlsxSheets } from "../core/tabular.js";
|
|
19
|
+
const PAGE = "https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html";
|
|
20
|
+
const REFUSED = "www.huduser.gov answers scripts with an empty HTTP 202 challenge (verified 2026-11-07), so the AHAR " +
|
|
21
|
+
"workbooks cannot be downloaded from here: use manual_bundle; the user opens the links in a web browser, saves " +
|
|
22
|
+
"the workbook and uploads it; then ingest";
|
|
23
|
+
const refusedHosts = (_spec) => REFUSED;
|
|
24
|
+
function checkSpec(spec) {
|
|
25
|
+
const bad = [];
|
|
26
|
+
for (const r of spec.requests || []) {
|
|
27
|
+
const p = r.params || {};
|
|
28
|
+
if (!/^(19|20)\d{2}$/.test(pyStr(p.year ?? "")) && !pyStr(p.sheet ?? "").trim()) {
|
|
29
|
+
bad.push(`request ${JSON.stringify(r.name)}: give year (the count year, e.g. 2025) or sheet`);
|
|
30
|
+
}
|
|
31
|
+
if (p.header_row !== undefined && !/^\d+$/.test(pyStr(p.header_row)))
|
|
32
|
+
bad.push(`request ${JSON.stringify(r.name)}: header_row is a line number`);
|
|
33
|
+
}
|
|
34
|
+
return bad;
|
|
35
|
+
}
|
|
36
|
+
function manualItems(spec, pack) {
|
|
37
|
+
const base = String(pack.endpoints?.base_url || "").replace(/\/+$/, "");
|
|
38
|
+
const files = pack.endpoints?.files || {};
|
|
39
|
+
return (spec.requests || []).filter((r) => files[r.endpoint || ""]).map((r) => {
|
|
40
|
+
const f = files[r.endpoint];
|
|
41
|
+
return { name: `${r.name}#1`, request: r.name, url: `${base}/${f}`, filename: f, also_ext: [".xlsb", ".xlsx", ".xls"],
|
|
42
|
+
needs_key: false, method: "GET", body: null,
|
|
43
|
+
note: `open in a web browser (huduser.gov refuses scripts); if the link has moved, find it on ${PAGE}. ` +
|
|
44
|
+
"Upload the workbook as saved." };
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
function parseUpload(_pack, req, files) {
|
|
48
|
+
const p = req.params || {};
|
|
49
|
+
const want = pyStr(p.sheet ?? "").trim() || pyStr(p.year ?? "").trim();
|
|
50
|
+
const rows = [];
|
|
51
|
+
const order = [];
|
|
52
|
+
for (const f of files) {
|
|
53
|
+
const data = fs.readFileSync(f);
|
|
54
|
+
const low = f.toLowerCase();
|
|
55
|
+
let sheets;
|
|
56
|
+
try {
|
|
57
|
+
sheets = low.endsWith(".xlsx") ? xlsxSheets(data) : xlsSheets(data);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
throw new DataError(`${path.basename(f)}: not a workbook (a saved web page or the empty challenge page?): open the ` +
|
|
61
|
+
"link in a web browser and save the file itself", "bad_request");
|
|
62
|
+
}
|
|
63
|
+
const names = sheetNames(sheets);
|
|
64
|
+
const name = names.find((n) => n.trim() === want) ?? names.find((n) => n.trim().toLowerCase() === want.toLowerCase());
|
|
65
|
+
if (!name) {
|
|
66
|
+
throw new DataError(`${path.basename(f)} has no sheet ${JSON.stringify(want)} (sheets: ${names.join(", ")}); give ` +
|
|
67
|
+
"the year of a sheet, or sheet", "bad_request");
|
|
68
|
+
}
|
|
69
|
+
const got = gridRecords({ sheet: name, header_row: Number(p.header_row ?? 1) }, sheets);
|
|
70
|
+
for (const c of columnsOf(got) || [])
|
|
71
|
+
if (!order.includes(c))
|
|
72
|
+
order.push(c);
|
|
73
|
+
rows.push(...got);
|
|
74
|
+
}
|
|
75
|
+
const out = rows.map((r) => Object.fromEntries(Object.entries(r).map(([k, v]) => [k, v === null || v === undefined ? "" : v])));
|
|
76
|
+
return order.length ? withColumns(out, order) : out;
|
|
77
|
+
}
|
|
78
|
+
const adapter = { refusedHosts, checkSpec, manualItems, parseUpload };
|
|
79
|
+
export default adapter;
|
|
80
|
+
//# sourceMappingURL=hud-ahar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hud-ahar.js","sourceRoot":"","sources":["../../src/adapters/hud-ahar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE5G,MAAM,IAAI,GAAG,4GAA4G,CAAC;AAC1H,MAAM,OAAO,GAAG,sGAAsG;IACpH,gHAAgH;IAChH,0CAA0C,CAAC;AAE7C,MAAM,YAAY,GAAG,CAAC,KAAW,EAAU,EAAE,CAAC,OAAO,CAAC;AAEtD,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAChF,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACnJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxE,MAAM,KAAK,GAA2B,IAAI,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;IAClE,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5E,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC;QAC7B,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;YACnH,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;YAC3C,IAAI,EAAE,0FAA0F,IAAI,IAAI;gBACtG,+BAA+B,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,KAAW,EAAE,GAAgB,EAAE,KAAe;IACjE,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACvE,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5B,IAAI,MAA2C,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,6EAA6E;gBAClH,gDAAgD,EAAE,aAAa,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU;gBACjH,+BAA+B,EAAE,aAAa,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACxF,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChI,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAC/E,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* National Center for Education Statistics: Digest of Education Statistics tables (Excel) and IPEDS complete data
|
|
3
|
+
* files (zipped CSV), read by core (published-table reader, $zip with "prefer" for the revised _rv file).
|
|
4
|
+
* Verified 2026-11-07: nces.ed.gov serves both to scripts; Digest files are
|
|
5
|
+
* /programs/digest/d<yy>/tables/xls/tabn<table>.xlsx (not every table is in every edition: a missing one is 404).
|
|
6
|
+
* This adapter checks the spec and filters IPEDS rows by UNITID or state.
|
|
7
|
+
*/
|
|
8
|
+
import type { Adapter } from "../core/packs.js";
|
|
9
|
+
declare const adapter: Adapter;
|
|
10
|
+
export default adapter;
|