data-analyst-mcp 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -6
- package/dist/adapters/_census_micro.js +2 -2
- package/dist/adapters/_census_micro.js.map +1 -1
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/analysis/compute.d.ts +22 -10
- package/dist/analysis/compute.js +75 -16
- package/dist/analysis/compute.js.map +1 -1
- package/dist/analysis/expr.d.ts +2 -0
- package/dist/analysis/expr.js +17 -0
- package/dist/analysis/expr.js.map +1 -1
- package/dist/analysis/replicate.d.ts +15 -0
- package/dist/analysis/replicate.js +228 -0
- package/dist/analysis/replicate.js.map +1 -0
- package/dist/analysis/rerun.js +2 -1
- package/dist/analysis/rerun.js.map +1 -1
- package/dist/analysis/table.d.ts +14 -0
- package/dist/analysis/table.js +79 -28
- package/dist/analysis/table.js.map +1 -1
- package/dist/analysis/tools.js +16 -4
- package/dist/analysis/tools.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/packs.d.ts +14 -5
- package/dist/core/packs.js.map +1 -1
- package/dist/{deep/_opm_parquet.d.ts → core/parquet.d.ts} +2 -1
- package/dist/{deep/_opm_parquet.js → core/parquet.js} +7 -5
- package/dist/core/parquet.js.map +1 -0
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/deep/_opm_io.js +2 -2
- package/dist/deep/_opm_io.js.map +1 -1
- package/dist/deep/bea.js +1 -0
- package/dist/deep/bea.js.map +1 -1
- package/dist/deep/fred.js +40 -9
- package/dist/deep/fred.js.map +1 -1
- package/dist/deep/ipums.js +1 -0
- package/dist/deep/ipums.js.map +1 -1
- package/dist/deep/opm.js +1 -1
- package/dist/deep/opm.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +3 -0
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.js +2 -13
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/report/tools.js +2 -1
- package/dist/report/tools.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +53 -17
- package/dist/server.js.map +1 -1
- package/dist/tools.d.ts +3 -1
- package/dist/tools.js +82 -9
- package/dist/tools.js.map +1 -1
- package/guide/WORKFLOW.md +6 -5
- package/guide/assets/download_data.ps1 +2 -0
- package/guide/assets/download_data.sh +8 -2
- package/guide/references/report_patterns.md +5 -2
- package/guide/references/report_schema.md +140 -0
- package/guide/references/routing.md +30 -3
- package/package.json +1 -1
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/census/source.md +8 -4
- package/packs/college-scorecard/source.md +1 -1
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/source.md +1 -1
- package/packs/fbi/source.md +3 -3
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fred/source.md +2 -1
- package/packs/fsa/source.md +1 -1
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/ipums/docs/report_patterns.md +1 -1
- package/packs/ipums/docs/variables.md +4 -3
- package/packs/ipums/source.md +10 -8
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/source.md +1 -1
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/opm/docs/opm_api.md +2 -1
- package/packs/opm/source.md +2 -1
- package/packs/port.json +65 -0
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/treasury/source.md +1 -1
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/uspto/source.md +1 -1
- package/dist/deep/_opm_parquet.js.map +0 -1
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FHWA Highway Statistics tables: /policyinformation/statistics/<year>/xls/<table>.xlsx, read by core's published-
|
|
3
|
+
* table reader (verified 2026-11-07 for 2023 and 2024: vm2, dl22, mv1, hm20, mf1, fe9; mf2 is not a 2023 file).
|
|
4
|
+
* Several workbooks carry a hidden report-tool sheet (CRYSTAL_PERSIST) first; skip_sheets leaves it out of the
|
|
5
|
+
* sheet numbering. This adapter only checks the spec.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
function checkSpec(spec) {
|
|
3
|
+
const bad = [];
|
|
4
|
+
for (const r of spec.requests || []) {
|
|
5
|
+
const p = r.params || {};
|
|
6
|
+
if (p.year !== undefined && !/^(19|20)\d{2}$/.test(pyStr(p.year)))
|
|
7
|
+
bad.push(`request ${JSON.stringify(r.name)}: year is the edition year, e.g. 2023`);
|
|
8
|
+
if (p.table !== undefined && !/^[a-z]{1,3}\d{1,3}[a-z]?$/.test(pyStr(p.table))) {
|
|
9
|
+
bad.push(`request ${JSON.stringify(r.name)}: table is the id in lower case without the hyphen (VM-2 -> vm2)`);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
return bad;
|
|
13
|
+
}
|
|
14
|
+
const adapter = { checkSpec };
|
|
15
|
+
export default adapter;
|
|
16
|
+
//# sourceMappingURL=fhwa.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fhwa.js","sourceRoot":"","sources":["../../src/adapters/fhwa.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACtJ,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kEAAkE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,CAAC;AACvC,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ForeignAssistance.gov aggregate data files (the Data page's CSV downloads under /static/, refreshed with the
|
|
3
|
+
* site). Manual only: on 2026-11-07 foreignassistance.gov answered this server's requests with HTTP 403 (its
|
|
4
|
+
* documented Data API at /data-api/ answered 404), so refusedHosts sends fetch to manual_bundle; the user
|
|
5
|
+
* downloads the CSV in a browser and ingest filters it here by country, fiscal year and transaction type.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
4
|
+
const REFUSED = "foreignassistance.gov answers this server with HTTP 403 (verified 2026-11-07), so its files cannot " +
|
|
5
|
+
"be fetched from here: use manual_bundle; the user downloads the CSV from https://foreignassistance.gov/data " +
|
|
6
|
+
"(Data Download) in a web browser and uploads it; then ingest applies the filters";
|
|
7
|
+
const refusedHosts = (_spec) => REFUSED;
|
|
8
|
+
function checkSpec(spec) {
|
|
9
|
+
const bad = [];
|
|
10
|
+
for (const r of spec.requests || []) {
|
|
11
|
+
const p = r.params || {};
|
|
12
|
+
for (const k of ["from", "to"]) {
|
|
13
|
+
if (p[k] !== undefined && p[k] !== "" && !/^\d{4}$/.test(pyStr(p[k])))
|
|
14
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} is a fiscal year, e.g. 2024`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return bad;
|
|
18
|
+
}
|
|
19
|
+
function normalize(req, _ep, rows) {
|
|
20
|
+
const p = req.params || {};
|
|
21
|
+
const countries = list(p.country), years = list(p.fiscal_year), types = list(p.transaction_type);
|
|
22
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
23
|
+
return rows.filter((r) => {
|
|
24
|
+
const fy = Number(pyStr(r["Fiscal Year"] ?? ""));
|
|
25
|
+
if (countries.length && !countries.includes(pyStr(r["Country Code"] ?? "").toLowerCase()) &&
|
|
26
|
+
!countries.includes(pyStr(r["Country Name"] ?? "").toLowerCase()))
|
|
27
|
+
return false;
|
|
28
|
+
if (years.length && !years.includes(pyStr(r["Fiscal Year"] ?? "").toLowerCase()))
|
|
29
|
+
return false;
|
|
30
|
+
if (from !== null && !(fy >= from))
|
|
31
|
+
return false;
|
|
32
|
+
if (to !== null && !(fy <= to))
|
|
33
|
+
return false;
|
|
34
|
+
if (types.length && !types.includes(pyStr(r["Transaction Type Name"] ?? "").toLowerCase()) &&
|
|
35
|
+
!types.includes(pyStr(r["Transaction Type ID"] ?? "").toLowerCase()))
|
|
36
|
+
return false;
|
|
37
|
+
return true;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
const adapter = { refusedHosts, checkSpec, normalize };
|
|
41
|
+
export default adapter;
|
|
42
|
+
//# sourceMappingURL=foreign-assistance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"foreign-assistance.js","sourceRoot":"","sources":["../../src/adapters/foreign-assistance.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,MAAM,OAAO,GAAG,qGAAqG;IACnH,8GAA8G;IAC9G,kFAAkF,CAAC;AAErF,MAAM,YAAY,GAAG,CAAC,KAAW,EAAU,EAAE,CAAC,OAAO,CAAC;AAEtD,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACzJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACjG,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACjD,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACvF,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAClF,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/F,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACjD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,uBAAuB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACxF,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACrF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAChE,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HUD AHAR data files: Point-in-Time (PIT) counts and Housing Inventory Counts (HIC), by CoC and by state, from
|
|
3
|
+
* the 2025 AHAR Part 1 page on HUD User. Manual only.
|
|
4
|
+
*
|
|
5
|
+
* www.huduser.gov answers scripts with HTTP 202 and an empty body (a WAF challenge; verified 2026-11-07 on the
|
|
6
|
+
* AHAR pages and the .xlsb/.xlsx files), so refusedHosts sends every request to manual_bundle. The file names come
|
|
7
|
+
* from the 2025 edition's page (verified 2026-11-07). parseUpload reads the saved workbook:
|
|
8
|
+
* - .xlsb (binary, the PIT files) and .xls through SheetJS, .xlsx with the core reader;
|
|
9
|
+
* - the sheet named for the requested year (or `sheet`), header at `header_row` (default 1);
|
|
10
|
+
* - columns as published.
|
|
11
|
+
* The workbook layout was not checked against a downloaded copy (the files cannot be fetched from here): a
|
|
12
|
+
* missing sheet fails with the list of sheets, and source.md says to read the columns before use.
|
|
13
|
+
*/
|
|
14
|
+
import fs from "node:fs";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { DataError } from "../core/http.js";
|
|
17
|
+
import { pyStr } from "../core/py.js";
|
|
18
|
+
import { columnsOf, gridRecords, sheetNames, withColumns, xlsSheets, xlsxSheets } from "../core/tabular.js";
|
|
19
|
+
const PAGE = "https://www.huduser.gov/portal/datasets/ahar/2025-ahar-part-1-pit-estimates-of-homelessness-in-the-us.html";
|
|
20
|
+
const REFUSED = "www.huduser.gov answers scripts with an empty HTTP 202 challenge (verified 2026-11-07), so the AHAR " +
|
|
21
|
+
"workbooks cannot be downloaded from here: use manual_bundle; the user opens the links in a web browser, saves " +
|
|
22
|
+
"the workbook and uploads it; then ingest";
|
|
23
|
+
const refusedHosts = (_spec) => REFUSED;
|
|
24
|
+
function checkSpec(spec) {
|
|
25
|
+
const bad = [];
|
|
26
|
+
for (const r of spec.requests || []) {
|
|
27
|
+
const p = r.params || {};
|
|
28
|
+
if (!/^(19|20)\d{2}$/.test(pyStr(p.year ?? "")) && !pyStr(p.sheet ?? "").trim()) {
|
|
29
|
+
bad.push(`request ${JSON.stringify(r.name)}: give year (the count year, e.g. 2025) or sheet`);
|
|
30
|
+
}
|
|
31
|
+
if (p.header_row !== undefined && !/^\d+$/.test(pyStr(p.header_row)))
|
|
32
|
+
bad.push(`request ${JSON.stringify(r.name)}: header_row is a line number`);
|
|
33
|
+
}
|
|
34
|
+
return bad;
|
|
35
|
+
}
|
|
36
|
+
function manualItems(spec, pack) {
|
|
37
|
+
const base = String(pack.endpoints?.base_url || "").replace(/\/+$/, "");
|
|
38
|
+
const files = pack.endpoints?.files || {};
|
|
39
|
+
return (spec.requests || []).filter((r) => files[r.endpoint || ""]).map((r) => {
|
|
40
|
+
const f = files[r.endpoint];
|
|
41
|
+
return { name: `${r.name}#1`, request: r.name, url: `${base}/${f}`, filename: f, also_ext: [".xlsb", ".xlsx", ".xls"],
|
|
42
|
+
needs_key: false, method: "GET", body: null,
|
|
43
|
+
note: `open in a web browser (huduser.gov refuses scripts); if the link has moved, find it on ${PAGE}. ` +
|
|
44
|
+
"Upload the workbook as saved." };
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
function parseUpload(_pack, req, files) {
|
|
48
|
+
const p = req.params || {};
|
|
49
|
+
const want = pyStr(p.sheet ?? "").trim() || pyStr(p.year ?? "").trim();
|
|
50
|
+
const rows = [];
|
|
51
|
+
const order = [];
|
|
52
|
+
for (const f of files) {
|
|
53
|
+
const data = fs.readFileSync(f);
|
|
54
|
+
const low = f.toLowerCase();
|
|
55
|
+
let sheets;
|
|
56
|
+
try {
|
|
57
|
+
sheets = low.endsWith(".xlsx") ? xlsxSheets(data) : xlsSheets(data);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
throw new DataError(`${path.basename(f)}: not a workbook (a saved web page or the empty challenge page?): open the ` +
|
|
61
|
+
"link in a web browser and save the file itself", "bad_request");
|
|
62
|
+
}
|
|
63
|
+
const names = sheetNames(sheets);
|
|
64
|
+
const name = names.find((n) => n.trim() === want) ?? names.find((n) => n.trim().toLowerCase() === want.toLowerCase());
|
|
65
|
+
if (!name) {
|
|
66
|
+
throw new DataError(`${path.basename(f)} has no sheet ${JSON.stringify(want)} (sheets: ${names.join(", ")}); give ` +
|
|
67
|
+
"the year of a sheet, or sheet", "bad_request");
|
|
68
|
+
}
|
|
69
|
+
const got = gridRecords({ sheet: name, header_row: Number(p.header_row ?? 1) }, sheets);
|
|
70
|
+
for (const c of columnsOf(got) || [])
|
|
71
|
+
if (!order.includes(c))
|
|
72
|
+
order.push(c);
|
|
73
|
+
rows.push(...got);
|
|
74
|
+
}
|
|
75
|
+
const out = rows.map((r) => Object.fromEntries(Object.entries(r).map(([k, v]) => [k, v === null || v === undefined ? "" : v])));
|
|
76
|
+
return order.length ? withColumns(out, order) : out;
|
|
77
|
+
}
|
|
78
|
+
const adapter = { refusedHosts, checkSpec, manualItems, parseUpload };
|
|
79
|
+
export default adapter;
|
|
80
|
+
//# sourceMappingURL=hud-ahar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hud-ahar.js","sourceRoot":"","sources":["../../src/adapters/hud-ahar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE5G,MAAM,IAAI,GAAG,4GAA4G,CAAC;AAC1H,MAAM,OAAO,GAAG,sGAAsG;IACpH,gHAAgH;IAChH,0CAA0C,CAAC;AAE7C,MAAM,YAAY,GAAG,CAAC,KAAW,EAAU,EAAE,CAAC,OAAO,CAAC;AAEtD,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAChF,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACnJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,WAAW,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACxE,MAAM,KAAK,GAA2B,IAAI,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE,CAAC;IAClE,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5E,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAS,CAAC,CAAC;QAC7B,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;YACnH,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;YAC3C,IAAI,EAAE,0FAA0F,IAAI,IAAI;gBACtG,+BAA+B,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW,CAAC,KAAW,EAAE,GAAgB,EAAE,KAAe;IACjE,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACvE,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5B,IAAI,MAA2C,CAAC;QAChD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,6EAA6E;gBAClH,gDAAgD,EAAE,aAAa,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU;gBACjH,+BAA+B,EAAE,aAAa,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACxF,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChI,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACtD,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAC/E,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* National Center for Education Statistics: Digest of Education Statistics tables (Excel) and IPEDS complete data
|
|
3
|
+
* files (zipped CSV), read by core (published-table reader, $zip with "prefer" for the revised _rv file).
|
|
4
|
+
* Verified 2026-11-07: nces.ed.gov serves both to scripts; Digest files are
|
|
5
|
+
* /programs/digest/d<yy>/tables/xls/tabn<table>.xlsx (not every table is in every edition: a missing one is 404).
|
|
6
|
+
* This adapter checks the spec and filters IPEDS rows by UNITID or state.
|
|
7
|
+
*/
|
|
8
|
+
import type { Adapter } from "../core/packs.js";
|
|
9
|
+
declare const adapter: Adapter;
|
|
10
|
+
export default adapter;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toUpperCase()).filter(Boolean);
|
|
4
|
+
function checkSpec(spec) {
|
|
5
|
+
const bad = [];
|
|
6
|
+
for (const r of spec.requests || []) {
|
|
7
|
+
const p = r.params || {};
|
|
8
|
+
const tag = `request ${JSON.stringify(r.name)}`;
|
|
9
|
+
if (r.endpoint === "digest_table") {
|
|
10
|
+
if (p.edition !== undefined && !/^\d{2}$/.test(pyStr(p.edition)))
|
|
11
|
+
bad.push(`${tag}: edition is two digits, e.g. 25`);
|
|
12
|
+
if (p.table !== undefined && !/^\d{3}\.\d{2,3}$/.test(pyStr(p.table)))
|
|
13
|
+
bad.push(`${tag}: table looks like 203.10`);
|
|
14
|
+
}
|
|
15
|
+
if ((r.endpoint === "ipeds" || r.endpoint === "ipeds_dictionary") && p.file !== undefined && !/^[A-Za-z0-9_]+$/.test(pyStr(p.file))) {
|
|
16
|
+
bad.push(`${tag}: file is an IPEDS file name such as HD2023 or C2023_A`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return bad;
|
|
20
|
+
}
|
|
21
|
+
function normalize(req, _ep, rows) {
|
|
22
|
+
if (req.endpoint !== "ipeds")
|
|
23
|
+
return rows;
|
|
24
|
+
const ids = list(req.params?.unitid), states = list(req.params?.state);
|
|
25
|
+
if (!ids.length && !states.length)
|
|
26
|
+
return rows;
|
|
27
|
+
return rows.filter((r) => (!ids.length || ids.includes(pyStr(r.UNITID ?? "").trim())) &&
|
|
28
|
+
(!states.length || states.includes(pyStr(r.STABBR ?? "").trim().toUpperCase())));
|
|
29
|
+
}
|
|
30
|
+
const adapter = { checkSpec, normalize };
|
|
31
|
+
export default adapter;
|
|
32
|
+
//# sourceMappingURL=nces.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nces.js","sourceRoot":"","sources":["../../src/adapters/nces.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;YAClC,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,kCAAkC,CAAC,CAAC;YACrH,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,2BAA2B,CAAC,CAAC;QACrH,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACpI,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,wDAAwD,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NSF Award Search API. Verified 2026-11-07:
|
|
3
|
+
* - offset is 1-based (the default sets it), rpp is at most 25, and response.metadata.totalCount gives the
|
|
4
|
+
* total; printFields is ignored, so every field comes back.
|
|
5
|
+
* - Date filters are MM/DD/YYYY.
|
|
6
|
+
* normalize drops the people's contact fields (PI and program officer e-mail and phone): names are public award
|
|
7
|
+
* data, personal contact details are not needed for analysis. Amounts stay as published text.
|
|
8
|
+
*/
|
|
9
|
+
import type { Adapter } from "../core/packs.js";
|
|
10
|
+
declare const adapter: Adapter;
|
|
11
|
+
export default adapter;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const CONTACT = new Set(["piEmail", "poEmail", "poPhone", "awardeePhone"]);
|
|
3
|
+
const DATES = ["dateStart", "dateEnd", "startDateStart", "startDateEnd", "expDateStart", "expDateEnd"];
|
|
4
|
+
const FILTERS = ["keyword", "awardeeName", "awardeeStateCode", "perfStateCode", "pdPIName", "fundProgramName", "id", ...DATES];
|
|
5
|
+
function checkSpec(spec) {
|
|
6
|
+
const bad = [];
|
|
7
|
+
for (const r of spec.requests || []) {
|
|
8
|
+
const p = r.params || {};
|
|
9
|
+
for (const k of DATES) {
|
|
10
|
+
if (p[k] !== undefined && p[k] !== "" && !/^\d{2}\/\d{2}\/\d{4}$/.test(pyStr(p[k]))) {
|
|
11
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} must be MM/DD/YYYY (got ${JSON.stringify(p[k])})`);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
if (!FILTERS.some((k) => p[k] !== undefined && p[k] !== "")) {
|
|
15
|
+
bad.push(`request ${JSON.stringify(r.name)}: give at least one filter (${FILTERS.join(", ")}); all NSF awards is too many`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return bad;
|
|
19
|
+
}
|
|
20
|
+
function normalize(_req, _ep, rows) {
|
|
21
|
+
return rows.map((r) => Object.fromEntries(Object.entries(r).filter(([k]) => !CONTACT.has(k))));
|
|
22
|
+
}
|
|
23
|
+
const adapter = { checkSpec, normalize };
|
|
24
|
+
export default adapter;
|
|
25
|
+
//# sourceMappingURL=nsf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nsf.js","sourceRoot":"","sources":["../../src/adapters/nsf.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;AAC3E,MAAM,KAAK,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;AACvG,MAAM,OAAO,GAAG,CAAC,SAAS,EAAE,aAAa,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;AAE/H,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpF,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACvG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9H,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,IAAiB,EAAE,GAAQ,EAAE,IAAW;IACzD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Office of Natural Resources Revenue data downloads (revenuedata.onrr.gov/downloads/<file>.csv; revenuedata.doi.gov
|
|
3
|
+
* redirects there). Verified 2026-11-07: revenue (fiscal, calendar, monthly), disbursements (fiscal) and production
|
|
4
|
+
* (fiscal, calendar) as CSV. Files are read whole and filtered here (state, years, commodity, revenue type, land
|
|
5
|
+
* class, fund type, product), values as published.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
4
|
+
const FILTERS = [["state", "State"], ["commodity", "Commodity"], ["revenue_type", "Revenue Type"],
|
|
5
|
+
["land_class", "Land Class"], ["fund_type", "Fund Type"], ["product", "Product"]];
|
|
6
|
+
function checkSpec(spec) {
|
|
7
|
+
const bad = [];
|
|
8
|
+
for (const r of spec.requests || []) {
|
|
9
|
+
for (const k of ["from", "to"]) {
|
|
10
|
+
const v = r.params?.[k];
|
|
11
|
+
if (v !== undefined && v !== "" && !/^\d{4}$/.test(pyStr(v)))
|
|
12
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} is a year, e.g. 2020`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return bad;
|
|
16
|
+
}
|
|
17
|
+
function normalize(req, _ep, rows) {
|
|
18
|
+
const p = req.params || {};
|
|
19
|
+
const wants = FILTERS.map(([k, col]) => [col, list(p[k])]).filter(([, v]) => v.length);
|
|
20
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
21
|
+
if (!wants.length && from === null && to === null)
|
|
22
|
+
return rows;
|
|
23
|
+
return rows.filter((r) => {
|
|
24
|
+
// monthly_revenue has Date as MM/DD/YYYY (verified 2026-11-07)
|
|
25
|
+
const y = Number(pyStr(r["Fiscal Year"] ?? r["Calendar Year"] ?? (/(\d{4})$/.exec(pyStr(r.Date ?? "")) || ["", ""])[1]));
|
|
26
|
+
if (from !== null && !(y >= from))
|
|
27
|
+
return false;
|
|
28
|
+
if (to !== null && !(y <= to))
|
|
29
|
+
return false;
|
|
30
|
+
return wants.every(([col, v]) => v.includes(pyStr(r[col] ?? "").trim().toLowerCase()));
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const adapter = { checkSpec, normalize };
|
|
34
|
+
export default adapter;
|
|
35
|
+
//# sourceMappingURL=onrr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onrr.js","sourceRoot":"","sources":["../../src/adapters/onrr.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC7D,MAAM,OAAO,GAAuB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,cAAc,EAAE,cAAc,CAAC;IACnH,CAAC,YAAY,EAAE,YAAY,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpF,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACzI,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7G,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/D,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,+DAA+D;QAC/D,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzH,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAChD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
import type { Adapter, Row } from "../core/packs.js";
|
|
3
|
+
/** The row filter of a foia request. */
|
|
4
|
+
export declare function keeper(p: Record<string, unknown>): (r: Row) => boolean;
|
|
5
|
+
/** Parse a CSV stream, keeping the rows that pass `keep` (the header names the columns). [rows, cut at max] */
|
|
6
|
+
export declare function streamRows(src: Readable, keep: (r: Row) => boolean, max: number): Promise<[Row[], boolean]>;
|
|
7
|
+
/** One row per file in data.sba.gov's catalog (data.json); `file` is set for CSVs (the foia request's path). */
|
|
8
|
+
export declare function catalogRows(cat: any, search: string): Row[];
|
|
9
|
+
declare const adapter: Adapter;
|
|
10
|
+
export default adapter;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U.S. Small Business Administration loan data on data.sba.gov. Verified 2026-11-07:
|
|
3
|
+
* - the CKAN API answers 404, but /data.json (the open-data catalog) lists every file;
|
|
4
|
+
* - the 7(a) and 504 FOIA CSVs are under /sites/default/files/uploaded_resources/, named with their as-of date
|
|
5
|
+
* (FOIA_504_FY2010_Present_asof_260930.csv) and 60 to 255 MB each.
|
|
6
|
+
* list_files reads the catalog. foia streams the CSV through the parser and keeps only the rows that pass the
|
|
7
|
+
* filters, so a 255 MB file never becomes a full table in memory; a filter is required.
|
|
8
|
+
*/
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import { Readable } from "node:stream";
|
|
11
|
+
import { parse } from "csv-parse";
|
|
12
|
+
import * as gr from "../core/generic-rest.js";
|
|
13
|
+
import { DataError } from "../core/http.js";
|
|
14
|
+
import { pyStr } from "../core/py.js";
|
|
15
|
+
import { columnsOf, withColumns } from "../core/tabular.js";
|
|
16
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
17
|
+
.map((x) => pyStr(x).trim().toUpperCase()).filter(Boolean);
|
|
18
|
+
/** The row filter of a foia request. */
|
|
19
|
+
export function keeper(p) {
|
|
20
|
+
const states = list(p.state), years = list(p.fiscal_year), naics = list(p.naics);
|
|
21
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
22
|
+
return (r) => {
|
|
23
|
+
if (states.length && !states.includes(pyStr(r.ProjectState ?? "").trim().toUpperCase()))
|
|
24
|
+
return false;
|
|
25
|
+
const fy = pyStr(r.ApprovalFY ?? "").trim();
|
|
26
|
+
if (years.length && !years.includes(fy))
|
|
27
|
+
return false;
|
|
28
|
+
if (from !== null && !(Number(fy) >= from))
|
|
29
|
+
return false;
|
|
30
|
+
if (to !== null && !(Number(fy) <= to))
|
|
31
|
+
return false;
|
|
32
|
+
if (naics.length && !naics.some((n) => pyStr(r.NaicsCode ?? "").trim().startsWith(n)))
|
|
33
|
+
return false;
|
|
34
|
+
return true;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/** Parse a CSV stream, keeping the rows that pass `keep` (the header names the columns). [rows, cut at max] */
|
|
38
|
+
export async function streamRows(src, keep, max) {
|
|
39
|
+
const out = [];
|
|
40
|
+
let cols = [];
|
|
41
|
+
let cut = false;
|
|
42
|
+
const parser = src.pipe(parse({ bom: true, relax_column_count: true, relax_quotes: true, skip_empty_lines: true }));
|
|
43
|
+
for await (const rec of parser) {
|
|
44
|
+
if (!cols.length) {
|
|
45
|
+
cols = rec;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const r = {};
|
|
49
|
+
cols.forEach((c, i) => (r[c] = i < rec.length ? rec[i] : ""));
|
|
50
|
+
if (!keep(r))
|
|
51
|
+
continue;
|
|
52
|
+
if (out.length >= max) {
|
|
53
|
+
cut = true;
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
out.push(r);
|
|
57
|
+
}
|
|
58
|
+
return [withColumns(out, cols), cut];
|
|
59
|
+
}
|
|
60
|
+
function checkSpec(spec) {
|
|
61
|
+
const bad = [];
|
|
62
|
+
for (const r of spec.requests || []) {
|
|
63
|
+
const p = r.params || {};
|
|
64
|
+
if (r.endpoint !== "foia")
|
|
65
|
+
continue;
|
|
66
|
+
if (!["state", "fiscal_year", "from", "naics"].some((k) => p[k] !== undefined && p[k] !== "")) {
|
|
67
|
+
bad.push(`request ${JSON.stringify(r.name)}: give state, fiscal_year, from or naics (the files hold hundreds of thousands of loans)`);
|
|
68
|
+
}
|
|
69
|
+
if (p.file !== undefined && (!/^[\w./-]+$/.test(pyStr(p.file)) || pyStr(p.file).includes(".."))) {
|
|
70
|
+
bad.push(`request ${JSON.stringify(r.name)}: file is a path from list_files`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return bad;
|
|
74
|
+
}
|
|
75
|
+
/** One row per file in data.sba.gov's catalog (data.json); `file` is set for CSVs (the foia request's path). */
|
|
76
|
+
export function catalogRows(cat, search) {
|
|
77
|
+
const out = [];
|
|
78
|
+
const want = search.trim().toLowerCase();
|
|
79
|
+
for (const d of cat?.dataset || []) {
|
|
80
|
+
for (const x of d.distribution || []) {
|
|
81
|
+
const url = pyStr(x.downloadURL || x.accessURL || "");
|
|
82
|
+
const m = /^https:\/\/data\.sba\.gov\/sites\/default\/files\/(.+?)(\.csv)?$/i.exec(url);
|
|
83
|
+
const row = { dataset: pyStr(d.title ?? ""), title: pyStr(x.title ?? ""), format: pyStr(x.format || x.mediaType || ""),
|
|
84
|
+
file: m && m[2] ? m[1] : "", url, modified: pyStr(d.modified ?? "").slice(0, 10) };
|
|
85
|
+
if (want && !`${row.dataset} ${row.title}`.toLowerCase().includes(want))
|
|
86
|
+
continue;
|
|
87
|
+
out.push(row);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
async function fetch(spec, pack) {
|
|
93
|
+
const c = gr.clientFor(pack);
|
|
94
|
+
const tables = {};
|
|
95
|
+
const notes = [];
|
|
96
|
+
for (const r of spec.requests) {
|
|
97
|
+
const [ep, params, probs] = gr.resolve(pack, r);
|
|
98
|
+
if (probs.length)
|
|
99
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
100
|
+
if (r.endpoint === "list_files") {
|
|
101
|
+
const resp = await c.request("GET", pack.endpoints.catalog_path, null);
|
|
102
|
+
tables[r.name] = catalogRows(resp.json(), pyStr(params.search ?? ""));
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const [p] = gr.split(ep, params);
|
|
106
|
+
const resp = await c.request("GET", p, null);
|
|
107
|
+
const max = Number(r.max_rows || ep.max_rows || 1000000);
|
|
108
|
+
const [rows, cut] = await streamRows(Readable.from([resp.body]), keeper(params), max);
|
|
109
|
+
if (cut)
|
|
110
|
+
notes.push(`${r.name}: stopped at ${rows.length.toLocaleString("en-US")} rows (max_rows); the source has more. Narrow the filter or raise max_rows.`);
|
|
111
|
+
if (!rows.length && !r.allow_empty) {
|
|
112
|
+
throw new DataError(`request '${r.name}' returned no rows: check the filters (ProjectState is two letters, ApprovalFY a year)`, "bad_request");
|
|
113
|
+
}
|
|
114
|
+
tables[r.name] = rows;
|
|
115
|
+
}
|
|
116
|
+
return [tables, notes];
|
|
117
|
+
}
|
|
118
|
+
async function parseUpload(pack, req, files) {
|
|
119
|
+
if (req.endpoint !== "foia")
|
|
120
|
+
return gr.parseUpload(pack, req, files);
|
|
121
|
+
const [, params] = gr.resolve(pack, req);
|
|
122
|
+
const rows = [];
|
|
123
|
+
let cols = [];
|
|
124
|
+
for (const f of [...files].sort()) {
|
|
125
|
+
const [got] = await streamRows(fs.createReadStream(f), keeper(params), 1000000);
|
|
126
|
+
if (!cols.length)
|
|
127
|
+
cols = columnsOf(got) || [];
|
|
128
|
+
rows.push(...got);
|
|
129
|
+
}
|
|
130
|
+
return withColumns(rows, cols);
|
|
131
|
+
}
|
|
132
|
+
const adapter = { fetch, checkSpec, parseUpload };
|
|
133
|
+
export default adapter;
|
|
134
|
+
//# sourceMappingURL=sba.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sba.js","sourceRoot":"","sources":["../../src/adapters/sba.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE5D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAA0B;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,OAAO,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACzD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACpG,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,+GAA+G;AAC/G,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAa,EAAE,IAAyB,EAAE,GAAW;IACpF,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACpH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAiC,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,GAAG,GAAG,CAAC;YACX,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAQ,EAAE,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,SAAS;QACvB,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YACtB,GAAG,GAAG,IAAI,CAAC;YACX,MAAM;QACR,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,QAAQ,KAAK,MAAM;YAAE,SAAS;QACpC,IAAI,CAAC,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YAC9F,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;QACxI,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAChG,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gHAAgH;AAChH,MAAM,UAAU,WAAW,CAAC,GAAQ,EAAE,MAAc;IAClD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,GAAG,mEAAmE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;gBACpH,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrF,IAAI,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAClF,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC,CAAC;QACvE,IAAI,CAAC,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YACvE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC;QACzD,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QACtF,IAAI,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,6EAA6E,CAAC,CAAC;QAC/J,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,IAAI,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,wFAAwF,EAAE,aAAa,CAAC,CAAC;QACjJ,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,GAAgB,EAAE,KAAe;IACtE,IAAI,GAAG,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;QAChF,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AAC3D,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Treasury International Capital (TIC): published tab-delimited text tables, read by the generic $tsv reader and
|
|
3
|
+
* made long here (one row per country and month), values as published.
|
|
4
|
+
*
|
|
5
|
+
* major_foreign_holders slt_table5.txt: "Country" then one column per month (2026-08 ...), billions
|
|
6
|
+
* major_foreign_holders_history mfhhis01.txt: one block per year, a month-name line ("Dec Nov ...") above a
|
|
7
|
+
* "Country 2025 2025 ..." line, rows to "Grand Total", then "Of which:" rows
|
|
8
|
+
* holdings_by_country slt_table1.txt: machine column names on line 9, then rows; a definitions
|
|
9
|
+
* footer ("for_lt_corp_net: 19 + 23 ...") follows the data
|
|
10
|
+
* The spec-only params country, from and to filter the rows (the files are small, so the whole file is read).
|
|
11
|
+
*/
|
|
12
|
+
import type { Adapter, Row, SpecRequest } from "../core/packs.js";
|
|
13
|
+
/** A published row label without footnote marks ("Grand Total 1/" -> "Grand Total"). */
|
|
14
|
+
export declare function label(s: unknown): string;
|
|
15
|
+
/** What a row is: a country, the All Other remainder, the Grand Total or a foreign-official line. */
|
|
16
|
+
export declare function kind(name: string): string;
|
|
17
|
+
/**
|
|
18
|
+
* The stacked yearly blocks of mfhhis01.txt (rows col1..colN from $tsv with header_row 0).
|
|
19
|
+
*
|
|
20
|
+
* Two quirks of the published file (verified 2026-11-07):
|
|
21
|
+
* - A series-break year shows one month twice (June 2002-2011, March 2000): the first column is the new
|
|
22
|
+
* series, the second the previous series "shown for comparison only" (the file's footnotes). The second
|
|
23
|
+
* is kept only with include_comparison, marked comparison_only.
|
|
24
|
+
* - In those years January is a 14th column one line up: its header "Jan" sits above the month line, the
|
|
25
|
+
* month line holds the year there, and each row's January value is on the line above the row. It is read
|
|
26
|
+
* from the line above.
|
|
27
|
+
*/
|
|
28
|
+
export declare function history(req: SpecRequest, rows: Row[]): Row[];
|
|
29
|
+
declare const adapter: Adapter;
|
|
30
|
+
export default adapter;
|