data-analyst-mcp 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -4
- package/dist/adapters/bjs.d.ts +13 -0
- package/dist/adapters/bjs.js +28 -0
- package/dist/adapters/bjs.js.map +1 -0
- package/dist/adapters/cbp.d.ts +5 -0
- package/dist/adapters/cbp.js +64 -0
- package/dist/adapters/cbp.js.map +1 -0
- package/dist/adapters/cdc-wonder.d.ts +5 -0
- package/dist/adapters/cdc-wonder.js +68 -0
- package/dist/adapters/cdc-wonder.js.map +1 -0
- package/dist/adapters/cpsc.d.ts +4 -0
- package/dist/adapters/cpsc.js +125 -0
- package/dist/adapters/cpsc.js.map +1 -0
- package/dist/adapters/fhwa.d.ts +9 -0
- package/dist/adapters/fhwa.js +16 -0
- package/dist/adapters/fhwa.js.map +1 -0
- package/dist/adapters/foreign-assistance.d.ts +9 -0
- package/dist/adapters/foreign-assistance.js +42 -0
- package/dist/adapters/foreign-assistance.js.map +1 -0
- package/dist/adapters/hud-ahar.d.ts +3 -0
- package/dist/adapters/hud-ahar.js +80 -0
- package/dist/adapters/hud-ahar.js.map +1 -0
- package/dist/adapters/nces.d.ts +10 -0
- package/dist/adapters/nces.js +32 -0
- package/dist/adapters/nces.js.map +1 -0
- package/dist/adapters/nsf.d.ts +11 -0
- package/dist/adapters/nsf.js +25 -0
- package/dist/adapters/nsf.js.map +1 -0
- package/dist/adapters/onrr.d.ts +9 -0
- package/dist/adapters/onrr.js +35 -0
- package/dist/adapters/onrr.js.map +1 -0
- package/dist/adapters/sba.d.ts +10 -0
- package/dist/adapters/sba.js +134 -0
- package/dist/adapters/sba.js.map +1 -0
- package/dist/adapters/treasury-tic.d.ts +30 -0
- package/dist/adapters/treasury-tic.js +149 -0
- package/dist/adapters/treasury-tic.js.map +1 -0
- package/dist/adapters/uscourts.d.ts +8 -0
- package/dist/adapters/uscourts.js +8 -0
- package/dist/adapters/uscourts.js.map +1 -0
- package/dist/analysis/cite.d.ts +9 -1
- package/dist/analysis/cite.js +9 -2
- package/dist/analysis/cite.js.map +1 -1
- package/dist/bin/cli.d.ts +2 -0
- package/dist/bin/cli.js +17 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/core/generic-rest.js +42 -17
- package/dist/core/generic-rest.js.map +1 -1
- package/dist/core/guard.d.ts +32 -0
- package/dist/core/guard.js +176 -0
- package/dist/core/guard.js.map +1 -0
- package/dist/core/http.d.ts +10 -0
- package/dist/core/http.js +36 -7
- package/dist/core/http.js.map +1 -1
- package/dist/core/keys.d.ts +13 -3
- package/dist/core/keys.js +67 -7
- package/dist/core/keys.js.map +1 -1
- package/dist/core/package.d.ts +12 -3
- package/dist/core/package.js +56 -12
- package/dist/core/package.js.map +1 -1
- package/dist/core/secret_patterns.d.ts +6 -0
- package/dist/core/secret_patterns.js +20 -0
- package/dist/core/secret_patterns.js.map +1 -0
- package/dist/core/tabular.d.ts +43 -1
- package/dist/core/tabular.js +149 -6
- package/dist/core/tabular.js.map +1 -1
- package/dist/envelope.js +3 -1
- package/dist/envelope.js.map +1 -1
- package/dist/index.d.ts +29 -22
- package/dist/index.js +45 -46
- package/dist/index.js.map +1 -1
- package/dist/remote/app.js +87 -31
- package/dist/remote/app.js.map +1 -1
- package/dist/remote/oauth.js +8 -1
- package/dist/remote/oauth.js.map +1 -1
- package/dist/remote/pages.d.ts +3 -7
- package/dist/remote/pages.js +49 -19
- package/dist/remote/pages.js.map +1 -1
- package/dist/remote/sign.d.ts +7 -1
- package/dist/remote/sign.js +33 -7
- package/dist/remote/sign.js.map +1 -1
- package/dist/remote/store.js +14 -5
- package/dist/remote/store.js.map +1 -1
- package/dist/report/audit_report.js +29 -7
- package/dist/report/audit_report.js.map +1 -1
- package/dist/report/build_report.js +34 -4
- package/dist/report/build_report.js.map +1 -1
- package/dist/report/method_audit.js +3 -0
- package/dist/report/method_audit.js.map +1 -1
- package/dist/report/scan_secrets.js +2 -13
- package/dist/report/scan_secrets.js.map +1 -1
- package/dist/server.d.ts +3 -1
- package/dist/server.js +44 -12
- package/dist/server.js.map +1 -1
- package/dist/tools.js +27 -6
- package/dist/tools.js.map +1 -1
- package/guide/assets/download_data.ps1 +2 -0
- package/guide/assets/download_data.sh +8 -2
- package/guide/references/routing.md +30 -3
- package/package.json +1 -1
- package/packs/bjs/endpoints.json +150 -0
- package/packs/bjs/policy.json +114 -0
- package/packs/bjs/source.md +98 -0
- package/packs/cbp/endpoints.json +31 -0
- package/packs/cbp/policy.json +85 -0
- package/packs/cbp/source.md +57 -0
- package/packs/cdc/source.md +1 -1
- package/packs/cdc-wonder/endpoints.json +26 -0
- package/packs/cdc-wonder/policy.json +84 -0
- package/packs/cdc-wonder/source.md +53 -0
- package/packs/college-scorecard/source.md +1 -1
- package/packs/cpsc/endpoints.json +91 -0
- package/packs/cpsc/policy.json +91 -0
- package/packs/cpsc/source.md +76 -0
- package/packs/dhs-ohss/source.md +1 -1
- package/packs/doj-news/source.md +1 -1
- package/packs/fbi/source.md +3 -3
- package/packs/fhwa/endpoints.json +48 -0
- package/packs/fhwa/policy.json +91 -0
- package/packs/fhwa/source.md +59 -0
- package/packs/foreign-assistance/endpoints.json +166 -0
- package/packs/foreign-assistance/policy.json +80 -0
- package/packs/foreign-assistance/source.md +76 -0
- package/packs/fsa/source.md +1 -1
- package/packs/hud/source.md +1 -1
- package/packs/hud-ahar/endpoints.json +92 -0
- package/packs/hud-ahar/policy.json +88 -0
- package/packs/hud-ahar/source.md +72 -0
- package/packs/naep/source.md +1 -1
- package/packs/nces/endpoints.json +101 -0
- package/packs/nces/policy.json +99 -0
- package/packs/nces/source.md +75 -0
- package/packs/nhtsa/source.md +1 -1
- package/packs/nsf/endpoints.json +33 -0
- package/packs/nsf/policy.json +83 -0
- package/packs/nsf/source.md +59 -0
- package/packs/onrr/endpoints.json +198 -0
- package/packs/onrr/policy.json +91 -0
- package/packs/onrr/source.md +55 -0
- package/packs/port.json +65 -0
- package/packs/sba/endpoints.json +34 -0
- package/packs/sba/policy.json +86 -0
- package/packs/sba/source.md +60 -0
- package/packs/treasury/source.md +1 -1
- package/packs/treasury-tic/endpoints.json +77 -0
- package/packs/treasury-tic/policy.json +89 -0
- package/packs/treasury-tic/source.md +80 -0
- package/packs/uscourts/endpoints.json +22 -0
- package/packs/uscourts/policy.json +87 -0
- package/packs/uscourts/source.md +56 -0
- package/packs/uspto/source.md +1 -1
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toUpperCase()).filter(Boolean);
|
|
4
|
+
function checkSpec(spec) {
|
|
5
|
+
const bad = [];
|
|
6
|
+
for (const r of spec.requests || []) {
|
|
7
|
+
const p = r.params || {};
|
|
8
|
+
const tag = `request ${JSON.stringify(r.name)}`;
|
|
9
|
+
if (r.endpoint === "digest_table") {
|
|
10
|
+
if (p.edition !== undefined && !/^\d{2}$/.test(pyStr(p.edition)))
|
|
11
|
+
bad.push(`${tag}: edition is two digits, e.g. 25`);
|
|
12
|
+
if (p.table !== undefined && !/^\d{3}\.\d{2,3}$/.test(pyStr(p.table)))
|
|
13
|
+
bad.push(`${tag}: table looks like 203.10`);
|
|
14
|
+
}
|
|
15
|
+
if ((r.endpoint === "ipeds" || r.endpoint === "ipeds_dictionary") && p.file !== undefined && !/^[A-Za-z0-9_]+$/.test(pyStr(p.file))) {
|
|
16
|
+
bad.push(`${tag}: file is an IPEDS file name such as HD2023 or C2023_A`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return bad;
|
|
20
|
+
}
|
|
21
|
+
function normalize(req, _ep, rows) {
|
|
22
|
+
if (req.endpoint !== "ipeds")
|
|
23
|
+
return rows;
|
|
24
|
+
const ids = list(req.params?.unitid), states = list(req.params?.state);
|
|
25
|
+
if (!ids.length && !states.length)
|
|
26
|
+
return rows;
|
|
27
|
+
return rows.filter((r) => (!ids.length || ids.includes(pyStr(r.UNITID ?? "").trim())) &&
|
|
28
|
+
(!states.length || states.includes(pyStr(r.STABBR ?? "").trim().toUpperCase())));
|
|
29
|
+
}
|
|
30
|
+
const adapter = { checkSpec, normalize };
|
|
31
|
+
export default adapter;
|
|
32
|
+
//# sourceMappingURL=nces.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nces.js","sourceRoot":"","sources":["../../src/adapters/nces.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,IAAI,CAAC,CAAC,QAAQ,KAAK,cAAc,EAAE,CAAC;YAClC,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,kCAAkC,CAAC,CAAC;YACrH,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,2BAA2B,CAAC,CAAC;QACrH,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACpI,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,wDAAwD,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACvE,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NSF Award Search API. Verified 2026-11-07:
|
|
3
|
+
* - offset is 1-based (the default sets it), rpp is at most 25, and response.metadata.totalCount gives the
|
|
4
|
+
* total; printFields is ignored, so every field comes back.
|
|
5
|
+
* - Date filters are MM/DD/YYYY.
|
|
6
|
+
* normalize drops the people's contact fields (PI and program officer e-mail and phone): names are public award
|
|
7
|
+
* data, personal contact details are not needed for analysis. Amounts stay as published text.
|
|
8
|
+
*/
|
|
9
|
+
import type { Adapter } from "../core/packs.js";
|
|
10
|
+
declare const adapter: Adapter;
|
|
11
|
+
export default adapter;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const CONTACT = new Set(["piEmail", "poEmail", "poPhone", "awardeePhone"]);
|
|
3
|
+
const DATES = ["dateStart", "dateEnd", "startDateStart", "startDateEnd", "expDateStart", "expDateEnd"];
|
|
4
|
+
const FILTERS = ["keyword", "awardeeName", "awardeeStateCode", "perfStateCode", "pdPIName", "fundProgramName", "id", ...DATES];
|
|
5
|
+
function checkSpec(spec) {
|
|
6
|
+
const bad = [];
|
|
7
|
+
for (const r of spec.requests || []) {
|
|
8
|
+
const p = r.params || {};
|
|
9
|
+
for (const k of DATES) {
|
|
10
|
+
if (p[k] !== undefined && p[k] !== "" && !/^\d{2}\/\d{2}\/\d{4}$/.test(pyStr(p[k]))) {
|
|
11
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} must be MM/DD/YYYY (got ${JSON.stringify(p[k])})`);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
if (!FILTERS.some((k) => p[k] !== undefined && p[k] !== "")) {
|
|
15
|
+
bad.push(`request ${JSON.stringify(r.name)}: give at least one filter (${FILTERS.join(", ")}); all NSF awards is too many`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return bad;
|
|
19
|
+
}
|
|
20
|
+
function normalize(_req, _ep, rows) {
|
|
21
|
+
return rows.map((r) => Object.fromEntries(Object.entries(r).filter(([k]) => !CONTACT.has(k))));
|
|
22
|
+
}
|
|
23
|
+
const adapter = { checkSpec, normalize };
|
|
24
|
+
export default adapter;
|
|
25
|
+
//# sourceMappingURL=nsf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nsf.js","sourceRoot":"","sources":["../../src/adapters/nsf.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;AAC3E,MAAM,KAAK,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;AACvG,MAAM,OAAO,GAAG,CAAC,SAAS,EAAE,aAAa,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;AAE/H,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpF,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACvG,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YAC5D,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9H,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,IAAiB,EAAE,GAAQ,EAAE,IAAW;IACzD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjG,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Office of Natural Resources Revenue data downloads (revenuedata.onrr.gov/downloads/<file>.csv; revenuedata.doi.gov
|
|
3
|
+
* redirects there). Verified 2026-11-07: revenue (fiscal, calendar, monthly), disbursements (fiscal) and production
|
|
4
|
+
* (fiscal, calendar) as CSV. Files are read whole and filtered here (state, years, commodity, revenue type, land
|
|
5
|
+
* class, fund type, product), values as published.
|
|
6
|
+
*/
|
|
7
|
+
import type { Adapter } from "../core/packs.js";
|
|
8
|
+
declare const adapter: Adapter;
|
|
9
|
+
export default adapter;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
3
|
+
.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
4
|
+
const FILTERS = [["state", "State"], ["commodity", "Commodity"], ["revenue_type", "Revenue Type"],
|
|
5
|
+
["land_class", "Land Class"], ["fund_type", "Fund Type"], ["product", "Product"]];
|
|
6
|
+
function checkSpec(spec) {
|
|
7
|
+
const bad = [];
|
|
8
|
+
for (const r of spec.requests || []) {
|
|
9
|
+
for (const k of ["from", "to"]) {
|
|
10
|
+
const v = r.params?.[k];
|
|
11
|
+
if (v !== undefined && v !== "" && !/^\d{4}$/.test(pyStr(v)))
|
|
12
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} is a year, e.g. 2020`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return bad;
|
|
16
|
+
}
|
|
17
|
+
function normalize(req, _ep, rows) {
|
|
18
|
+
const p = req.params || {};
|
|
19
|
+
const wants = FILTERS.map(([k, col]) => [col, list(p[k])]).filter(([, v]) => v.length);
|
|
20
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
21
|
+
if (!wants.length && from === null && to === null)
|
|
22
|
+
return rows;
|
|
23
|
+
return rows.filter((r) => {
|
|
24
|
+
// monthly_revenue has Date as MM/DD/YYYY (verified 2026-11-07)
|
|
25
|
+
const y = Number(pyStr(r["Fiscal Year"] ?? r["Calendar Year"] ?? (/(\d{4})$/.exec(pyStr(r.Date ?? "")) || ["", ""])[1]));
|
|
26
|
+
if (from !== null && !(y >= from))
|
|
27
|
+
return false;
|
|
28
|
+
if (to !== null && !(y <= to))
|
|
29
|
+
return false;
|
|
30
|
+
return wants.every(([col, v]) => v.includes(pyStr(r[col] ?? "").trim().toLowerCase()));
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const adapter = { checkSpec, normalize };
|
|
34
|
+
export default adapter;
|
|
35
|
+
//# sourceMappingURL=onrr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onrr.js","sourceRoot":"","sources":["../../src/adapters/onrr.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC7D,MAAM,OAAO,GAAuB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,cAAc,EAAE,cAAc,CAAC;IACnH,CAAC,YAAY,EAAE,YAAY,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpF,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACzI,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAuB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7G,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC/D,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,+DAA+D;QAC/D,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzH,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAChD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Readable } from "node:stream";
|
|
2
|
+
import type { Adapter, Row } from "../core/packs.js";
|
|
3
|
+
/** The row filter of a foia request. */
|
|
4
|
+
export declare function keeper(p: Record<string, unknown>): (r: Row) => boolean;
|
|
5
|
+
/** Parse a CSV stream, keeping the rows that pass `keep` (the header names the columns). [rows, cut at max] */
|
|
6
|
+
export declare function streamRows(src: Readable, keep: (r: Row) => boolean, max: number): Promise<[Row[], boolean]>;
|
|
7
|
+
/** One row per file in data.sba.gov's catalog (data.json); `file` is set for CSVs (the foia request's path). */
|
|
8
|
+
export declare function catalogRows(cat: any, search: string): Row[];
|
|
9
|
+
declare const adapter: Adapter;
|
|
10
|
+
export default adapter;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U.S. Small Business Administration loan data on data.sba.gov. Verified 2026-11-07:
|
|
3
|
+
* - the CKAN API answers 404, but /data.json (the open-data catalog) lists every file;
|
|
4
|
+
* - the 7(a) and 504 FOIA CSVs are under /sites/default/files/uploaded_resources/, named with their as-of date
|
|
5
|
+
* (FOIA_504_FY2010_Present_asof_260930.csv) and 60 to 255 MB each.
|
|
6
|
+
* list_files reads the catalog. foia streams the CSV through the parser and keeps only the rows that pass the
|
|
7
|
+
* filters, so a 255 MB file never becomes a full table in memory; a filter is required.
|
|
8
|
+
*/
|
|
9
|
+
import fs from "node:fs";
|
|
10
|
+
import { Readable } from "node:stream";
|
|
11
|
+
import { parse } from "csv-parse";
|
|
12
|
+
import * as gr from "../core/generic-rest.js";
|
|
13
|
+
import { DataError } from "../core/http.js";
|
|
14
|
+
import { pyStr } from "../core/py.js";
|
|
15
|
+
import { columnsOf, withColumns } from "../core/tabular.js";
|
|
16
|
+
const list = (v) => (Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v])
|
|
17
|
+
.map((x) => pyStr(x).trim().toUpperCase()).filter(Boolean);
|
|
18
|
+
/** The row filter of a foia request. */
|
|
19
|
+
export function keeper(p) {
|
|
20
|
+
const states = list(p.state), years = list(p.fiscal_year), naics = list(p.naics);
|
|
21
|
+
const from = p.from ? Number(p.from) : null, to = p.to ? Number(p.to) : null;
|
|
22
|
+
return (r) => {
|
|
23
|
+
if (states.length && !states.includes(pyStr(r.ProjectState ?? "").trim().toUpperCase()))
|
|
24
|
+
return false;
|
|
25
|
+
const fy = pyStr(r.ApprovalFY ?? "").trim();
|
|
26
|
+
if (years.length && !years.includes(fy))
|
|
27
|
+
return false;
|
|
28
|
+
if (from !== null && !(Number(fy) >= from))
|
|
29
|
+
return false;
|
|
30
|
+
if (to !== null && !(Number(fy) <= to))
|
|
31
|
+
return false;
|
|
32
|
+
if (naics.length && !naics.some((n) => pyStr(r.NaicsCode ?? "").trim().startsWith(n)))
|
|
33
|
+
return false;
|
|
34
|
+
return true;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/** Parse a CSV stream, keeping the rows that pass `keep` (the header names the columns). [rows, cut at max] */
|
|
38
|
+
export async function streamRows(src, keep, max) {
|
|
39
|
+
const out = [];
|
|
40
|
+
let cols = [];
|
|
41
|
+
let cut = false;
|
|
42
|
+
const parser = src.pipe(parse({ bom: true, relax_column_count: true, relax_quotes: true, skip_empty_lines: true }));
|
|
43
|
+
for await (const rec of parser) {
|
|
44
|
+
if (!cols.length) {
|
|
45
|
+
cols = rec;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const r = {};
|
|
49
|
+
cols.forEach((c, i) => (r[c] = i < rec.length ? rec[i] : ""));
|
|
50
|
+
if (!keep(r))
|
|
51
|
+
continue;
|
|
52
|
+
if (out.length >= max) {
|
|
53
|
+
cut = true;
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
out.push(r);
|
|
57
|
+
}
|
|
58
|
+
return [withColumns(out, cols), cut];
|
|
59
|
+
}
|
|
60
|
+
function checkSpec(spec) {
|
|
61
|
+
const bad = [];
|
|
62
|
+
for (const r of spec.requests || []) {
|
|
63
|
+
const p = r.params || {};
|
|
64
|
+
if (r.endpoint !== "foia")
|
|
65
|
+
continue;
|
|
66
|
+
if (!["state", "fiscal_year", "from", "naics"].some((k) => p[k] !== undefined && p[k] !== "")) {
|
|
67
|
+
bad.push(`request ${JSON.stringify(r.name)}: give state, fiscal_year, from or naics (the files hold hundreds of thousands of loans)`);
|
|
68
|
+
}
|
|
69
|
+
if (p.file !== undefined && (!/^[\w./-]+$/.test(pyStr(p.file)) || pyStr(p.file).includes(".."))) {
|
|
70
|
+
bad.push(`request ${JSON.stringify(r.name)}: file is a path from list_files`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return bad;
|
|
74
|
+
}
|
|
75
|
+
/** One row per file in data.sba.gov's catalog (data.json); `file` is set for CSVs (the foia request's path). */
|
|
76
|
+
export function catalogRows(cat, search) {
|
|
77
|
+
const out = [];
|
|
78
|
+
const want = search.trim().toLowerCase();
|
|
79
|
+
for (const d of cat?.dataset || []) {
|
|
80
|
+
for (const x of d.distribution || []) {
|
|
81
|
+
const url = pyStr(x.downloadURL || x.accessURL || "");
|
|
82
|
+
const m = /^https:\/\/data\.sba\.gov\/sites\/default\/files\/(.+?)(\.csv)?$/i.exec(url);
|
|
83
|
+
const row = { dataset: pyStr(d.title ?? ""), title: pyStr(x.title ?? ""), format: pyStr(x.format || x.mediaType || ""),
|
|
84
|
+
file: m && m[2] ? m[1] : "", url, modified: pyStr(d.modified ?? "").slice(0, 10) };
|
|
85
|
+
if (want && !`${row.dataset} ${row.title}`.toLowerCase().includes(want))
|
|
86
|
+
continue;
|
|
87
|
+
out.push(row);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
async function fetch(spec, pack) {
|
|
93
|
+
const c = gr.clientFor(pack);
|
|
94
|
+
const tables = {};
|
|
95
|
+
const notes = [];
|
|
96
|
+
for (const r of spec.requests) {
|
|
97
|
+
const [ep, params, probs] = gr.resolve(pack, r);
|
|
98
|
+
if (probs.length)
|
|
99
|
+
throw new DataError(probs.join("; "), "bad_request");
|
|
100
|
+
if (r.endpoint === "list_files") {
|
|
101
|
+
const resp = await c.request("GET", pack.endpoints.catalog_path, null);
|
|
102
|
+
tables[r.name] = catalogRows(resp.json(), pyStr(params.search ?? ""));
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const [p] = gr.split(ep, params);
|
|
106
|
+
const resp = await c.request("GET", p, null);
|
|
107
|
+
const max = Number(r.max_rows || ep.max_rows || 1000000);
|
|
108
|
+
const [rows, cut] = await streamRows(Readable.from([resp.body]), keeper(params), max);
|
|
109
|
+
if (cut)
|
|
110
|
+
notes.push(`${r.name}: stopped at ${rows.length.toLocaleString("en-US")} rows (max_rows); the source has more. Narrow the filter or raise max_rows.`);
|
|
111
|
+
if (!rows.length && !r.allow_empty) {
|
|
112
|
+
throw new DataError(`request '${r.name}' returned no rows: check the filters (ProjectState is two letters, ApprovalFY a year)`, "bad_request");
|
|
113
|
+
}
|
|
114
|
+
tables[r.name] = rows;
|
|
115
|
+
}
|
|
116
|
+
return [tables, notes];
|
|
117
|
+
}
|
|
118
|
+
async function parseUpload(pack, req, files) {
|
|
119
|
+
if (req.endpoint !== "foia")
|
|
120
|
+
return gr.parseUpload(pack, req, files);
|
|
121
|
+
const [, params] = gr.resolve(pack, req);
|
|
122
|
+
const rows = [];
|
|
123
|
+
let cols = [];
|
|
124
|
+
for (const f of [...files].sort()) {
|
|
125
|
+
const [got] = await streamRows(fs.createReadStream(f), keeper(params), 1000000);
|
|
126
|
+
if (!cols.length)
|
|
127
|
+
cols = columnsOf(got) || [];
|
|
128
|
+
rows.push(...got);
|
|
129
|
+
}
|
|
130
|
+
return withColumns(rows, cols);
|
|
131
|
+
}
|
|
132
|
+
const adapter = { fetch, checkSpec, parseUpload };
|
|
133
|
+
export default adapter;
|
|
134
|
+
//# sourceMappingURL=sba.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sba.js","sourceRoot":"","sources":["../../src/adapters/sba.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE5D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvG,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE7D,wCAAwC;AACxC,MAAM,UAAU,MAAM,CAAC,CAA0B;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7E,OAAO,CAAC,CAAC,EAAE,EAAE;QACX,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACtD,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACzD,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC;QACrD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACpG,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,+GAA+G;AAC/G,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAa,EAAE,IAAyB,EAAE,GAAW;IACpF,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,GAAG,GAAG,KAAK,CAAC;IAChB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACpH,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAiC,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,GAAG,GAAG,CAAC;YACX,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAQ,EAAE,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAAE,SAAS;QACvB,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YACtB,GAAG,GAAG,IAAI,CAAC;YACX,MAAM;QACR,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,QAAQ,KAAK,MAAM;YAAE,SAAS;QACpC,IAAI,CAAC,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;YAC9F,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;QACxI,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAChG,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gHAAgH;AAChH,MAAM,UAAU,WAAW,CAAC,GAAQ,EAAE,MAAc;IAClD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;YACtD,MAAM,CAAC,GAAG,mEAAmE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,GAAG,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;gBACpH,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrF,IAAI,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAAE,SAAS;YAClF,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAU,EAAE,IAAU;IACzC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC,CAAC;QACvE,IAAI,CAAC,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;YACvE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC;QACzD,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;QACtF,IAAI,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,6EAA6E,CAAC,CAAC;QAC/J,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,IAAI,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,wFAAwF,EAAE,aAAa,CAAC,CAAC;QACjJ,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,GAAgB,EAAE,KAAe;IACtE,IAAI,GAAG,CAAC,QAAQ,KAAK,MAAM;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;QAChF,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AAC3D,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Treasury International Capital (TIC): published tab-delimited text tables, read by the generic $tsv reader and
|
|
3
|
+
* made long here (one row per country and month), values as published.
|
|
4
|
+
*
|
|
5
|
+
* major_foreign_holders slt_table5.txt: "Country" then one column per month (2026-08 ...), billions
|
|
6
|
+
* major_foreign_holders_history mfhhis01.txt: one block per year, a month-name line ("Dec Nov ...") above a
|
|
7
|
+
* "Country 2025 2025 ..." line, rows to "Grand Total", then "Of which:" rows
|
|
8
|
+
* holdings_by_country slt_table1.txt: machine column names on line 9, then rows; a definitions
|
|
9
|
+
* footer ("for_lt_corp_net: 19 + 23 ...") follows the data
|
|
10
|
+
* The spec-only params country, from and to filter the rows (the files are small, so the whole file is read).
|
|
11
|
+
*/
|
|
12
|
+
import type { Adapter, Row, SpecRequest } from "../core/packs.js";
|
|
13
|
+
/** A published row label without footnote marks ("Grand Total 1/" -> "Grand Total"). */
|
|
14
|
+
export declare function label(s: unknown): string;
|
|
15
|
+
/** What a row is: a country, the All Other remainder, the Grand Total or a foreign-official line. */
|
|
16
|
+
export declare function kind(name: string): string;
|
|
17
|
+
/**
|
|
18
|
+
* The stacked yearly blocks of mfhhis01.txt (rows col1..colN from $tsv with header_row 0).
|
|
19
|
+
*
|
|
20
|
+
* Two quirks of the published file (verified 2026-11-07):
|
|
21
|
+
* - A series-break year shows one month twice (June 2002-2011, March 2000): the first column is the new
|
|
22
|
+
* series, the second the previous series "shown for comparison only" (the file's footnotes). The second
|
|
23
|
+
* is kept only with include_comparison, marked comparison_only.
|
|
24
|
+
* - In those years January is a 14th column one line up: its header "Jan" sits above the month line, the
|
|
25
|
+
* month line holds the year there, and each row's January value is on the line above the row. It is read
|
|
26
|
+
* from the line above.
|
|
27
|
+
*/
|
|
28
|
+
export declare function history(req: SpecRequest, rows: Row[]): Row[];
|
|
29
|
+
declare const adapter: Adapter;
|
|
30
|
+
export default adapter;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
const MONTHS = { jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06", jul: "07",
|
|
3
|
+
aug: "08", sep: "09", oct: "10", nov: "11", dec: "12" };
|
|
4
|
+
const PERIOD = /^\d{4}-\d{2}$/;
|
|
5
|
+
const NUMBER = /^-?\d+(\.\d+)?$/;
|
|
6
|
+
/** A published row label without footnote marks ("Grand Total 1/" -> "Grand Total"). */
|
|
7
|
+
export function label(s) {
|
|
8
|
+
return pyStr(s ?? "").replace(/\s+\d+\/\s*$/g, "").replace(/\s+/g, " ").trim();
|
|
9
|
+
}
|
|
10
|
+
/** What a row is: a country, the All Other remainder, the Grand Total or a foreign-official line. */
|
|
11
|
+
export function kind(name) {
|
|
12
|
+
const n = name.toLowerCase();
|
|
13
|
+
if (n === "grand total")
|
|
14
|
+
return "grand_total";
|
|
15
|
+
if (n === "all other")
|
|
16
|
+
return "all_other";
|
|
17
|
+
if (n.startsWith("of which") || /^(for\. official|treasury bills|t-bonds & notes)$/.test(n))
|
|
18
|
+
return "foreign_official";
|
|
19
|
+
return "country";
|
|
20
|
+
}
|
|
21
|
+
/** "Of which" lines named as table 5 names them, so both files give the same labels. */
|
|
22
|
+
const OFFICIAL = { "for. official": "Of Which: Foreign Official",
|
|
23
|
+
"treasury bills": "Of Which: Foreign Official Treasury Bills",
|
|
24
|
+
"t-bonds & notes": "Of Which: Foreign Official T-Bonds & Notes" };
|
|
25
|
+
function list(v) {
|
|
26
|
+
const xs = Array.isArray(v) ? v : v === undefined || v === null || v === "" ? [] : [v];
|
|
27
|
+
return xs.map((x) => pyStr(x).trim().toLowerCase()).filter(Boolean);
|
|
28
|
+
}
|
|
29
|
+
function keep(req, country, code, period) {
|
|
30
|
+
const p = req.params || {};
|
|
31
|
+
const want = list(p.country);
|
|
32
|
+
if (want.length && !want.includes(country.toLowerCase()) && !(code && want.includes(code.toLowerCase())))
|
|
33
|
+
return false;
|
|
34
|
+
if (p.from && period < pyStr(p.from))
|
|
35
|
+
return false;
|
|
36
|
+
if (p.to && period > pyStr(p.to))
|
|
37
|
+
return false;
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
function table5(req, rows) {
|
|
41
|
+
const out = [];
|
|
42
|
+
for (const r of rows) {
|
|
43
|
+
const name = label(r.Country);
|
|
44
|
+
if (!name)
|
|
45
|
+
continue;
|
|
46
|
+
const periods = Object.keys(r).filter((k) => PERIOD.test(k));
|
|
47
|
+
if (!periods.some((k) => NUMBER.test(pyStr(r[k] ?? ""))))
|
|
48
|
+
continue; // notes and blank lines
|
|
49
|
+
for (const period of periods) {
|
|
50
|
+
if (!keep(req, name, "", period))
|
|
51
|
+
continue;
|
|
52
|
+
out.push({ country: name, kind: kind(name), period, holdings_billions: pyStr(r[period] ?? "") });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The stacked yearly blocks of mfhhis01.txt (rows col1..colN from $tsv with header_row 0).
|
|
59
|
+
*
|
|
60
|
+
* Two quirks of the published file (verified 2026-11-07):
|
|
61
|
+
* - A series-break year shows one month twice (June 2002-2011, March 2000): the first column is the new
|
|
62
|
+
* series, the second the previous series "shown for comparison only" (the file's footnotes). The second
|
|
63
|
+
* is kept only with include_comparison, marked comparison_only.
|
|
64
|
+
* - In those years January is a 14th column one line up: its header "Jan" sits above the month line, the
|
|
65
|
+
* month line holds the year there, and each row's January value is on the line above the row. It is read
|
|
66
|
+
* from the line above.
|
|
67
|
+
*/
|
|
68
|
+
export function history(req, rows) {
|
|
69
|
+
const out = [];
|
|
70
|
+
const withComparison = ["true", "1", "yes"].includes(pyStr(req.params?.include_comparison ?? "").toLowerCase());
|
|
71
|
+
let months = [];
|
|
72
|
+
let cols = [];
|
|
73
|
+
let inBlock = false;
|
|
74
|
+
let prev = [];
|
|
75
|
+
const cells = (r) => Object.keys(r).filter((k) => /^col\d+$/.test(k)).sort((a, b) => Number(a.slice(3)) - Number(b.slice(3))).map((k) => pyStr(r[k] ?? "").trim());
|
|
76
|
+
for (const r of rows) {
|
|
77
|
+
const c = cells(r);
|
|
78
|
+
const first = label(c[0]);
|
|
79
|
+
const low = first.toLowerCase();
|
|
80
|
+
const before = prev;
|
|
81
|
+
prev = c;
|
|
82
|
+
if (!first && c.slice(1).filter((x) => MONTHS[x.slice(0, 3).toLowerCase()]).length >= 2) {
|
|
83
|
+
months = c.slice(1).map((x) => (MONTHS[x.slice(0, 3).toLowerCase()] ? { m: MONTHS[x.slice(0, 3).toLowerCase()], shifted: false }
|
|
84
|
+
: /^\d{4}$/.test(x) && before.at(c.indexOf(x))?.toLowerCase() === "jan" ? { m: "01", shifted: true, year: x }
|
|
85
|
+
: { m: "", shifted: false }));
|
|
86
|
+
inBlock = false;
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (low === "country") {
|
|
90
|
+
const seen = new Set();
|
|
91
|
+
cols = c.slice(1).map((y, i) => {
|
|
92
|
+
const mo = months[i];
|
|
93
|
+
const year = mo?.shifted ? mo.year : y;
|
|
94
|
+
const period = mo?.m && /^\d{4}$/.test(year) ? `${year}-${mo.m}` : "";
|
|
95
|
+
const comparison = !!period && seen.has(period);
|
|
96
|
+
if (period)
|
|
97
|
+
seen.add(period);
|
|
98
|
+
return { period, shifted: !!mo?.shifted, comparison };
|
|
99
|
+
});
|
|
100
|
+
inBlock = true;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (!inBlock || !first || low === "of which:")
|
|
104
|
+
continue;
|
|
105
|
+
if (/^\d+\/$/.test(first) || first.length > 60) {
|
|
106
|
+
inBlock = false; // the footnotes after the last block
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
const name = OFFICIAL[low] ?? first;
|
|
110
|
+
cols.forEach((col, i) => {
|
|
111
|
+
const v = (col.shifted ? before[i + 1] : c[i + 1]) ?? "";
|
|
112
|
+
if (!col.period || v === "" || /^-+$/.test(v) || (col.comparison && !withComparison) || !keep(req, name, "", col.period))
|
|
113
|
+
return;
|
|
114
|
+
out.push({ country: name, kind: kind(name), period: col.period, holdings_billions: v, comparison_only: col.comparison });
|
|
115
|
+
});
|
|
116
|
+
if (low === "t-bonds & notes")
|
|
117
|
+
inBlock = false;
|
|
118
|
+
}
|
|
119
|
+
const per = (r) => String(r.period);
|
|
120
|
+
return out.sort((a, b) => (per(a) === per(b) ? 0 : per(a) < per(b) ? 1 : -1));
|
|
121
|
+
}
|
|
122
|
+
function table1(req, rows) {
|
|
123
|
+
return rows.filter((r) => PERIOD.test(pyStr(r.date ?? "")) &&
|
|
124
|
+
keep(req, label(r.country), pyStr(r.country_code ?? ""), pyStr(r.date)));
|
|
125
|
+
}
|
|
126
|
+
function normalize(req, _ep, rows) {
|
|
127
|
+
if (req.endpoint === "major_foreign_holders")
|
|
128
|
+
return table5(req, rows);
|
|
129
|
+
if (req.endpoint === "major_foreign_holders_history")
|
|
130
|
+
return history(req, rows);
|
|
131
|
+
if (req.endpoint === "holdings_by_country")
|
|
132
|
+
return table1(req, rows);
|
|
133
|
+
return rows;
|
|
134
|
+
}
|
|
135
|
+
function checkSpec(spec) {
|
|
136
|
+
const bad = [];
|
|
137
|
+
for (const r of spec.requests || []) {
|
|
138
|
+
for (const k of ["from", "to"]) {
|
|
139
|
+
const v = r.params?.[k];
|
|
140
|
+
if (v !== undefined && v !== null && v !== "" && !PERIOD.test(pyStr(v))) {
|
|
141
|
+
bad.push(`request ${JSON.stringify(r.name)}: ${k} must be a month as YYYY-MM (got ${JSON.stringify(v)})`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return bad;
|
|
146
|
+
}
|
|
147
|
+
const adapter = { normalize, checkSpec };
|
|
148
|
+
export default adapter;
|
|
149
|
+
//# sourceMappingURL=treasury-tic.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"treasury-tic.js","sourceRoot":"","sources":["../../src/adapters/treasury-tic.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,MAAM,MAAM,GAA2B,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI;IAClH,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAC1D,MAAM,MAAM,GAAG,eAAe,CAAC;AAC/B,MAAM,MAAM,GAAG,iBAAiB,CAAC;AAEjC,wFAAwF;AACxF,MAAM,UAAU,KAAK,CAAC,CAAU;IAC9B,OAAO,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACjF,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,IAAI,CAAC,IAAY;IAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,IAAI,CAAC,KAAK,aAAa;QAAE,OAAO,aAAa,CAAC;IAC9C,IAAI,CAAC,KAAK,WAAW;QAAE,OAAO,WAAW,CAAC;IAC1C,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,mDAAmD,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACvH,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,wFAAwF;AACxF,MAAM,QAAQ,GAA2B,EAAE,eAAe,EAAE,4BAA4B;IACtF,gBAAgB,EAAE,2CAA2C;IAC7D,iBAAiB,EAAE,4CAA4C,EAAE,CAAC;AAEpE,SAAS,IAAI,CAAC,CAAU;IACtB,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,IAAI,CAAC,GAAgB,EAAE,OAAe,EAAE,IAAY,EAAE,MAAc;IAC3E,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACvH,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACnD,IAAI,CAAC,CAAC,EAAE,IAAI,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,MAAM,CAAC,GAAgB,EAAE,IAAW;IAC3C,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAAE,SAAS,CAAC,wBAAwB;QAC5F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC;gBAAE,SAAS;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,GAAgB,EAAE,IAAW;IACnD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAChH,IAAI,MAAM,GAAqD,EAAE,CAAC;IAClE,IAAI,IAAI,GAAgE,EAAE,CAAC;IAC3E,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,MAAM,KAAK,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxK,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC;QACpB,IAAI,GAAG,CAAC,CAAC;QACT,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxF,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;gBAC9H,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE;oBAC3G,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;YAClC,OAAO,GAAG,KAAK,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC7B,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,IAAI,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,IAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,MAAM,GAAG,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAChD,IAAI,MAAM;oBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;YACxD,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,IAAI,CAAC;YACf,SAAS;QACX,CAAC;QACD,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,GAAG,KAAK,WAAW;YAAE,SAAS;QACxD,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC/C,OAAO,GAAG,KAAK,CAAC,CAAC,qCAAqC;YACtD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;QACpC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzD,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC;gBAAE,OAAO;YACjI,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,iBAAiB,EAAE,CAAC,EAAE,eAAe,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3H,CAAC,CAAC,CAAC;QACH,IAAI,GAAG,KAAK,iBAAiB;YAAE,OAAO,GAAG,KAAK,CAAC;IACjD,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,MAAM,CAAC,GAAgB,EAAE,IAAW;IAC3C,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,SAAS,CAAC,GAAgB,EAAE,GAAQ,EAAE,IAAW;IACxD,IAAI,GAAG,CAAC,QAAQ,KAAK,uBAAuB;QAAE,OAAO,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACvE,IAAI,GAAG,CAAC,QAAQ,KAAK,+BAA+B;QAAE,OAAO,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChF,IAAI,GAAG,CAAC,QAAQ,KAAK,qBAAqB;QAAE,OAAO,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,SAAS,CAAC,IAAU;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AAClD,eAAe,OAAO,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* U.S. Courts caseload statistics tables: /sites/default/files/document/<file>.xlsx, read by core's published-table
|
|
3
|
+
* reader (verified 2026-11-07: bf_f2_0930.2026, stfj_b_630.2026). File names come from the Data Tables pages; the
|
|
4
|
+
* date part is not zero-padded the same way in every series (0930.2026, 630.2026), so the name is taken as given.
|
|
5
|
+
*/
|
|
6
|
+
import type { Adapter } from "../core/packs.js";
|
|
7
|
+
declare const adapter: Adapter;
|
|
8
|
+
export default adapter;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { pyStr } from "../core/py.js";
|
|
2
|
+
function checkSpec(spec) {
|
|
3
|
+
return (spec.requests || []).filter((r) => r.params?.file !== undefined && !/^[a-z0-9_.-]+$/i.test(pyStr(r.params.file)))
|
|
4
|
+
.map((r) => `request ${JSON.stringify(r.name)}: file is the table's file name without .xlsx, e.g. bf_f2_0930.2026`);
|
|
5
|
+
}
|
|
6
|
+
const adapter = { checkSpec };
|
|
7
|
+
export default adapter;
|
|
8
|
+
//# sourceMappingURL=uscourts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"uscourts.js","sourceRoot":"","sources":["../../src/adapters/uscourts.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEtC,SAAS,SAAS,CAAC,IAAU;IAC3B,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;SACtH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;AACxH,CAAC;AAED,MAAM,OAAO,GAAY,EAAE,SAAS,EAAE,CAAC;AACvC,eAAe,OAAO,CAAC"}
|
package/dist/analysis/cite.d.ts
CHANGED
|
@@ -15,8 +15,16 @@ export type OpenedRecord = {
|
|
|
15
15
|
};
|
|
16
16
|
/** Is an address private, loopback, link-local or otherwise not a public host? */
|
|
17
17
|
export declare function privateAddress(addr: string): boolean;
|
|
18
|
-
/**
|
|
18
|
+
/** Hosted mode: this server's own public host, whose pages (the keys page, signed links) are never opened. */
|
|
19
|
+
export declare function setOwnHost(publicUrl: string): void;
|
|
20
|
+
/** Refuse a URL whose host is or resolves to a private address, or is this server itself. */
|
|
19
21
|
export declare function checkPublicHost(url: string): Promise<void>;
|
|
22
|
+
/** GET with redirects followed by hand, every connection through the guarded lookup (remote mode). */
|
|
23
|
+
export declare function guardedGet(url: string, timeout: number): Promise<{
|
|
24
|
+
status: number;
|
|
25
|
+
body: Buffer;
|
|
26
|
+
contentType: string;
|
|
27
|
+
}>;
|
|
20
28
|
export type CiteResult = {
|
|
21
29
|
status: "ok";
|
|
22
30
|
id: string;
|
package/dist/analysis/cite.js
CHANGED
|
@@ -99,7 +99,12 @@ export function privateAddress(addr) {
|
|
|
99
99
|
}
|
|
100
100
|
return true; // not an address at all: refuse
|
|
101
101
|
}
|
|
102
|
-
|
|
102
|
+
const OWN_HOSTS = new Set();
|
|
103
|
+
/** Hosted mode: this server's own public host, whose pages (the keys page, signed links) are never opened. */
|
|
104
|
+
export function setOwnHost(publicUrl) {
|
|
105
|
+
OWN_HOSTS.add(new URL(publicUrl).hostname.toLowerCase());
|
|
106
|
+
}
|
|
107
|
+
/** Refuse a URL whose host is or resolves to a private address, or is this server itself. */
|
|
103
108
|
export async function checkPublicHost(url) {
|
|
104
109
|
let host;
|
|
105
110
|
try {
|
|
@@ -108,6 +113,8 @@ export async function checkPublicHost(url) {
|
|
|
108
113
|
catch {
|
|
109
114
|
throw new CiteError(`not a valid URL: ${url}`);
|
|
110
115
|
}
|
|
116
|
+
if (OWN_HOSTS.has(host.toLowerCase()))
|
|
117
|
+
throw new CiteError("refused: this server's own pages are never opened", url);
|
|
111
118
|
if (net.isIP(host)) {
|
|
112
119
|
if (privateAddress(host))
|
|
113
120
|
throw new CiteError(`refused: ${host} is a private or loopback address; this server opens public pages only`, url);
|
|
@@ -140,7 +147,7 @@ const guardedLookup = (hostname, options, cb) => {
|
|
|
140
147
|
});
|
|
141
148
|
};
|
|
142
149
|
/** GET with redirects followed by hand, every connection through the guarded lookup (remote mode). */
|
|
143
|
-
async function guardedGet(url, timeout) {
|
|
150
|
+
export async function guardedGet(url, timeout) {
|
|
144
151
|
let cur = url;
|
|
145
152
|
for (let hop = 0; hop < 6; hop++) {
|
|
146
153
|
await checkPublicHost(cur);
|