@cliwant/mcp-sam-gov 0.2.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.ja.md +48 -16
- package/README.ko.md +48 -16
- package/README.md +279 -67
- package/dist/attachments.d.ts +63 -0
- package/dist/attachments.d.ts.map +1 -0
- package/dist/attachments.js +576 -0
- package/dist/attachments.js.map +1 -0
- package/dist/bls.d.ts +267 -0
- package/dist/bls.d.ts.map +1 -0
- package/dist/bls.js +1549 -0
- package/dist/bls.js.map +1 -0
- package/dist/cache.d.ts +42 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +64 -0
- package/dist/cache.js.map +1 -0
- package/dist/census.d.ts +182 -0
- package/dist/census.d.ts.map +1 -0
- package/dist/census.js +576 -0
- package/dist/census.js.map +1 -0
- package/dist/ckan.d.ts +141 -0
- package/dist/ckan.d.ts.map +1 -0
- package/dist/ckan.js +393 -0
- package/dist/ckan.js.map +1 -0
- package/dist/clinicaltrials.d.ts +180 -0
- package/dist/clinicaltrials.d.ts.map +1 -0
- package/dist/clinicaltrials.js +730 -0
- package/dist/clinicaltrials.js.map +1 -0
- package/dist/cms.d.ts +140 -0
- package/dist/cms.d.ts.map +1 -0
- package/dist/cms.js +482 -0
- package/dist/cms.js.map +1 -0
- package/dist/coerce.d.ts +32 -0
- package/dist/coerce.d.ts.map +1 -0
- package/dist/coerce.js +50 -0
- package/dist/coerce.js.map +1 -0
- package/dist/datagov-catalog.d.ts +84 -0
- package/dist/datagov-catalog.d.ts.map +1 -0
- package/dist/datagov-catalog.js +233 -0
- package/dist/datagov-catalog.js.map +1 -0
- package/dist/datagov.d.ts +146 -0
- package/dist/datagov.d.ts.map +1 -0
- package/dist/datagov.js +689 -0
- package/dist/datagov.js.map +1 -0
- package/dist/datagovKey.d.ts +36 -0
- package/dist/datagovKey.d.ts.map +1 -0
- package/dist/datagovKey.js +60 -0
- package/dist/datagovKey.js.map +1 -0
- package/dist/datasource.d.ts +313 -0
- package/dist/datasource.d.ts.map +1 -0
- package/dist/datasource.js +551 -0
- package/dist/datasource.js.map +1 -0
- package/dist/disclosure.d.ts +55 -0
- package/dist/disclosure.d.ts.map +1 -0
- package/dist/disclosure.js +57 -0
- package/dist/disclosure.js.map +1 -0
- package/dist/ecfr.d.ts +4 -2
- package/dist/ecfr.d.ts.map +1 -1
- package/dist/ecfr.js +92 -17
- package/dist/ecfr.js.map +1 -1
- package/dist/echo.d.ts +143 -0
- package/dist/echo.d.ts.map +1 -0
- package/dist/echo.js +424 -0
- package/dist/echo.js.map +1 -0
- package/dist/edgar.d.ts +377 -0
- package/dist/edgar.d.ts.map +1 -0
- package/dist/edgar.js +2418 -0
- package/dist/edgar.js.map +1 -0
- package/dist/errors.d.ts +102 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +247 -0
- package/dist/errors.js.map +1 -0
- package/dist/fac.d.ts +180 -0
- package/dist/fac.d.ts.map +1 -0
- package/dist/fac.js +416 -0
- package/dist/fac.js.map +1 -0
- package/dist/far.d.ts +170 -0
- package/dist/far.d.ts.map +1 -0
- package/dist/far.js +804 -0
- package/dist/far.js.map +1 -0
- package/dist/fdic.d.ts +599 -0
- package/dist/fdic.d.ts.map +1 -0
- package/dist/fdic.js +1624 -0
- package/dist/fdic.js.map +1 -0
- package/dist/federal-register.d.ts +139 -2
- package/dist/federal-register.d.ts.map +1 -1
- package/dist/federal-register.js +432 -15
- package/dist/federal-register.js.map +1 -1
- package/dist/fema.d.ts +181 -0
- package/dist/fema.d.ts.map +1 -0
- package/dist/fema.js +436 -0
- package/dist/fema.js.map +1 -0
- package/dist/fpds.d.ts +108 -0
- package/dist/fpds.d.ts.map +1 -0
- package/dist/fpds.js +519 -0
- package/dist/fpds.js.map +1 -0
- package/dist/gao.d.ts +64 -0
- package/dist/gao.d.ts.map +1 -0
- package/dist/gao.js +640 -0
- package/dist/gao.js.map +1 -0
- package/dist/govinfo.d.ts +111 -0
- package/dist/govinfo.d.ts.map +1 -0
- package/dist/govinfo.js +422 -0
- package/dist/govinfo.js.map +1 -0
- package/dist/grants.d.ts +27 -4
- package/dist/grants.d.ts.map +1 -1
- package/dist/grants.js +114 -11
- package/dist/grants.js.map +1 -1
- package/dist/gsa-csv.d.ts +249 -0
- package/dist/gsa-csv.d.ts.map +1 -0
- package/dist/gsa-csv.js +784 -0
- package/dist/gsa-csv.js.map +1 -0
- package/dist/integrity.d.ts +212 -0
- package/dist/integrity.d.ts.map +1 -0
- package/dist/integrity.js +707 -0
- package/dist/integrity.js.map +1 -0
- package/dist/meta.d.ts +165 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +162 -0
- package/dist/meta.js.map +1 -0
- package/dist/nih.d.ts +117 -0
- package/dist/nih.d.ts.map +1 -0
- package/dist/nih.js +291 -0
- package/dist/nih.js.map +1 -0
- package/dist/nppes.d.ts +157 -0
- package/dist/nppes.d.ts.map +1 -0
- package/dist/nppes.js +648 -0
- package/dist/nppes.js.map +1 -0
- package/dist/nsf.d.ts +176 -0
- package/dist/nsf.d.ts.map +1 -0
- package/dist/nsf.js +554 -0
- package/dist/nsf.js.map +1 -0
- package/dist/nvd.d.ts +176 -0
- package/dist/nvd.d.ts.map +1 -0
- package/dist/nvd.js +912 -0
- package/dist/nvd.js.map +1 -0
- package/dist/ofac.d.ts +205 -0
- package/dist/ofac.d.ts.map +1 -0
- package/dist/ofac.js +919 -0
- package/dist/ofac.js.map +1 -0
- package/dist/pricing.d.ts +110 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +843 -0
- package/dist/pricing.js.map +1 -0
- package/dist/sam-gov/client.d.ts +60 -2
- package/dist/sam-gov/client.d.ts.map +1 -1
- package/dist/sam-gov/client.js +320 -54
- package/dist/sam-gov/client.js.map +1 -1
- package/dist/sam-gov/index.d.ts +1 -1
- package/dist/sam-gov/index.d.ts.map +1 -1
- package/dist/sam-gov/index.js +1 -1
- package/dist/sam-gov/index.js.map +1 -1
- package/dist/sam-gov/types.d.ts +24 -0
- package/dist/sam-gov/types.d.ts.map +1 -1
- package/dist/sba.d.ts +72 -0
- package/dist/sba.d.ts.map +1 -0
- package/dist/sba.js +281 -0
- package/dist/sba.js.map +1 -0
- package/dist/server.d.ts +14 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3897 -295
- package/dist/server.js.map +1 -1
- package/dist/snapshot.d.ts +98 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +146 -0
- package/dist/snapshot.js.map +1 -0
- package/dist/socrata.d.ts +157 -0
- package/dist/socrata.d.ts.map +1 -0
- package/dist/socrata.js +448 -0
- package/dist/socrata.js.map +1 -0
- package/dist/treasury.d.ts +143 -0
- package/dist/treasury.d.ts.map +1 -0
- package/dist/treasury.js +436 -0
- package/dist/treasury.js.map +1 -0
- package/dist/usaspending.d.ts +260 -65
- package/dist/usaspending.d.ts.map +1 -1
- package/dist/usaspending.js +1664 -228
- package/dist/usaspending.js.map +1 -1
- package/dist/usitc.d.ts +142 -0
- package/dist/usitc.d.ts.map +1 -0
- package/dist/usitc.js +339 -0
- package/dist/usitc.js.map +1 -0
- package/package.json +24 -2
- package/src/attachments.ts +652 -0
- package/src/bls.ts +1943 -0
- package/src/cache.ts +73 -0
- package/src/census.ts +735 -0
- package/src/ckan.ts +495 -0
- package/src/clinicaltrials.ts +923 -0
- package/src/cms.ts +634 -0
- package/src/coerce.ts +47 -0
- package/src/datagov-catalog.ts +296 -0
- package/src/datagov.ts +907 -0
- package/src/datagovKey.ts +68 -0
- package/src/datasource.ts +721 -0
- package/src/disclosure.ts +61 -0
- package/src/ecfr.ts +231 -127
- package/src/echo.ts +496 -0
- package/src/edgar.ts +3014 -0
- package/src/errors.ts +303 -0
- package/src/fac.ts +529 -0
- package/src/far.ts +1007 -0
- package/src/fdic.ts +2052 -0
- package/src/federal-register.ts +706 -191
- package/src/fema.ts +541 -0
- package/src/fpds.ts +620 -0
- package/src/gao.ts +744 -0
- package/src/govinfo.ts +497 -0
- package/src/grants.ts +290 -155
- package/src/gsa-csv.ts +992 -0
- package/src/integrity.ts +928 -0
- package/src/meta.ts +292 -0
- package/src/nih.ts +375 -0
- package/src/nppes.ts +834 -0
- package/src/nsf.ts +706 -0
- package/src/nvd.ts +1124 -0
- package/src/ofac.ts +1166 -0
- package/src/pricing.ts +1075 -0
- package/src/sam-gov/client.ts +345 -63
- package/src/sam-gov/index.ts +5 -1
- package/src/sam-gov/types.ts +22 -0
- package/src/sba.ts +357 -0
- package/src/server.ts +4559 -327
- package/src/snapshot.ts +192 -0
- package/src/socrata.ts +532 -0
- package/src/treasury.ts +575 -0
- package/src/usaspending.ts +2680 -925
- package/src/usitc.ts +420 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* datagov-catalog.ts — data.gov v4 Catalog API (`api.gsa.gov/technology/datagov/v4`)
|
|
3
|
+
* federal DATASET DISCOVERY (ADR-0046, resilience-initiative Phase 3).
|
|
4
|
+
*
|
|
5
|
+
* WHAT IT ADDS: data.gov RETIRED its CKAN `package_search` endpoint in 2025; the
|
|
6
|
+
* v4 Catalog API is its replacement. This restores federal open-dataset DISCOVERY
|
|
7
|
+
* across all publishing agencies (hundreds of thousands of datasets) as a NEW
|
|
8
|
+
* keyless-first source. It is a SEPARATE host from datagov.ts's api.data.gov trio
|
|
9
|
+
* (Regulations.gov / Congress.gov), but it shares the IDENTICAL api.data.gov key
|
|
10
|
+
* (`api.gsa.gov` accepts the same DATA_GOV_API_KEY / DEMO_KEY via `X-Api-Key`), so
|
|
11
|
+
* it REUSES the audited `datagovKey.ts` key seam verbatim (keyHeader / keyModeLabel
|
|
12
|
+
* / pushKeyNote) — keylessMode:false (genuinely keyed, mirroring the regulations trio).
|
|
13
|
+
*
|
|
14
|
+
* The module writes ZERO fetch/coercion/error/meta code — it REUSES `getJson`
|
|
15
|
+
* (redirect:"error", the X-Api-Key header) / `driftError` / `str` (coerce.ts,
|
|
16
|
+
* null-never-empty-string) / `withMeta`·`buildMeta` / `ResponseMeta.nextCursor`,
|
|
17
|
+
* and MIRRORS datagov.ts's `searchDockets` schema_drift catch-ladder verbatim.
|
|
18
|
+
*
|
|
19
|
+
* ★ SSRF: the host is a compile-time literal (`DATAGOV_CATALOG_HOST`); every filter
|
|
20
|
+
* (`_q`/`organization`/`_size`/`_format`) rides in a MODULE-BUILT `URLSearchParams`
|
|
21
|
+
* assembled key-by-key from validated typed args — NO raw-query passthrough. The
|
|
22
|
+
* opaque `cursor` is charclass-validated (`^[A-Za-z0-9+/=_-]{1,4096}$`, rejecting
|
|
23
|
+
* `../` / spaces / `%`) BEFORE it rides the `after=` query param. A post-construction
|
|
24
|
+
* hostname/protocol assertion + `redirect:"error"` lock it (fail closed on any
|
|
25
|
+
* off-host 3xx — a 3xx off api.gsa.gov could carry the X-Api-Key header away).
|
|
26
|
+
*
|
|
27
|
+
* ★ THE HONESTY PILLARS (P1-P4, captured live 2026-07-14 — the v4 facts):
|
|
28
|
+
* P1 (NO total): the v4 search response is `{ after, results, sort }` — it reports
|
|
29
|
+
* NO match count. `totalAvailable = null` (NEVER results.length, NEVER a fabricated
|
|
30
|
+
* total). Pagination is an OPAQUE `after` cursor: `hasMore = after is a non-empty
|
|
31
|
+
* string`; `nextCursor = hasMore ? after : null` (passed back VERBATIM as the next
|
|
32
|
+
* `cursor` argument). offset/nextOffset are null (a numeric offset is meaningless).
|
|
33
|
+
* P2: getJson→fetchWithRetry THROWS on 429 (rate_limited — very likely at DEMO_KEY's
|
|
34
|
+
* ~10/hr), 5xx (upstream_unavailable), timeout — NEVER a fake empty. A genuine
|
|
35
|
+
* no-match (results:[], no after) ⇒ honest empty (datasets:[], returned:0,
|
|
36
|
+
* nextCursor:null, complete:true).
|
|
37
|
+
* P3: every scalar via `str` (null-never-empty-string — a missing accessLevel /
|
|
38
|
+
* license / landingPage is null, NEVER "").
|
|
39
|
+
* P4: `body.results` absent/non-array ⇒ driftError; a 200 non-JSON body ⇒ schema_drift
|
|
40
|
+
* via the catch-ladder (ToolErrorCarrier rethrow FIRST so a 429/5xx keeps its
|
|
41
|
+
* taxonomy → SyntaxError→driftError → bare rethrow).
|
|
42
|
+
* accessLevel is surfaced VERBATIM (public / restricted public / non-public), null
|
|
43
|
+
* when absent — the consumer judges the dataset's openness (this tool only
|
|
44
|
+
* DISCOVERS datasets; it does not ingest distributions).
|
|
45
|
+
*/
|
|
46
|
+
import { type MetaBundle } from "./meta.js";
|
|
47
|
+
export declare const DATAGOV_CATALOG_HOST = "api.gsa.gov";
|
|
48
|
+
export declare const DATAGOV_CURSOR_RE: RegExp;
|
|
49
|
+
export type CatalogDistribution = {
|
|
50
|
+
title: string | null;
|
|
51
|
+
format: string | null;
|
|
52
|
+
};
|
|
53
|
+
export type CatalogDataset = {
|
|
54
|
+
id: string | null;
|
|
55
|
+
title: string | null;
|
|
56
|
+
organization: string | null;
|
|
57
|
+
description: string | null;
|
|
58
|
+
accessLevel: string | null;
|
|
59
|
+
license: string | null;
|
|
60
|
+
landingPage: string | null;
|
|
61
|
+
modified: string | null;
|
|
62
|
+
lastHarvested: string | null;
|
|
63
|
+
keywords: string[];
|
|
64
|
+
themes: string[];
|
|
65
|
+
distributions: CatalogDistribution[];
|
|
66
|
+
identifier: string | null;
|
|
67
|
+
};
|
|
68
|
+
export type DatagovSearchDatasetsArgs = {
|
|
69
|
+
query?: string;
|
|
70
|
+
organization?: string;
|
|
71
|
+
limit?: number;
|
|
72
|
+
cursor?: string;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Search the data.gov v4 dataset catalog (the CKAN-retirement replacement).
|
|
76
|
+
* Filters: `query` (→_q), `organization` (publisher slug), `limit` (→_size),
|
|
77
|
+
* `cursor` (→after, the opaque continuation). The query is MODULE-BUILT from
|
|
78
|
+
* validated typed args through URLSearchParams (NO raw passthrough); `_format=json`
|
|
79
|
+
* is ALWAYS appended. Returns curated dataset rows + honest `_meta`: totalAvailable
|
|
80
|
+
* is NULL (the v4 API reports no count — P1), the opaque-cursor continuation, the
|
|
81
|
+
* accessLevel openness field surfaced verbatim, and the DEMO_KEY rate disclosure.
|
|
82
|
+
*/
|
|
83
|
+
export declare function searchDatasets(args: DatagovSearchDatasetsArgs): Promise<MetaBundle>;
|
|
84
|
+
//# sourceMappingURL=datagov-catalog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datagov-catalog.d.ts","sourceRoot":"","sources":["../src/datagov-catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAKH,OAAO,EAAY,KAAK,UAAU,EAAqB,MAAM,WAAW,CAAC;AAOzE,eAAO,MAAM,oBAAoB,gBAAgB,CAAC;AAelD,eAAO,MAAM,iBAAiB,QAA2C,CAAC;AAO1E,MAAM,MAAM,mBAAmB,GAAG;IAChC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,aAAa,EAAE,mBAAmB,EAAE,CAAC;IACrC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B,CAAC;AA2EF,MAAM,MAAM,yBAAyB,GAAG;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,yBAAyB,GAC9B,OAAO,CAAC,UAAU,CAAC,CAyGrB"}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* datagov-catalog.ts — data.gov v4 Catalog API (`api.gsa.gov/technology/datagov/v4`)
|
|
3
|
+
* federal DATASET DISCOVERY (ADR-0046, resilience-initiative Phase 3).
|
|
4
|
+
*
|
|
5
|
+
* WHAT IT ADDS: data.gov RETIRED its CKAN `package_search` endpoint in 2025; the
|
|
6
|
+
* v4 Catalog API is its replacement. This restores federal open-dataset DISCOVERY
|
|
7
|
+
* across all publishing agencies (hundreds of thousands of datasets) as a NEW
|
|
8
|
+
* keyless-first source. It is a SEPARATE host from datagov.ts's api.data.gov trio
|
|
9
|
+
* (Regulations.gov / Congress.gov), but it shares the IDENTICAL api.data.gov key
|
|
10
|
+
* (`api.gsa.gov` accepts the same DATA_GOV_API_KEY / DEMO_KEY via `X-Api-Key`), so
|
|
11
|
+
* it REUSES the audited `datagovKey.ts` key seam verbatim (keyHeader / keyModeLabel
|
|
12
|
+
* / pushKeyNote) — keylessMode:false (genuinely keyed, mirroring the regulations trio).
|
|
13
|
+
*
|
|
14
|
+
* The module writes ZERO fetch/coercion/error/meta code — it REUSES `getJson`
|
|
15
|
+
* (redirect:"error", the X-Api-Key header) / `driftError` / `str` (coerce.ts,
|
|
16
|
+
* null-never-empty-string) / `withMeta`·`buildMeta` / `ResponseMeta.nextCursor`,
|
|
17
|
+
* and MIRRORS datagov.ts's `searchDockets` schema_drift catch-ladder verbatim.
|
|
18
|
+
*
|
|
19
|
+
* ★ SSRF: the host is a compile-time literal (`DATAGOV_CATALOG_HOST`); every filter
|
|
20
|
+
* (`_q`/`organization`/`_size`/`_format`) rides in a MODULE-BUILT `URLSearchParams`
|
|
21
|
+
* assembled key-by-key from validated typed args — NO raw-query passthrough. The
|
|
22
|
+
* opaque `cursor` is charclass-validated (`^[A-Za-z0-9+/=_-]{1,4096}$`, rejecting
|
|
23
|
+
* `../` / spaces / `%`) BEFORE it rides the `after=` query param. A post-construction
|
|
24
|
+
* hostname/protocol assertion + `redirect:"error"` lock it (fail closed on any
|
|
25
|
+
* off-host 3xx — a 3xx off api.gsa.gov could carry the X-Api-Key header away).
|
|
26
|
+
*
|
|
27
|
+
* ★ THE HONESTY PILLARS (P1-P4, captured live 2026-07-14 — the v4 facts):
|
|
28
|
+
* P1 (NO total): the v4 search response is `{ after, results, sort }` — it reports
|
|
29
|
+
* NO match count. `totalAvailable = null` (NEVER results.length, NEVER a fabricated
|
|
30
|
+
* total). Pagination is an OPAQUE `after` cursor: `hasMore = after is a non-empty
|
|
31
|
+
* string`; `nextCursor = hasMore ? after : null` (passed back VERBATIM as the next
|
|
32
|
+
* `cursor` argument). offset/nextOffset are null (a numeric offset is meaningless).
|
|
33
|
+
* P2: getJson→fetchWithRetry THROWS on 429 (rate_limited — very likely at DEMO_KEY's
|
|
34
|
+
* ~10/hr), 5xx (upstream_unavailable), timeout — NEVER a fake empty. A genuine
|
|
35
|
+
* no-match (results:[], no after) ⇒ honest empty (datasets:[], returned:0,
|
|
36
|
+
* nextCursor:null, complete:true).
|
|
37
|
+
* P3: every scalar via `str` (null-never-empty-string — a missing accessLevel /
|
|
38
|
+
* license / landingPage is null, NEVER "").
|
|
39
|
+
* P4: `body.results` absent/non-array ⇒ driftError; a 200 non-JSON body ⇒ schema_drift
|
|
40
|
+
* via the catch-ladder (ToolErrorCarrier rethrow FIRST so a 429/5xx keeps its
|
|
41
|
+
* taxonomy → SyntaxError→driftError → bare rethrow).
|
|
42
|
+
* accessLevel is surfaced VERBATIM (public / restricted public / non-public), null
|
|
43
|
+
* when absent — the consumer judges the dataset's openness (this tool only
|
|
44
|
+
* DISCOVERS datasets; it does not ingest distributions).
|
|
45
|
+
*/
|
|
46
|
+
import { ToolErrorCarrier } from "./errors.js";
|
|
47
|
+
import { getJson, driftError } from "./datasource.js";
|
|
48
|
+
import { str } from "./coerce.js";
|
|
49
|
+
import { withMeta } from "./meta.js";
|
|
50
|
+
// The SHARED api.data.gov key seam (ADR-0010 §2). api.gsa.gov accepts the SAME
|
|
51
|
+
// DATA_GOV_API_KEY / DEMO_KEY via the X-Api-Key header, so this is a THIRD consumer
|
|
52
|
+
// of the audited key discipline — a key-leak regression now fails this suite too.
|
|
53
|
+
import { keyHeader, keyModeLabel, pushKeyNote } from "./datagovKey.js";
|
|
54
|
+
// ─── Fixed endpoint (SSRF core — compile-time CONSTANTS) ──────────
|
|
55
|
+
export const DATAGOV_CATALOG_HOST = "api.gsa.gov";
|
|
56
|
+
const DATAGOV_CATALOG_BASE = "/technology/datagov/v4";
|
|
57
|
+
const DATAGOV_CATALOG_SEARCH_PATH = `${DATAGOV_CATALOG_BASE}/search`;
|
|
58
|
+
// HOST+path label — surfaces in ToolError.upstreamEndpoint; the key rides ONLY in
|
|
59
|
+
// the X-Api-Key header, so no token can ever appear here.
|
|
60
|
+
const DATAGOV_CATALOG_LABEL = "datagov-catalog:/technology/datagov/v4/search";
|
|
61
|
+
const DATAGOV_CATALOG_SOURCE = (mode) => `${DATAGOV_CATALOG_HOST} via data.gov v4 Catalog API (${mode})`;
|
|
62
|
+
// The opaque `after` cursor grammar (SSRF + injection guard). data.gov's cursor is
|
|
63
|
+
// a base64/URL-safe token — `+/=_-`, no `%` (URLSearchParams would double-encode a
|
|
64
|
+
// literal `%` ('%2e'→'%252e') and corrupt the cursor; the real tokens carry none).
|
|
65
|
+
// Validated BEFORE the value rides `after=` — a `../` / space / `%` ⇒ invalid_input,
|
|
66
|
+
// 0 fetch (mirrors clinicaltrials CT_TOKEN_RE).
|
|
67
|
+
export const DATAGOV_CURSOR_RE = new RegExp("^[A-Za-z0-9+/=_-]{1,4096}$");
|
|
68
|
+
// The P1 no-total disclosure — the load-bearing honesty caveat carried EVERY response.
|
|
69
|
+
const DATAGOV_CATALOG_NO_TOTAL_NOTE = "data.gov catalog does not report a total match count — use nextCursor to page; totalAvailable is unknown (null). Pagination is an OPAQUE cursor (offset/nextOffset are meaningless/null); pass _meta.nextCursor back as the `cursor` argument. nextCursor:null / hasMore:false means this is the last page.";
|
|
70
|
+
/** A string[] from a mixed value (drops null/empty via str), else [] when absent/non-array. */
|
|
71
|
+
function strArray(x) {
|
|
72
|
+
if (!Array.isArray(x))
|
|
73
|
+
return [];
|
|
74
|
+
return x.map((v) => str(v)).filter((v) => v !== null);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Map ONE `results[]` row → the curated dataset shape. Every scalar via `str`
|
|
78
|
+
* (null-never-empty-string — a missing accessLevel/license/landingPage is null,
|
|
79
|
+
* NEVER ""). description falls back from the row-level to the DCAT-US field.
|
|
80
|
+
* keywords/themes/distributions default to [] (an honest "none listed"); a
|
|
81
|
+
* distribution is kept only when it carries a title OR a format.
|
|
82
|
+
*/
|
|
83
|
+
function mapDataset(row) {
|
|
84
|
+
const r = (row ?? {});
|
|
85
|
+
const dcat = (r.dcat ?? {});
|
|
86
|
+
const distributions = Array.isArray(dcat.distribution)
|
|
87
|
+
? dcat.distribution
|
|
88
|
+
.map((d) => {
|
|
89
|
+
const it = (d ?? {});
|
|
90
|
+
return { title: str(it.title), format: str(it.format ?? it.mediaType) };
|
|
91
|
+
})
|
|
92
|
+
.filter((x) => x.title !== null || x.format !== null)
|
|
93
|
+
: [];
|
|
94
|
+
return {
|
|
95
|
+
id: str(r.slug),
|
|
96
|
+
title: str(r.title),
|
|
97
|
+
organization: str(r.organization),
|
|
98
|
+
description: str(r.description ?? dcat.description),
|
|
99
|
+
accessLevel: str(dcat.accessLevel),
|
|
100
|
+
license: str(dcat.license),
|
|
101
|
+
landingPage: str(dcat.landingPage),
|
|
102
|
+
modified: str(dcat.modified),
|
|
103
|
+
lastHarvested: str(r.last_harvested_date),
|
|
104
|
+
keywords: strArray(r.keyword),
|
|
105
|
+
themes: strArray(r.theme),
|
|
106
|
+
distributions,
|
|
107
|
+
identifier: str(dcat.identifier),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
// ─── SSRF-guarded fetch (fixed host + hostname assertion + redirect) ──
|
|
111
|
+
/**
|
|
112
|
+
* GET one data.gov v4 Catalog JSON resource. `path` is a fixed base path; all
|
|
113
|
+
* caller params ride in `params` (URLSearchParams, encoded). Builds
|
|
114
|
+
* `https://${DATAGOV_CATALOG_HOST}${path}?${params}` on the FIXED host, asserts the
|
|
115
|
+
* CONSTRUCTED URL's hostname === the host over https (belt-and-suspenders), sets
|
|
116
|
+
* `redirect:"error"` (an off-host 3xx must NOT be followed — it could carry the
|
|
117
|
+
* X-Api-Key header to a foreign host), and attaches the key ONLY in the X-Api-Key
|
|
118
|
+
* header. `label` is host+path only (→ ToolError.upstreamEndpoint).
|
|
119
|
+
*/
|
|
120
|
+
async function getDatagovCatalog(path, label, params) {
|
|
121
|
+
const qs = params.toString();
|
|
122
|
+
const url = `https://${DATAGOV_CATALOG_HOST}${path}${qs ? `?${qs}` : ""}`;
|
|
123
|
+
const built = new URL(url);
|
|
124
|
+
if (built.hostname !== DATAGOV_CATALOG_HOST || built.protocol !== "https:") {
|
|
125
|
+
throw new ToolErrorCarrier({
|
|
126
|
+
kind: "invalid_input",
|
|
127
|
+
message: `Constructed data.gov catalog URL host ${JSON.stringify(built.hostname)} (${built.protocol}) does not match the fixed host ${JSON.stringify(DATAGOV_CATALOG_HOST)} over https — refusing to fetch (SSRF safety).`,
|
|
128
|
+
retryable: false,
|
|
129
|
+
upstreamEndpoint: label,
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
// The key rides in the X-Api-Key header ONLY (never the URL/label/_meta);
|
|
133
|
+
// redirect:"error" (fail closed on any off-host 3xx).
|
|
134
|
+
return getJson(url, { label, headers: keyHeader(), redirect: "error" });
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Search the data.gov v4 dataset catalog (the CKAN-retirement replacement).
|
|
138
|
+
* Filters: `query` (→_q), `organization` (publisher slug), `limit` (→_size),
|
|
139
|
+
* `cursor` (→after, the opaque continuation). The query is MODULE-BUILT from
|
|
140
|
+
* validated typed args through URLSearchParams (NO raw passthrough); `_format=json`
|
|
141
|
+
* is ALWAYS appended. Returns curated dataset rows + honest `_meta`: totalAvailable
|
|
142
|
+
* is NULL (the v4 API reports no count — P1), the opaque-cursor continuation, the
|
|
143
|
+
* accessLevel openness field surfaced verbatim, and the DEMO_KEY rate disclosure.
|
|
144
|
+
*/
|
|
145
|
+
export async function searchDatasets(args) {
|
|
146
|
+
const label = DATAGOV_CATALOG_LABEL;
|
|
147
|
+
const limit = args.limit ?? 20;
|
|
148
|
+
// ── Belt-and-suspenders cursor grammar (behind the server's Zod). A bad cursor
|
|
149
|
+
// would ride `after=` and either 400 or silently mis-page; reject it pre-fetch
|
|
150
|
+
// (0 network call) so a `../` / space / `%` can never reach the query. ──
|
|
151
|
+
if (args.cursor !== undefined && !DATAGOV_CURSOR_RE.test(args.cursor)) {
|
|
152
|
+
throw new ToolErrorCarrier({
|
|
153
|
+
kind: "invalid_input",
|
|
154
|
+
message: `Invalid cursor (opaque continuation token) — must be a ≤4096-char base64/URL-safe token (no spaces, '../', or '%'). Pass back the _meta.nextCursor from the previous page.`,
|
|
155
|
+
retryable: false,
|
|
156
|
+
upstreamEndpoint: label,
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
// ── Build the query from VALIDATED typed args, key-by-key (SSRF: no raw
|
|
160
|
+
// passthrough). _format=json is ALWAYS appended. ──
|
|
161
|
+
const params = new URLSearchParams();
|
|
162
|
+
const filtersApplied = [];
|
|
163
|
+
if (args.query !== undefined) {
|
|
164
|
+
params.set("_q", args.query);
|
|
165
|
+
filtersApplied.push("query");
|
|
166
|
+
}
|
|
167
|
+
if (args.organization !== undefined) {
|
|
168
|
+
params.set("organization", args.organization);
|
|
169
|
+
filtersApplied.push("organization");
|
|
170
|
+
}
|
|
171
|
+
params.set("_size", String(limit));
|
|
172
|
+
if (args.cursor !== undefined) {
|
|
173
|
+
params.set("after", args.cursor);
|
|
174
|
+
filtersApplied.push("cursor");
|
|
175
|
+
}
|
|
176
|
+
params.set("_format", "json");
|
|
177
|
+
// ── The typed catch-ladder (datagov.ts searchDockets shape, VERBATIM). Preserve
|
|
178
|
+
// the 429/404/5xx/400/timeout ToolErrorCarrier taxonomy FIRST (LOAD-BEARING:
|
|
179
|
+
// the DEMO_KEY-~10/hr 429→rate_limited frontier would regress to schema_drift
|
|
180
|
+
// under a broader catch); reclassify a 200 non-JSON `.json()` SyntaxError to
|
|
181
|
+
// schema_drift SECOND; bare-rethrow LAST. The host-assert ToolErrorCarrier is
|
|
182
|
+
// also rethrown first. ──
|
|
183
|
+
let body;
|
|
184
|
+
try {
|
|
185
|
+
body = await getDatagovCatalog(DATAGOV_CATALOG_SEARCH_PATH, label, params);
|
|
186
|
+
}
|
|
187
|
+
catch (e) {
|
|
188
|
+
if (e instanceof ToolErrorCarrier)
|
|
189
|
+
throw e;
|
|
190
|
+
if (e instanceof SyntaxError)
|
|
191
|
+
throw driftError(label, "data.gov catalog returned a non-JSON body at HTTP 200 — schema drift.");
|
|
192
|
+
throw e;
|
|
193
|
+
}
|
|
194
|
+
const b = (body ?? {});
|
|
195
|
+
// ── P4: `results` MUST be an array (a missing/string/null results is drift,
|
|
196
|
+
// never a fabricated empty — a TypeError must never mask drift as
|
|
197
|
+
// upstream_unavailable). ──
|
|
198
|
+
if (!Array.isArray(b.results)) {
|
|
199
|
+
throw driftError(label, "data.gov catalog shape drift — /search response.results must be an array.");
|
|
200
|
+
}
|
|
201
|
+
const datasets = b.results.map(mapDataset);
|
|
202
|
+
const returned = datasets.length;
|
|
203
|
+
// ── P1: NO total is reported ⇒ totalAvailable = null (NEVER results.length,
|
|
204
|
+
// NEVER a fabricated total). Pagination is the opaque `after` cursor: hasMore
|
|
205
|
+
// from cursor-presence; nextCursor passed back VERBATIM (never derived). ──
|
|
206
|
+
const rawAfter = b.after;
|
|
207
|
+
const hasMore = typeof rawAfter === "string" && rawAfter.length > 0;
|
|
208
|
+
const nextCursor = hasMore ? rawAfter : null;
|
|
209
|
+
const notes = [DATAGOV_CATALOG_NO_TOTAL_NOTE];
|
|
210
|
+
pushKeyNote(notes);
|
|
211
|
+
if (filtersApplied.length === 0) {
|
|
212
|
+
notes.push("No filters were applied — this is an unscoped scan of the WHOLE data.gov catalog. Add `query` and/or `organization` for a meaningful scoped result set.");
|
|
213
|
+
}
|
|
214
|
+
return withMeta({ datasets }, {
|
|
215
|
+
source: DATAGOV_CATALOG_SOURCE(keyModeLabel()),
|
|
216
|
+
keylessMode: false, // genuinely keyed (api.data.gov X-Api-Key)
|
|
217
|
+
returned,
|
|
218
|
+
// P1 — the v4 API reports NO match count. NULL, never results.length.
|
|
219
|
+
totalAvailable: null,
|
|
220
|
+
// complete is DERIVED by buildMeta: a page with no `after` (hasMore:false) and
|
|
221
|
+
// no dropped filters ⇒ complete:true (an honest exact empty on a no-match); a
|
|
222
|
+
// page WITH an `after` cursor ⇒ hasMore:true ⇒ complete:false.
|
|
223
|
+
filtersApplied,
|
|
224
|
+
filtersDropped: [],
|
|
225
|
+
fieldsUnavailable: [],
|
|
226
|
+
// Cursor page: offset/nextOffset null (no numeric offset); continuation is
|
|
227
|
+
// nextCursor (the opaque `after` token, passed back verbatim as `cursor`).
|
|
228
|
+
pagination: { offset: null, limit, hasMore, nextOffset: null },
|
|
229
|
+
nextCursor,
|
|
230
|
+
notes,
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
//# sourceMappingURL=datagov-catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datagov-catalog.js","sourceRoot":"","sources":["../src/datagov-catalog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAsC,MAAM,WAAW,CAAC;AACzE,+EAA+E;AAC/E,oFAAoF;AACpF,kFAAkF;AAClF,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEvE,qEAAqE;AACrE,MAAM,CAAC,MAAM,oBAAoB,GAAG,aAAa,CAAC;AAClD,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AACtD,MAAM,2BAA2B,GAAG,GAAG,oBAAoB,SAAS,CAAC;AACrE,kFAAkF;AAClF,0DAA0D;AAC1D,MAAM,qBAAqB,GAAG,+CAA+C,CAAC;AAE9E,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAE,EAAE,CAC9C,GAAG,oBAAoB,iCAAiC,IAAI,GAAG,CAAC;AAElE,mFAAmF;AACnF,mFAAmF;AACnF,mFAAmF;AACnF,qFAAqF;AACrF,gDAAgD;AAChD,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,4BAA4B,CAAC,CAAC;AAE1E,uFAAuF;AACvF,MAAM,6BAA6B,GACjC,6SAA6S,CAAC;AAwBhT,+FAA+F;AAC/F,SAAS,QAAQ,CAAC,CAAU;IAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,GAAY;IAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,CAA4B,CAAC;IACjD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAA4B,CAAC;IACvD,MAAM,aAAa,GAA0B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;QAC3E,CAAC,CAAE,IAAI,CAAC,YAA0B;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAA4B,CAAC;YAChD,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1E,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QACzD,CAAC,CAAC,EAAE,CAAC;IACP,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACf,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QACnB,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC;QACjC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC;QACnD,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;QAClC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;QAC1B,WAAW,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;QAClC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC5B,aAAa,EAAE,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC;QACzC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;QAC7B,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;QACzB,aAAa;QACb,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE;;;;;;;;GAQG;AACH,KAAK,UAAU,iBAAiB,CAC9B,IAAY,EACZ,KAAa,EACb,MAAuB;IAEvB,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,WAAW,oBAAoB,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,oBAAoB,IAAI,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3E,MAAM,IAAI,gBAAgB,CAAC;YACzB,IAAI,EAAE,eAAe;YACrB,OAAO,EAAE,yCAAyC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,mCAAmC,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,gDAAgD;YAC1N,SAAS,EAAE,KAAK;YAChB,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;IACL,CAAC;IACD,0EAA0E;IAC1E,sDAAsD;IACtD,OAAO,OAAO,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AAC1E,CAAC;AAUD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA+B;IAE/B,MAAM,KAAK,GAAG,qBAAqB,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IAE/B,gFAAgF;IAChF,kFAAkF;IAClF,6EAA6E;IAC7E,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,gBAAgB,CAAC;YACzB,IAAI,EAAE,eAAe;YACrB,OAAO,EAAE,4KAA4K;YACrL,SAAS,EAAE,KAAK;YAChB,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;IACrC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9C,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE9B,iFAAiF;IACjF,gFAAgF;IAChF,iFAAiF;IACjF,gFAAgF;IAChF,iFAAiF;IACjF,6BAA6B;IAC7B,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,iBAAiB,CAAC,2BAA2B,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,gBAAgB;YAAE,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,YAAY,WAAW;YAC1B,MAAM,UAAU,CACd,KAAK,EACL,uEAAuE,CACxE,CAAC;QACJ,MAAM,CAAC,CAAC;IACV,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAA2C,CAAC;IAEjE,6EAA6E;IAC7E,qEAAqE;IACrE,+BAA+B;IAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,MAAM,UAAU,CACd,KAAK,EACL,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAI,CAAC,CAAC,OAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC;IAEjC,6EAA6E;IAC7E,iFAAiF;IACjF,+EAA+E;IAC/E,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC;IACzB,MAAM,OAAO,GAAG,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAE,QAAmB,CAAC,CAAC,CAAC,IAAI,CAAC;IAEzD,MAAM,KAAK,GAAa,CAAC,6BAA6B,CAAC,CAAC;IACxD,WAAW,CAAC,KAAK,CAAC,CAAC;IACnB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CACR,yJAAyJ,CAC1J,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CACb,EAAE,QAAQ,EAAE,EACZ;QACE,MAAM,EAAE,sBAAsB,CAAC,YAAY,EAAE,CAAC;QAC9C,WAAW,EAAE,KAAK,EAAE,2CAA2C;QAC/D,QAAQ;QACR,sEAAsE;QACtE,cAAc,EAAE,IAAI;QACpB,+EAA+E;QAC/E,8EAA8E;QAC9E,+DAA+D;QAC/D,cAAc;QACd,cAAc,EAAE,EAAE;QAClB,iBAAiB,EAAE,EAAE;QACrB,2EAA2E;QAC3E,2EAA2E;QAC3E,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE;QAC9D,UAAU;QACV,KAAK;KAC0B,CAClC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* datagov.ts — the api.data.gov KEYED trio (slice 1: Regulations.gov +
|
|
3
|
+
* Congress.gov). ADR-0007.
|
|
4
|
+
*
|
|
5
|
+
* This is the project's FIRST KEYED (non-keyless) source — every prior source
|
|
6
|
+
* (Treasury, EDGAR, Socrata, CKAN) is anonymous. It exercises policy② ("keyless-
|
|
7
|
+
* first, but when a free public key is required, research + automate its
|
|
8
|
+
* issuance"), so the load-bearing concern is a SAFE env-key adapter whose secret
|
|
9
|
+
* can NEVER leak into a ToolError, a URL, a log, or `_meta`. Built on the R2
|
|
10
|
+
* `DataSource` port (ADR-0005) exactly like CKAN — it writes ZERO fetch/coercion/
|
|
11
|
+
* error/meta code (reuses `getJson`/`driftError`/`num`·`str`/`withMeta`) and
|
|
12
|
+
* COPIES (does not import) the Socrata/CKAN SSRF + honesty PATTERN.
|
|
13
|
+
*
|
|
14
|
+
* ★ THE KEY-SECURITY DISCIPLINE (ADR-0007 §2 — the load-bearing rules):
|
|
15
|
+
* 1. The key travels in `headers:{ "X-Api-Key": <key> }` ONLY — NEVER in the
|
|
16
|
+
* URL/query (no `?api_key=`). Live-verified: the `X-Api-Key` header works for
|
|
17
|
+
* BOTH api.regulations.gov and api.congress.gov, so the key is structurally
|
|
18
|
+
* absent from the request URL (→ never in a CDN/proxy/access log, never in
|
|
19
|
+
* `getJson`'s `label`, never in `ToolError.upstreamEndpoint`).
|
|
20
|
+
* 2. `label` is HOST+PATH only (e.g. "regulations:/v4/documents",
|
|
21
|
+
* "congress:/v3/bill") — never the full URL, never a token. This is what
|
|
22
|
+
* reaches `ToolError.upstreamEndpoint`.
|
|
23
|
+
* 3. `_meta.source`/`notes` are host + key-MODE only ("…(DEMO_KEY)" /
|
|
24
|
+
* "…(DATA_GOV_API_KEY)") — never the URL, never the key value, never the
|
|
25
|
+
* `X-Api-Key` header.
|
|
26
|
+
* 4. Headers are never logged/echoed; upstream `next`/`pagination.next` URLs are
|
|
27
|
+
* NEVER surfaced verbatim — pagination is re-derived NUMERICALLY.
|
|
28
|
+
* 5. Never commit the key — read from env only; the DEMO_KEY fallback is a
|
|
29
|
+
* literal public constant (safe in source), the real key never is.
|
|
30
|
+
*
|
|
31
|
+
* ★ KEYLESS-FIRST UX (policy② / §3): DATA_GOV_API_KEY from env, else the public
|
|
32
|
+
* literal "DEMO_KEY" + a `_meta.notes` disclosure of the ~10 req/hr shared ceiling
|
|
33
|
+
* and the free-key signup path. `keylessMode:false` (this is the FIRST source to
|
|
34
|
+
* report it — it is genuinely keyed).
|
|
35
|
+
*
|
|
36
|
+
* ★ SSRF (§4): two fixed hosts (constants, no free host param); all caller params
|
|
37
|
+
* via URLSearchParams; path segments that interpolate caller input (Congress
|
|
38
|
+
* `/v3/bill/{congress}/{billType}/{billNumber}`) are Zod-constrained to int/enum;
|
|
39
|
+
* post-construction `hostname===host && https` assertion; `redirect:"error"`.
|
|
40
|
+
*
|
|
41
|
+
* ★ HONESTY (§5): totals are EXACT integers read from the PRIMARY container
|
|
42
|
+
* (`meta.totalElements` / `pagination.count`) with container guards (a null/absent
|
|
43
|
+
* container or a non-number total → `driftError`, NOT a TypeError/upstream_unavail);
|
|
44
|
+
* the primary array (`data`/`bills`) must be an array or → `driftError`; a
|
|
45
|
+
* genuine-empty (0 results) → complete:true/total:0; an outage/5xx/timeout throws
|
|
46
|
+
* (never a fake empty). Regulations.gov's 40-page/10,000-record HARD CAP is
|
|
47
|
+
* handled with the EDGAR window-cap pattern (B1): `hasMore:true` (more genuinely
|
|
48
|
+
* exists) BUT `nextOffset:null` (no reachable continuation) + a disclosing note.
|
|
49
|
+
*/
|
|
50
|
+
import { num } from "./coerce.js";
|
|
51
|
+
import { type MetaBundle } from "./meta.js";
|
|
52
|
+
export { num };
|
|
53
|
+
export declare const REGULATIONS_HOST = "api.regulations.gov";
|
|
54
|
+
export declare const CONGRESS_HOST = "api.congress.gov";
|
|
55
|
+
export declare const REGULATIONS_SORTS: readonly ["-postedDate", "postedDate", "-lastModifiedDate", "lastModifiedDate", "-commentEndDate"];
|
|
56
|
+
export type RegulationsSort = (typeof REGULATIONS_SORTS)[number];
|
|
57
|
+
export declare const REGULATIONS_DOCUMENT_TYPES: readonly ["Rule", "Proposed Rule", "Notice", "Supporting & Related Material", "Other"];
|
|
58
|
+
export type RegulationsDocumentType = (typeof REGULATIONS_DOCUMENT_TYPES)[number];
|
|
59
|
+
export declare const REGULATIONS_DOCKET_TYPES: readonly ["Rulemaking", "Nonrulemaking"];
|
|
60
|
+
export type RegulationsDocketType = (typeof REGULATIONS_DOCKET_TYPES)[number];
|
|
61
|
+
export declare const REGULATIONS_DOCKET_SORTS: readonly ["-lastModifiedDate", "lastModifiedDate", "title", "-title"];
|
|
62
|
+
export type RegulationsDocketSort = (typeof REGULATIONS_DOCKET_SORTS)[number];
|
|
63
|
+
export declare const CONGRESS_BILL_TYPES: readonly ["hr", "s", "hjres", "sjres", "hconres", "sconres", "hres", "sres"];
|
|
64
|
+
export type CongressBillType = (typeof CONGRESS_BILL_TYPES)[number];
|
|
65
|
+
type RegulationsSearchArgs = {
|
|
66
|
+
searchTerm?: string;
|
|
67
|
+
query?: string;
|
|
68
|
+
agencyId?: string;
|
|
69
|
+
docketId?: string;
|
|
70
|
+
documentType?: RegulationsDocumentType;
|
|
71
|
+
withinCommentPeriod?: boolean;
|
|
72
|
+
postedDateGe?: string;
|
|
73
|
+
postedDateLe?: string;
|
|
74
|
+
sort?: RegulationsSort;
|
|
75
|
+
pageNumber?: number;
|
|
76
|
+
pageSize?: number;
|
|
77
|
+
};
|
|
78
|
+
/** Tool: regulations_search_documents. */
|
|
79
|
+
export declare function searchDocuments(args: RegulationsSearchArgs): Promise<MetaBundle>;
|
|
80
|
+
/** Tool: regulations_search_comments. */
|
|
81
|
+
export declare function searchComments(args: RegulationsSearchArgs): Promise<MetaBundle>;
|
|
82
|
+
type RegulationsSearchDocketsArgs = {
|
|
83
|
+
searchTerm?: string;
|
|
84
|
+
query?: string;
|
|
85
|
+
agencyId?: string;
|
|
86
|
+
docketType?: RegulationsDocketType;
|
|
87
|
+
lastModifiedDateGe?: string;
|
|
88
|
+
lastModifiedDateLe?: string;
|
|
89
|
+
sort?: RegulationsDocketSort;
|
|
90
|
+
limit?: number;
|
|
91
|
+
pageNumber?: number;
|
|
92
|
+
};
|
|
93
|
+
type RegulationsGetDocketArgs = {
|
|
94
|
+
docketId: string;
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Tool: regulations_search_dockets (`GET /v4/dockets`). Lists rulemaking/
|
|
98
|
+
* nonrulemaking docket CONTAINERS with the same 40-page/10,000-record ceiling and
|
|
99
|
+
* `totalElements`-exact total doctrine as regulationsSearch.
|
|
100
|
+
*
|
|
101
|
+
* ★ min-5 floor (ADR-0044): the API 400s on page[size]<5. The friendly `limit`
|
|
102
|
+
* exposes 1..250; the wire page[size] is `max(5, limit)` and a `limit<5` returns
|
|
103
|
+
* the first `limit` of the fetched rows client-side (disclosed) — `totalAvailable`
|
|
104
|
+
* stays the EXACT server total.
|
|
105
|
+
*/
|
|
106
|
+
export declare function searchDockets(args: RegulationsSearchDocketsArgs): Promise<MetaBundle>;
|
|
107
|
+
/**
|
|
108
|
+
* Tool: regulations_get_docket (`GET /v4/dockets/{docketId}`). Single-docket
|
|
109
|
+
* detail — the ONLY view carrying `rin`. `docketId` is charclass-validated at the
|
|
110
|
+
* Zod layer (S1); it is the only caller value reaching a path segment.
|
|
111
|
+
*/
|
|
112
|
+
export declare function getDocket(args: RegulationsGetDocketArgs): Promise<MetaBundle>;
|
|
113
|
+
type CongressSearchBillsArgs = {
|
|
114
|
+
query?: string;
|
|
115
|
+
congress?: number;
|
|
116
|
+
billType?: CongressBillType;
|
|
117
|
+
fromDateTime?: string;
|
|
118
|
+
toDateTime?: string;
|
|
119
|
+
offset?: number;
|
|
120
|
+
limit?: number;
|
|
121
|
+
};
|
|
122
|
+
/**
|
|
123
|
+
* Tool: congress_search_bills. Lists bills via `/v3/bill`, `/v3/bill/{congress}`,
|
|
124
|
+
* or `/v3/bill/{congress}/{billType}` (congress/billType are Zod-constrained PATH
|
|
125
|
+
* segments — §4). Numeric offset/limit pagination; `totalAvailable =
|
|
126
|
+
* pagination.count` (EXACT). `nextOffset` is re-derived NUMERICALLY (never the
|
|
127
|
+
* upstream `pagination.next` URL — §2 rule 4).
|
|
128
|
+
*
|
|
129
|
+
* HONESTY on `query`: the Congress.gov `/v3/bill` endpoint has NO keyword-search
|
|
130
|
+
* parameter, so a supplied `query` is NOT sent and is disclosed in
|
|
131
|
+
* `filtersDropped` (a filter we cannot honor is surfaced, never silently ignored).
|
|
132
|
+
*/
|
|
133
|
+
export declare function searchBills(args: CongressSearchBillsArgs): Promise<MetaBundle>;
|
|
134
|
+
type CongressGetBillArgs = {
|
|
135
|
+
congress: number;
|
|
136
|
+
billType: CongressBillType;
|
|
137
|
+
billNumber: number;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Tool: congress_get_bill. Fetches one bill via
|
|
141
|
+
* `/v3/bill/{congress}/{billType}/{billNumber}` (all three path segments are
|
|
142
|
+
* Zod-constrained to int/enum — §4, no injection). Single-record honest `_meta`
|
|
143
|
+
* (complete:true). A 404 (nonexistent bill) → not_found (never fabricated).
|
|
144
|
+
*/
|
|
145
|
+
export declare function getBill(args: CongressGetBillArgs): Promise<MetaBundle>;
|
|
146
|
+
//# sourceMappingURL=datagov.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datagov.d.ts","sourceRoot":"","sources":["../src/datagov.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAIH,OAAO,EAAE,GAAG,EAAO,MAAM,aAAa,CAAC;AACvC,OAAO,EAAY,KAAK,UAAU,EAAqB,MAAM,WAAW,CAAC;AAWzE,OAAO,EAAE,GAAG,EAAE,CAAC;AAGf,eAAO,MAAM,gBAAgB,wBAAwB,CAAC;AACtD,eAAO,MAAM,aAAa,qBAAqB,CAAC;AAahD,eAAO,MAAM,iBAAiB,oGAMpB,CAAC;AACX,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAIjE,eAAO,MAAM,0BAA0B,wFAM7B,CAAC;AACX,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,0BAA0B,CAAC,CAAC,MAAM,CAAC,CAAC;AAKlF,eAAO,MAAM,wBAAwB,0CAA2C,CAAC;AACjF,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAO9E,eAAO,MAAM,wBAAwB,uEAK3B,CAAC;AACX,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAI9E,eAAO,MAAM,mBAAmB,8EAStB,CAAC;AACX,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AA4CpE,KAAK,qBAAqB,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,uBAAuB,CAAC;IACvC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AA0NF,0CAA0C;AAC1C,wBAAsB,eAAe,CACnC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAErB;AAED,yCAAyC;AACzC,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAErB;AASD,KAAK,4BAA4B,GAAG;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,IAAI,CAAC,EAAE,qBAAqB,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,KAAK,wBAAwB,GAAG;IAC9B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAoEF;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,4BAA4B,GACjC,OAAO,CAAC,UAAU,CAAC,CA0IrB;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,wBAAwB,GAC7B,OAAO,CAAC,UAAU,CAAC,CAoDrB;AA2BD,KAAK,uBAAuB,GAAG;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,uBAAuB,GAC5B,OAAO,CAAC,UAAU,CAAC,CA0FrB;AAED,KAAK,mBAAmB,GAAG;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,OAAO,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAgC5E"}
|