@cliwant/mcp-sam-gov 1.9.0 → 1.10.0

package/src/socrata.ts

@@ -1,5 +1,5 @@
 /**
- * Socrata / SODA — keyless open data for state / local (SLED) + E-rate portals.
+ * Socrata / SODA — keyless open data for state / local (SLED) + federal + E-rate portals.
  *
  * First SLED source (ADR-0004); 3rd consumer of the fetch/map/meta shape after
  * treasury.ts / edgar.ts. Fully PUBLIC, KEYLESS. ONE connector reaches ~a dozen
@@ -7,7 +7,8 @@
  * 4x4 dataset id. Hosts are a CURATED allowlist (see SSRF below); the caller
  * never supplies a free host or a free path.
  *   Row query:  https://{domain}/resource/{4x4}.json?$select=…&$where=…&$limit=…
- *   Catalog:    https://api.us.socrata.com/api/catalog/v1?domains={domain}&q=…
+ *   Catalog (host-scoped): https://{domain}/api/catalog/v1?search_context={domain}&q=…
+ *   Catalog (all-host):    https://api.us.socrata.com/api/catalog/v1?domains=…&q=…
  *
  * Three layers (mirror treasury.ts / edgar.ts):
  *   fetch — `getSocrataResource` / `getCatalog`: SSRF-guard, build the URL,
@@ -67,14 +68,19 @@
  * wrong shape (possible upstream API change). A hard schema_drift throw is
  * reserved for a PRIMARY query (rows / catalog), NEVER the secondary count.
  *
- * ALLOWLIST — LIVE-VERIFIED 2026-07-10 (each carries a real sample 4x4). All
- * `.gov` except `opendata.usac.org`:
+ * ALLOWLIST — each entry carries a real sample 4x4, live-verified on its tier's
+ * date (base state slice 2026-07-10; local + major-city + federal 2026-07-18).
+ * All `.gov` except the documented non-.gov exceptions: `opendata.usac.org` and
+ * the four major-city portals (.us/.org — see the inline blocks below):
  *   m6 — `opendata.usac.org` is a `.org` (USAC, a Congress-designated non-profit;
  *        E-rate). It is on the periodic re-verification checklist. NOTE: the
- *        federated discovery catalog (api.us.socrata.com) does NOT index USAC
- *        (returns resultSetSize 0), so `socrata_discover_datasets` will not
- *        surface it — but `socrata_query` works against it with a known 4x4
- *        (live: opendata.usac.org/resource/avi8-svp9.json → 200 bare array).
+ *        FEDERATED discovery catalog (api.us.socrata.com) does NOT index USAC
+ *        (resultSetSize 0), so the ALL-HOST `socrata_discover_datasets` (domain
+ *        omitted) won't surface it; as of loop cycle 8 a DOMAIN-scoped discover
+ *        uses USAC's own catalog (search_context) and DOES surface it (live:
+ *        opendata.usac.org/api/catalog/v1?search_context=… → resultSetSize 8).
+ *        `socrata_query` works regardless with a known 4x4 (live:
+ *        opendata.usac.org/resource/avi8-svp9.json → 200 bare array).
  *   M1 — MA is DROPPED from slice 1: `cthru.data.socrata.com` is a commercial
  *        vendor host (Tyler Technologies `.socrata.com`, not gov-controlled) and
  *        no `.gov` MA Socrata host verifies (`data.mass.gov` → the catalog
@@ -139,7 +145,55 @@ export const SOCRATA_DOMAINS = [
   "data.montgomerycountymd.gov", // Montgomery County MD — e.g. vmu2-pnrc (Contracts)
   "data.mesaaz.gov", // Mesa AZ (city) — e.g. j7s9-qiuq (Vendor Payments)
   "data.cambridgema.gov", // Cambridge MA (city) — e.g. gp98-ja4f (Contracts bid list)
+  // ── Federal open-data portal tier (loop cycle 7, 2026-07-18). The FIRST
+  //    federal Socrata hosts (prior tiers were state + local only); all .gov,
+  //    host-scoped catalog + /resource/<4x4>.json 200 bare-array + count(*)
+  //    companion live-verified. High-value B2G federal datasets (carrier/company
+  //    census, transportation infrastructure & stats, public health).
+  //    NOTE (m3/under-index): the FEDERATED aggregator (api.us.socrata.com)
+  //    under-indexes these hosts (DOT: 3 federated vs 1,873 host-scoped). As of
+  //    loop cycle 8, `socrata_discover_datasets` WITH a domain uses each host's
+  //    OWN catalog (search_context) → complete per-host discovery; only the
+  //    all-host search (domain omitted) still relies on the federated aggregator
+  //    (its `_meta` note discloses the gap). ──────────────────────────────────
+  "data.transportation.gov", // US DOT — e.g. az4n-8mr2 (Company Census File, ~4.47M carriers)
+  "data.cdc.gov", // US CDC — e.g. 9bhg-hcku (Provisional COVID-19 Deaths)
+  "data.bts.gov", // US BTS (DOT) — e.g. keg4-3bc2 (Border Crossing Entry Data)
+  // ── SLED procurement bid-catalog tier (loop cycle 11, 2026-07-19; from the
+  //    exhaustive US state+local bid-site research). All .gov; host-scoped catalog
+  //    + /resource/<4x4>.json 200 bare-array live-verified. Distinctive value:
+  //    these carry LIVE bid-cycle data (bid tabulations / anticipated
+  //    solicitations), not only award/spend. (★4 other candidate hosts —
+  //    data.iowa.gov, data.scottsdaleaz.gov, data.gilbertaz.gov, opendata.hawaii.gov
+  //    — were REJECTED: their /api/catalog/v1 AND /resource endpoints 404 (Next.js/
+  //    Express apps, not Socrata — a research-agent "Socrata" claim that live
+  //    verification disproved). Never add a host without a 200 bare-array probe.) ─
+  "datacatalog.cookcountyil.gov", // Cook County IL — e.g. 32au-zaqn (Bid Tabulations ~5,607; +awards qh8j-6k63, intent-to-award bgq7-v7ms — 17 procurement datasets)
+  "data.illinois.gov", // IL — e.g. 6rb8-ntpm (Future Solicitations = anticipated construction bids); re-included (cycle-1 excluded on federated-discover 0, but host-scoped catalog + /resource verified 2026-07-19)
+  "data.cincinnati-oh.gov", // Cincinnati OH (city) — e.g. 2iq3-bugw (Certified Vendors MBE/WBE; +contracts 85xi-xdtw)
+  // ── Major-city OFFICIAL portals (loop cycle 5, 2026-07-18). NON-.gov but the
+  //    unambiguously-official municipal open-data portals for the largest local
+  //    procurement markets (NYC OpenData / Chicago / DataSF / LA Controller) —
+  //    host-scoped catalog + /resource 200 bare-array verified. Trust-boundary
+  //    (like the USAC .org exception): the SSRF guard is the CURATED, FROZEN
+  //    allowlist, not the TLD; these five are the documented non-.gov entries. ──
+  "data.cityofnewyork.us", // NYC OpenData (.us, official) — e.g. dg92-zbpx (City Record: procurement notices)
+  "data.cityofchicago.org", // Chicago (.org, official) — e.g. rsxa-ify5 (Contracts)
+  "data.sfgov.org", // San Francisco / DataSF (.org, official) — e.g. cqi5-hm2d (Supplier Contracts)
+  "controllerdata.lacity.org", // LA City Controller (.org, official) — e.g. pggv-e4fn (Checkbook L.A.)
   "opendata.usac.org", // USAC E-rate (.org, m6) — e.g. avi8-svp9
+  // ── County/city procurement sweep (loop cycle 17, 2026-07-20). Discovered via
+  //    the Socrata federated catalog (api.us.socrata.com) filtered to procurement/
+  //    bid/contract datasets, then each host-scoped $select=count(*) + /resource
+  //    200 bare-array live-verified. US local govs only (Canada/AU + demo/test +
+  //    off-theme aggregate hosts filtered out). Mix of official .gov/.org/.com
+  //    municipal portals (same documented non-.gov trust-boundary as above). ──
+  "data.kcmo.org", // Kansas City MO (.org, official) — e.g. 4mdg-usvj (Vendor Payments ~144k; 22 procurement datasets)
+  "data.brla.gov", // Baton Rouge / East Baton Rouge Parish LA (.gov) — e.g. e5pk-us93 (Upcoming Procurement Opportunities; 19 procurement datasets)
+  "www.dallasopendata.com", // Dallas TX (.com, official) — e.g. x5ih-idh7 (Vendor Payments FY2019–present ~166k)
+  "data.lacity.org", // Los Angeles CA (.org, official) — e.g. hf3r-utnq (RAMP Open Bid Opportunities — live bids)
+  "data.ramseycountymn.gov", // Ramsey County MN (.gov) — e.g. iu7r-dzmj (Solicitations & Addenda, with due_date/download_url ~516)
+  "data.richmondgov.com", // Richmond VA (.com, official) — e.g. xqn7-jvv2 (City Contracts: contract_value/supplier/procurement_type ~1,387)
 ] as const;
 
 export type SocrataDomain = (typeof SOCRATA_DOMAINS)[number];
@@ -262,6 +316,44 @@ async function getCatalog(params: URLSearchParams): Promise<unknown> {
   });
 }
 
+/**
+ * GET a HOST-SCOPED discovery catalog (`https://{domain}/api/catalog/v1?
+ * search_context={domain}&…`). Unlike the FEDERATED `getCatalog`
+ * (api.us.socrata.com), each portal's OWN catalog COMPLETELY indexes its own
+ * datasets — the federated aggregator under-indexes many hosts (loop cycle 8:
+ * USAC → 0, DOT → 3 of 1,873). Used whenever a specific `domain` is requested;
+ * the all-host search (domain omitted) still needs the federated aggregator.
+ * SSRF: identical guard to getSocrataResource — domain ∈ allowlist + the
+ * CONSTRUCTED URL's hostname === domain (https); redirect:"error" (B1); the
+ * host-only label carries the domain (never the token, m7).
+ */
+async function getHostCatalog(
+  domain: string,
+  params: URLSearchParams,
+): Promise<unknown> {
+  if (!SOCRATA_DOMAIN_SET.has(domain)) {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Socrata domain ${JSON.stringify(domain)} is not on the curated allowlist. Allowed: ${SOCRATA_DOMAINS.join(", ")}.`,
+      retryable: false,
+    });
+  }
+  const url = `https://${domain}/api/catalog/v1?${params.toString()}`;
+  const built = new URL(url);
+  if (built.hostname !== domain || built.protocol !== "https:") {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Constructed Socrata host-catalog URL host ${JSON.stringify(built.hostname)} (${built.protocol}) does not match the allowlisted domain ${JSON.stringify(domain)} over https — refusing to fetch (SSRF safety).`,
+      retryable: false,
+    });
+  }
+  return getJson(url, {
+    label: "socrata:catalog:" + domain,
+    headers: appTokenHeader(),
+    redirect: "error",
+  });
+}
+
 // ─── map + meta helpers ───────────────────────────────────────────
 const STRING_COERCION_NOTE =
   "Row value fields arrive as strings verbatim from SODA (e.g. \"13650.00\", \"1\") — parse client-side. A missing value is absent, never 0.";
@@ -464,16 +556,21 @@ function mapCatalogRow(row: unknown): CatalogDataset {
 }
 
 /**
- * Discover dataset 4x4 ids via the Socrata catalog (memoized ~10 min). Omitting
- * `domain` searches the WHOLE allowlist (repeated `domains=`); passing one scopes
- * to it. Returns `[{ id, name, description, domain, updatedAt, link }]` +
- * `totalAvailable = resultSetSize`. Feeds `datasetId` to socrata_query.
+ * Discover dataset 4x4 ids via the Socrata catalog (memoized ~10 min). Passing a
+ * `domain` queries that portal's OWN catalog (search_context) — a COMPLETE
+ * per-host index; omitting it searches the WHOLE allowlist via the federated
+ * api.us.socrata.com aggregator (repeated `domains=`). Returns `[{ id, name,
+ * description, domain, updatedAt, link }]` + `totalAvailable = resultSetSize`.
+ * Feeds `datasetId` to socrata_query.
  *
  * m3 — this is the catalog's PRIMARY response: a non-number `resultSetSize` is a
  * hard schema_drift throw (nothing valid to return), never a fabricated total.
- * (Note: the catalog does not index every allowlisted host — e.g. USAC — so a
- * host may return 0 here yet still be queryable via socrata_query with a known
- * 4x4.)
+ *
+ * UNDER-INDEX (loop cycle 8 fix): the FEDERATED aggregator under-indexes many
+ * hosts (USAC → 0; DOT → 3 of 1,873), so the all-host search (domain omitted) can
+ * miss datasets — its `_meta` note discloses this. A DOMAIN-scoped search now
+ * uses that host's OWN catalog, which indexes it completely (USAC/DOT/etc. fully
+ * discoverable). socrata_query works regardless with a known 4x4.
  */
 export async function discoverDatasets(args: {
   q: string;
@@ -485,8 +582,11 @@ export async function discoverDatasets(args: {
   params.set("q", args.q);
   params.set("only", "datasets");
   params.set("limit", String(limit));
+  // A specific domain → that portal's OWN catalog (search_context), a COMPLETE
+  // per-host index. Omitting domain → the federated aggregator (repeated
+  // domains=), the only way to search across all hosts (loop cycle 8).
   if (args.domain) {
-    params.append("domains", args.domain);
+    params.set("search_context", args.domain);
   } else {
     for (const d of SOCRATA_DOMAINS) params.append("domains", d);
   }
@@ -495,7 +595,9 @@ export async function discoverDatasets(args: {
   const { totalAvailable, results } = await memoize(
     key,
     async () => {
-      const body = await getCatalog(params);
+      const body = args.domain
+        ? await getHostCatalog(args.domain, params)
+        : await getCatalog(params);
       const b = (body ?? {}) as { results?: unknown; resultSetSize?: unknown };
       // m3 — hard drift on the PRIMARY response (contrast the best-effort count).
       // The check stays INSIDE the memoize callback so a bad shape is never
@@ -517,6 +619,9 @@ export async function discoverDatasets(args: {
   const scope = args.domain ? `domain ${args.domain}` : `the ${SOCRATA_DOMAINS.length}-host allowlist`;
   const notes: string[] = [
     `Catalog search over ${scope} (only=datasets). Feed a result's id to socrata_query as datasetId.`,
+    args.domain
+      ? `Source: the ${args.domain} portal's OWN catalog (search_context) — a COMPLETE per-host index.`
+      : `Source: the federated ${CATALOG_HOST} aggregator, which UNDER-INDEXES some hosts (e.g. USAC, DOT) — pass a specific domain to use that host's own complete catalog.`,
     `App token: ${appTokenPresent() ? "present (X-App-Token sent; value never logged)" : "absent (keyless)"}.`,
   ];
   if (totalAvailable !== null && returned < totalAvailable) {
@@ -528,7 +633,7 @@ export async function discoverDatasets(args: {
   return withMeta(
     { query: args.q, domain: args.domain ?? null, results },
     {
-      source: `${CATALOG_HOST} catalog ${SOURCE_SUFFIX}`,
+      source: `${args.domain ?? CATALOG_HOST} catalog ${SOURCE_SUFFIX}`,
       keylessMode: true,
       returned,
       totalAvailable,