@cliwant/mcp-sam-gov 1.14.0 → 1.16.0

package/src/socrata.ts

@@ -179,7 +179,8 @@ export const SOCRATA_DOMAINS = [
   //    allowlist, not the TLD; these five are the documented non-.gov entries. ──
   "data.cityofnewyork.us", // NYC OpenData (.us, official) — e.g. dg92-zbpx (City Record: procurement notices)
   "data.cityofchicago.org", // Chicago (.org, official) — e.g. rsxa-ify5 (Contracts)
-  "data.sfgov.org", // San Francisco / DataSF (.org, official) — e.g. cqi5-hm2d (Supplier Contracts)
+  "data.sfgov.org", // San Francisco / DataSF — PERMANENTLY MIGRATED to data.sf.gov (301 all paths, 2026-09); kept allowlisted so requests reach the migration handler (see MIGRATED_SOCRATA_HOSTS below) instead of returning invalid_input. Use data.sf.gov for all new calls.
+  "data.sf.gov", // San Francisco / DataSF (canonical post-migration .gov domain) — e.g. cqi5-hm2d (Supplier Contracts 49,186 rows), hmh3-ff63 (Airport/SFO Contract Opportunities)
   "controllerdata.lacity.org", // LA City Controller (.org, official) — e.g. pggv-e4fn (Checkbook L.A.)
   "opendata.usac.org", // USAC E-rate (.org, m6) — e.g. avi8-svp9
   // ── County/city procurement sweep (loop cycle 17, 2026-07-20). Discovered via
@@ -203,6 +204,7 @@ export const SOCRATA_DOMAINS = [
   "data.coloradosprings.gov", // Colorado Springs CO (.gov) — e.g. yn6y-xikx (Open Checkbook Vendors ~19k)
   "data.framinghamma.gov", // Framingham MA (.gov) — e.g. cqve-ehkr (Checkbook ~324k)
   "data.fultoncountyga.gov", // Fulton County GA (.gov) — e.g. mxhc-krcg (Vendor Payments/disbursements ~217k)
+  "sharefulton.fultoncountyga.gov", // Fulton County GA (.gov, second portal) — e.g. kp4p-scak (Vendor Payments 226,797 rows 2014–present, updated 2026-09-14, attribution: "Fulton County Government (GA)")
   "atlanta.data.socrata.com", // City of Atlanta GA (Socrata-hosted official portal) — e.g. jmke-icfi (Open Checkbook Ledger ~1.78M)
   "opendata.cityofmesquite.com", // Mesquite TX (.com, official) — e.g. 6tva-azs5 (Check Register ~144k)
   // ── County/city procurement sweep, wave 3 (loop cycle 20, 2026-07-20). Expanded
@@ -230,6 +232,24 @@ export type SocrataDomain = (typeof SOCRATA_DOMAINS)[number];
 
 const SOCRATA_DOMAIN_SET: ReadonlySet<string> = new Set(SOCRATA_DOMAINS);
 
+/**
+ * Known host migrations (permanent 301 redirects). A domain here has permanently
+ * moved; redirect:"error" means any fetch to the old host will fail immediately.
+ * Rather than returning a retryable upstream_unavailable (which implies the
+ * problem may resolve on its own), we pre-flight check this map and return
+ * invalid_input — non-retryable, naming the replacement — so the caller can
+ * update the `domain` parameter instead of spinning on a doomed retry loop.
+ *
+ * Keep the old host in SOCRATA_DOMAINS so it passes the allowlist gate (invalid
+ * domains are rejected before reaching this check). Only add an entry here when
+ * the migration is confirmed by a live 301 probe with a stable redirect_url.
+ *
+ *   data.sfgov.org → data.sf.gov   (confirmed 2026-09-21; all paths 301)
+ */
+export const MIGRATED_SOCRATA_HOSTS: ReadonlyMap<string, string> = new Map([
+  ["data.sfgov.org", "data.sf.gov"],
+]);
+
 const CATALOG_HOST = "api.us.socrata.com";
 const CATALOG_URL = `https://${CATALOG_HOST}/api/catalog/v1`;
 
@@ -297,6 +317,18 @@ async function getSocrataResource(
       retryable: false,
     });
   }
+  // Migration pre-flight (MIGRATED_SOCRATA_HOSTS): if the domain has permanently
+  // moved, redirect:"error" means any fetch will immediately fail with a 301.
+  // Return invalid_input (non-retryable) now — before wasting a network round-trip
+  // — naming the replacement so the caller can update the domain parameter.
+  const migratedTo = MIGRATED_SOCRATA_HOSTS.get(domain);
+  if (migratedTo !== undefined) {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Socrata host ${JSON.stringify(domain)} has permanently moved to ${JSON.stringify(migratedTo)}. Update the \`domain\` parameter to ${JSON.stringify(migratedTo)} — retrying ${JSON.stringify(domain)} will not succeed.`,
+      retryable: false,
+    });
+  }
   if (datasetId.length !== 9 || !DATASET_ID_RE.test(datasetId)) {
     throw new ToolErrorCarrier({
       kind: "invalid_input",
@@ -375,6 +407,15 @@ async function getHostCatalog(
       retryable: false,
     });
   }
+  // Migration pre-flight — same policy as getSocrataResource (see comment there).
+  const migratedTo = MIGRATED_SOCRATA_HOSTS.get(domain);
+  if (migratedTo !== undefined) {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Socrata host ${JSON.stringify(domain)} has permanently moved to ${JSON.stringify(migratedTo)}. Update the \`domain\` parameter to ${JSON.stringify(migratedTo)} — retrying ${JSON.stringify(domain)} will not succeed.`,
+      retryable: false,
+    });
+  }
   const url = `https://${domain}/api/catalog/v1?${params.toString()}`;
   const built = new URL(url);
   if (built.hostname !== domain || built.protocol !== "https:") {
@@ -419,6 +460,12 @@ async function fetchCount(
   const params = new URLSearchParams();
   if (where) params.set("$where", where);
   if (q) params.set("$q", q);
+  // ★ MUST stay count(*) — DO NOT change to count(1) or SELECT count(1).
+  // Cloudflare-fronted Socrata hosts (e.g. opendata.maryland.gov) deterministically
+  // 403 ("Just a moment...") on count(1) and `$query=SELECT count(1)` — their WAF
+  // treats the pattern as SQLi-like — while count(*) consistently returns 200.
+  // Verified 3/3 on opendata.maryland.gov (2026-09-21): count(1)→403, count(*)→200
+  // (342 rows). Maryland is not broken; the shape here is load-bearing.
   params.set("$select", "count(*)");
   let body: unknown;
   try {
@@ -547,6 +594,22 @@ export async function query(args: {
     );
   }
 
+  // ── Response-side aggregate nudge (§A-nudge) ──
+  // When the result is truncated (hasMore) and the caller used no aggregate select,
+  // the agent only sees a PAGE of rows — NOT the total. Nudge toward an aggregate
+  // query so the agent does not try to sum one page manually.
+  if (!isAggregateSelect && hasMore) {
+    // Find the most likely amount-like column from the first row's keys.
+    const firstRow = rows[0];
+    const amountKey =
+      firstRow !== undefined
+        ? (Object.keys(firstRow).find((k) => /amount|amt|total|paid|payment/i.test(k)) ?? "<amount column>")
+        : "<amount column>";
+    notes.push(
+      `This is a PAGE of raw rows (truncated — more rows exist). Do NOT sum this page to get a total. For a grand total: re-query with select='sum(${amountKey})' plus a where filter for vendor/fiscal year. For a top-N ranking: select='vendor_name, sum(${amountKey}) as total' with order='total DESC'. These are aggregate queries — the result is the final answer, not another page.`,
+    );
+  }
+
   return withMeta(
     { domain: args.domain, datasetId: args.datasetId, rows },
     {
@@ -666,6 +729,25 @@ export async function discoverDatasets(args: {
       `Showing ${returned} of ${totalAvailable} matches; raise limit (≤100) or narrow q for the rest.`,
     );
   }
+  // HONESTY (false-zero guard). An empty catalog result is easy to read as "this
+  // portal has no such data", and the per-host note above even calls the index
+  // COMPLETE — but the catalog behaves as if EVERY q term must match, and datasets
+  // rarely repeat their own jurisdiction's name. Measured 2026-09-21 on
+  // data.illinois.gov: q="Illinois state solicitations" -> 0 and
+  // q="Illinois procurement" -> 0, while q="solicitations" -> 1 (6rb8-ntpm). An eval
+  // agent took that zero at face value and told the user no Illinois procurement
+  // datasets exist. So a zero is reported as possibly false, with the fix.
+  if (returned === 0) {
+    // Only "more than one word?" matters here, so test for it directly rather than
+    // splitting: a raw whitespace split is reserved for disclosure tokenizing
+    // (tokenizeForDisclosure, ADR-0022) and this is not a disclosure.
+    const multiWord = /\S\s+\S/.test(args.q);
+    notes.push(
+      multiWord
+        ? `0 matches for a multi-word q — likely a FALSE zero, not proof the data is absent: the catalog behaves as if every term must match, and datasets rarely repeat their jurisdiction's name in their title. Put the place in \`domain\` (not in q) and retry with ONE topical term, e.g. 'solicitations', 'bids', 'contract', 'vendor', 'payments', 'purchase'.`
+        : `0 matches for q=${JSON.stringify(args.q)} — this is not proof the data is absent: publishers title the same thing differently. Retry with a synonym ('solicitations', 'bids', 'contract', 'vendor', 'payments', 'purchase') before concluding the portal has none.`,
+    );
+  }
 
   return withMeta(
     { query: args.q, domain: args.domain ?? null, results },