@cliwant/mcp-sam-gov 1.10.0 → 1.11.0

package/src/nhtsa.ts

@@ -19,16 +19,20 @@
  *   entirely — never surfaced, logged, or stored. The B2G value is the
  *   manufacturer / component / safety signal, NOT the VIN.
  *
- * ★ THE HONESTY PILLARS (P1-P4, live-verified 2026-07-15):
+ * ★ THE HONESTY PILLARS (P1/P3/P4 live-verified 2026-07-15; P2 corrected +
+ *   re-verified 2026-07-20):
  *   P1: totalAvailable = `Count` (recalls) / `count` (complaints) — the REAL total.
  *       NHTSA returns the COMPLETE filtered set (no pagination), so in the normal
  *       case Count === results.length ⇒ complete:true. totalAvailable is NEVER
  *       fabricated: a PRESENT numeric Count is trusted verbatim; a MISSING Count
  *       falls back to results.length WITH an honest note (never invented).
- *   P2: results:[] (Count 0) ⇒ an HONEST EMPTY (returned:0, complete:true) — a bad
- *       make/model that returns 200+Count 0 is an honest no-match, NOT an error. A
- *       4xx ⇒ invalid_input; a 5xx/timeout ⇒ THROW (never a fake empty); a 200
- *       non-JSON body ⇒ schema_drift.
+ *   P2: ★NHTSA returns HTTP 400 (NOT 200) + {Count/count:0, results:[]} (Message
+ *       "Results returned successfully") for a VALID make/model/year that simply
+ *       has ZERO records (live-verified 2026-07-20). getNhtsa reads the body and
+ *       reclassifies THAT idiom as an HONEST EMPTY (returned:0, totalAvailable:0,
+ *       complete:true) with a note; any OTHER 400 ⇒ invalid_input, a 404 ⇒
+ *       not_found, a 5xx/timeout ⇒ THROW (never a fake empty), a 200 non-JSON
+ *       body ⇒ schema_drift.
  *   P3: booleans (crash/fire/parkIt/parkOutSide/overTheAirUpdate) preserved AS
  *       booleans (a non-boolean ⇒ null, never a fabricated false); counts
  *       (numberOfInjuries/numberOfDeaths) via `num` (a genuine 0 stays 0, NEVER
@@ -42,8 +46,8 @@
  *       so a `../` or `%` can never reach the fixed path).
  */
 
-import { ToolErrorCarrier } from "./errors.js";
-import { getJson, driftError } from "./datasource.js";
+import { ToolErrorCarrier, errorFromResponse } from "./errors.js";
+import { driftError, isRedirectError } from "./datasource.js";
 import { num, str } from "./coerce.js";
 import { withMeta, type MetaBundle, type ResponseMeta } from "./meta.js";
 
@@ -109,10 +113,31 @@ function validateVehicleArgs(args: NhtsaVehicleArgs, label: string): void {
 
 // ─── SSRF-guarded fetch (fixed host + hostname assertion + redirect) ──
 /**
- * GET one NHTSA JSON resource on the FIXED host. Builds
- * `https://api.nhtsa.gov${path}?${params}`, asserts the CONSTRUCTED URL's
- * hostname === the fixed host over https (belt-and-suspenders), and sets
- * `redirect:"error"` (fail closed on any off-host 3xx). Keyless — no header/token.
+ * ★ NHTSA's "400-for-empty" idiom. A VALID make/model/year that simply has ZERO
+ * records returns HTTP 400 with a body `{Count/count:0, ..., results:[]}` and
+ * `Message/message:"Results returned successfully"` (live-verified 2026-07-20:
+ * `make=tesla&model=model 3&modelYear=2015` → HTTP 400, Count 0). Recalls use
+ * `Count`/`Message`, complaints use lowercase `count`/`message` — accept either.
+ * This is a genuine no-match, NOT a bad request; the caller reclassifies it to an
+ * honest empty. (A 400 that is NOT this idiom stays a real invalid_input.)
+ */
+function isNhtsaEmptyIdiom(body: unknown): boolean {
+  if (body === null || typeof body !== "object") return false;
+  const b = body as Record<string, unknown>;
+  const rawCount = b.Count ?? b.count;
+  return num(rawCount) === 0 && Array.isArray(b.results) && b.results.length === 0;
+}
+
+/**
+ * GET one NHTSA JSON resource on the FIXED host. Bespoke `fetch` (NOT the shared
+ * getJson) because we must READ a non-2xx body: NHTSA returns HTTP 400 for a valid
+ * query with zero records (the ★400-for-empty idiom above), and getJson would
+ * throw invalid_input and DISCARD the body — turning "zero recalls" (a valid,
+ * high-value answer) into a phantom "Bad request". Asserts the constructed URL's
+ * hostname === the fixed host over https (belt-and-suspenders) + `redirect:"error"`
+ * (fail closed on any off-host 3xx). Keyless — no header/token. Non-2xx that is NOT
+ * the empty idiom keeps the standard taxonomy (400→invalid_input, 404→not_found,
+ * 429→rate_limited, 5xx→upstream_unavailable); a 200 non-JSON body ⇒ schema_drift.
  */
 async function getNhtsa(
   path: string,
@@ -129,7 +154,57 @@ async function getNhtsa(
       upstreamEndpoint: label,
     });
   }
-  return getJson(url, { label, redirect: "error" });
+  let res: Response;
+  try {
+    res = await fetch(built.toString(), {
+      redirect: "error",
+      signal: AbortSignal.timeout(15_000),
+    });
+  } catch (e) {
+    if (isRedirectError(e)) {
+      throw driftError(
+        label,
+        `NHTSA returned an off-host redirect (redirect:"error") while fetching ${label} — refusing to follow it (SSRF safety).`,
+      );
+    }
+    if (e instanceof Error && (e.name === "TimeoutError" || e.name === "AbortError")) {
+      throw new ToolErrorCarrier({
+        kind: "upstream_unavailable",
+        retryable: false,
+        message: `Request to ${label} timed out.`,
+        upstreamEndpoint: label,
+      });
+    }
+    throw new ToolErrorCarrier({
+      kind: "upstream_unavailable",
+      retryable: true,
+      retryAfterSeconds: 30,
+      message: `Network error reaching ${label}: ${e instanceof Error ? e.message : String(e)}`,
+      upstreamEndpoint: label,
+    });
+  }
+  if (!res.ok) {
+    // ★P2 CRUX: read the body — a 400 that is NHTSA's empty idiom is an HONEST
+    // no-match (returned via resolveTotal as Count 0 → total 0), NOT an error.
+    let body: unknown = null;
+    try {
+      body = await res.json();
+    } catch {
+      body = null;
+    }
+    if (res.status === 400 && isNhtsaEmptyIdiom(body)) return body;
+    // Any other non-2xx keeps the standard taxonomy (never a fabricated empty).
+    // errorFromResponse returns a plain ToolError → wrap it in the carrier to throw.
+    throw new ToolErrorCarrier(errorFromResponse(res, label));
+  }
+  try {
+    return await res.json();
+  } catch {
+    throw driftError(
+      label,
+      `NHTSA ${label} returned a non-JSON body at HTTP 200 — schema drift (never read as an empty result).`,
+    );
+  }
 }
 
 /** Build the shared make/model/modelYear query (module-built; no raw passthrough). */
@@ -142,27 +217,17 @@ function vehicleParams(args: NhtsaVehicleArgs): URLSearchParams {
 }
 
 /**
- * Fetch + parse a NHTSA resource, mirroring datagov-catalog's catch-ladder: a
- * ToolErrorCarrier (host-assert / 4xx-5xx taxonomy) rethrows FIRST (preserving its
- * kind); a 200 non-JSON `.json()` SyntaxError reclassifies to schema_drift; a bare
- * error rethrows LAST.
+ * Build the vehicle query and fetch through getNhtsa, which fully classifies the
+ * response: the ★400-for-empty idiom → an honest empty; every other non-2xx → the
+ * standard taxonomy; a 200 non-JSON body → schema_drift. A thin wrapper — getNhtsa
+ * owns the error contract.
  */
 async function fetchNhtsa(
   path: string,
   label: string,
   args: NhtsaVehicleArgs,
 ): Promise<unknown> {
-  try {
-    return await getNhtsa(path, label, vehicleParams(args));
-  } catch (e) {
-    if (e instanceof ToolErrorCarrier) throw e;
-    if (e instanceof SyntaxError)
-      throw driftError(
-        label,
-        `NHTSA ${label} returned a non-JSON body at HTTP 200 — schema drift (never read as an empty result).`,
-      );
-    throw e;
-  }
+  return await getNhtsa(path, label, vehicleParams(args));
 }
 
 /**

package/src/nist-controls.ts

@@ -46,8 +46,10 @@ export type NistControl = {
   id: string; // display form, e.g. "AC-2"
   family: string; // e.g. "AC — Access Control"
   title: string;
-  statement: string; // assembled requirement prose (labelled, multi-line)
+  status: string | null; // OSCAL status prop, e.g. "withdrawn"; null when active
+  statement: string | null; // requirement prose; NULL when absent (a WITHDRAWN control has none) — never ""
   guidance: string | null; // discussion prose
+  incorporatedInto: string[]; // withdrawn control ⇒ the control id(s) it was folded into (e.g. ["AC-2","AU-6"])
   enhancements: { id: string; title: string }[]; // e.g. AC-2(1)
 };
 
@@ -60,6 +62,8 @@ type OscalPart = {
 type OscalControl = {
   id?: string;
   title?: string;
+  props?: { name?: string; value?: string }[];
+  links?: { href?: string; rel?: string }[];
   parts?: OscalPart[];
   controls?: OscalControl[];
 };
@@ -100,12 +104,26 @@ function partProse(control: OscalControl, name: string): string | null {
 }
 
 function mapControl(control: OscalControl, family: string): NistControl {
+  const status =
+    (control.props ?? []).find((p) => p.name === "status")?.value?.trim() || null;
+  // A withdrawn control carries `links[] rel="incorporated-into"` → the control(s)
+  // it was folded into (e.g. AC-13 → AC-2, AU-6). Surface them so a withdrawn
+  // control is never mistaken for an active requirement.
+  const incorporatedInto = (control.links ?? [])
+    .filter((l) => l.rel === "incorporated-into" && typeof l.href === "string")
+    .map((l) => displayId(String(l.href).replace(/^#/, "")))
+    .filter((x) => x.length > 0);
   return {
     id: displayId(control.id ?? ""),
     family,
     title: (control.title ?? "").trim(),
-    statement: partProse(control, "statement") ?? "",
+    status,
+    // [P3] absent statement ⇒ null, NEVER "" — a withdrawn control has no
+    // requirement text; "" would misread as "an active control with a blank
+    // requirement".
+    statement: partProse(control, "statement"),
     guidance: partProse(control, "guidance"),
+    incorporatedInto,
     enhancements: (control.controls ?? []).map((e) => ({
       id: displayId(e.id ?? ""),
       title: (e.title ?? "").trim(),
@@ -119,7 +137,12 @@ function mapControl(control: OscalControl, family: string): NistControl {
  * families (else driftError — a truncated catalog is NEVER a fake empty). Enhancements
  * are carried on each control (not indexed as top-level entries).
  */
-async function loadControls(): Promise<NistControl[]> {
+type OscalMetadata = { version: string | null; lastModified: string | null };
+
+async function loadControls(): Promise<{
+  controls: NistControl[];
+  metadata: OscalMetadata;
+}> {
   return memoize(
     "nist:sp800-53r5",
     async () => {
@@ -131,7 +154,12 @@ async function loadControls(): Promise<NistControl[]> {
         label: OSCAL_LABEL,
         redirect: "error",
         timeoutMs: OSCAL_TIMEOUT_MS,
-      })) as { catalog?: { groups?: unknown } };
+      })) as {
+        catalog?: {
+          groups?: unknown;
+          metadata?: { version?: unknown; "last-modified"?: unknown };
+        };
+      };
       const groups = body.catalog?.groups;
       if (!Array.isArray(groups) || groups.length < FAMILY_FLOOR) {
         throw driftError(
@@ -139,12 +167,21 @@ async function loadControls(): Promise<NistControl[]> {
           `OSCAL catalog.groups missing or implausibly small (${Array.isArray(groups) ? groups.length : "not-an-array"} < ${FAMILY_FLOOR} families) — treating as schema drift / truncation, never a fake-empty catalog.`,
         );
       }
+      // [P5 freshness] The OSCAL catalog is a moving static file on the `main`
+      // branch — surface its exact version + last-modified so a compliance caller
+      // knows WHICH point-release (5.1.1 / 5.2.0 …) they are reading.
+      const md = body.catalog?.metadata ?? {};
+      const metadata: OscalMetadata = {
+        version: typeof md.version === "string" ? md.version : null,
+        lastModified:
+          typeof md["last-modified"] === "string" ? md["last-modified"] : null,
+      };
       const controls: NistControl[] = [];
       for (const g of groups as OscalGroup[]) {
         const family = `${(g.id ?? "").toUpperCase()} — ${(g.title ?? "").trim()}`;
         for (const c of g.controls ?? []) controls.push(mapControl(c, family));
       }
-      return controls;
+      return { controls, metadata };
     },
     OSCAL_CACHE_TTL_MS,
   );
@@ -166,7 +203,7 @@ export async function searchControls(args: {
 }): Promise<MetaBundle> {
   const limit = args.limit ?? 25;
   const offset = args.offset ?? 0;
-  const all = await loadControls();
+  const { controls: all, metadata } = await loadControls();
 
   const filtersApplied: string[] = [];
   const idQ = args.controlId !== undefined ? displayId(args.controlId) : undefined;
@@ -189,7 +226,7 @@ export async function searchControls(args: {
       // Search the title + requirement statement AND each enhancement's title, so a
       // term that lives only in an enhancement (e.g. "multi-factor" → IA-2(1)) still
       // surfaces the parent control. filtersApplied still lists 'keyword'.
-      const hay = `${c.title}\n${c.statement}\n${c.enhancements.map((e) => e.title).join("\n")}`.toLowerCase();
+      const hay = `${c.title}\n${c.statement ?? ""}\n${c.enhancements.map((e) => e.title).join("\n")}`.toLowerCase();
       if (!hay.includes(kwQ)) return false;
     }
     return true;
@@ -201,10 +238,22 @@ export async function searchControls(args: {
   const hasMore = offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
+  const versionLabel = metadata.version ? `Rev ${metadata.version}` : "Rev 5";
+  const notes: string[] = [PROVENANCE_NOTE, CLIENT_FILTER_NOTE, REFERENCE_NOTE];
+  notes.push(
+    `OSCAL catalog ${metadata.version ? `version ${metadata.version}` : "revision 5"}${metadata.lastModified ? `, last-modified ${metadata.lastModified.slice(0, 10)}` : ""} — fetched live from the usnistgov/oscal-content \`main\` branch (a MOVING target; control text can change between point releases, e.g. 5.1.1 → 5.2.0). Cite the version above, not just "Rev 5".`,
+  );
+  const withdrawnOnPage = page.filter((c) => c.status === "withdrawn").length;
+  if (withdrawnOnPage > 0) {
+    notes.push(
+      `${withdrawnOnPage} of the returned control(s) are WITHDRAWN (status:"withdrawn") — a withdrawn control has NO requirement text (statement:null) and is NOT an active requirement; see its incorporatedInto for the control(s) that superseded it.`,
+    );
+  }
+
   return withMeta(
     { controls: page },
     {
-      source: "NIST SP 800-53 Rev 5 (OSCAL catalog, keyless)",
+      source: `NIST SP 800-53 ${versionLabel} (OSCAL catalog, keyless)`,
       keylessMode: true,
       returned,
       totalAvailable,
@@ -213,7 +262,7 @@ export async function searchControls(args: {
       filtersDropped: [],
       fieldsUnavailable: [],
       pagination: { offset, limit, hasMore, nextOffset },
-      notes: [PROVENANCE_NOTE, CLIENT_FILTER_NOTE, REFERENCE_NOTE],
+      notes,
     } satisfies Partial<ResponseMeta>,
   );
 }

package/src/ofac.ts

@@ -206,7 +206,18 @@ export function classifyMatch(
   const allQinE = qt.length > 0 && qt.every((t) => eSet.has(t));
   const allEinQ = et.length > 0 && et.every((t) => qSet.has(t));
   if (allQinE || allEinQ) return "strong";
-  if (entryNorm.includes(queryNorm) || queryNorm.includes(entryNorm)) {
+  // Substring containment is "strong" ONLY when the CONTAINED (shorter) string is
+  // a substantial fragment. Without a floor, a tiny sub-word substring graded
+  // "strong" ("TS" ⊂ "WIDGETS", "IBB" ⊂ "QUIBBLEFARB", "D" ⊂ "WIDGETS") —
+  // overstating confidence and flooding nearly every input with false "strong"
+  // matches (alert fatigue). Whole-token containment is already handled above by
+  // allQinE/allEinQ; this mid-word tier is for the space-stripped joined form and
+  // must clear the same ≥3 bar the token "weak" tier uses — here a slightly higher
+  // floor since a contiguous substring is a weaker signal than a shared whole token.
+  if (
+    Math.min(queryNorm.length, entryNorm.length) >= 5 &&
+    (entryNorm.includes(queryNorm) || queryNorm.includes(entryNorm))
+  ) {
     return "strong";
   }
   for (const t of qt) {

package/src/sam-gov/client.ts

@@ -353,11 +353,20 @@ export class SamGovClient {
     filters: SamSearchFilters,
   ): Promise<SamSearchResult> {
     const url = new URL(`${PUBLIC_BASE}/sgs/v1/search/`);
+    // Keyless HAL pagination is by PAGE (page × size), not a row offset — the
+    // list endpoint pages correctly (VERIFIED LIVE 2026-07: page=0 and page=1
+    // return disjoint result sets for the same query). Map the caller's row
+    // `offset` onto the page grid; a non-page-aligned offset snaps DOWN to its
+    // page boundary and we return the SERVED offset (page × size) so `_meta`
+    // never claims an offset the upstream didn't honor. (The authenticated path,
+    // buildAuthSearchUrl, honors an arbitrary `offset` directly.)
+    const size = filters.limit ?? 25;
+    const page = Math.max(0, Math.floor((filters.offset ?? 0) / size));
     url.searchParams.set("index", "opp");
-    url.searchParams.set("page", "0");
+    url.searchParams.set("page", String(page));
     url.searchParams.set("mode", "search");
     url.searchParams.set("sort", "-modifiedDate");
-    url.searchParams.set("size", String(filters.limit ?? 25));
+    url.searchParams.set("size", String(size));
     url.searchParams.set("is_active", "true");
     // Keyless HAL facet params — VERIFIED LIVE (2026-07). The list endpoint
     // honors these server-side: result counts drop correctly AND every returned
@@ -457,8 +466,8 @@ export class SamGovClient {
     });
     return {
       totalRecords,
-      limit: filters.limit ?? 25,
-      offset: filters.offset ?? 0,
+      limit: size,
+      offset: page * size, // the SERVED offset (page-aligned), never the raw request
       opportunitiesData: data,
     };
   }

package/src/server.ts

@@ -106,7 +106,7 @@ import { realpathSync } from "node:fs";
 const SERVER_NAME = "mcp-sam-gov";
 // Kept in lockstep with package.json / manifest.json / server.json.
 // Keep in sync with package.json "version" (asserted at release; see CHANGELOG).
-const SERVER_VERSION = "1.10.0";
+const SERVER_VERSION = "1.11.0";
 
 // ─── Tool input schemas (Zod) ────────────────────────────────────
 
@@ -4418,7 +4418,7 @@ const GsaPerdiemRatesInput = z
       .regex(/^\d{4}$/)
       .optional()
       .describe(
-        "The per-diem fiscal year (default '2025'). Validated ^\\d{4}$ (it rides in the request path).",
+        "The per-diem fiscal year (default: the current U.S. federal fiscal year, computed at call time — GSA sets rates per FY, Oct 1–Sep 30). Validated ^\\d{4}$ (it rides in the request path).",
       ),
   })
   .describe(
@@ -4543,7 +4543,7 @@ const LdaSearchFilingsInput = z.object({
     .string()
     .min(1)
     .optional()
-    .describe("Filter by the federal government entity lobbied (maps to government_entity — the B2G signal), e.g. 'DEPARTMENT OF DEFENSE'."),
+    .describe("NOTE: the keyless /filings/ endpoint has NO server-side government-entity filter — the LDA API silently ignores it, so this value is NOT applied (reported in _meta.filtersDropped, never as a narrowed total). Government entities are nested per lobbying activity (each filing's lobbyingActivities[].governmentEntities); to find who lobbied an agency, narrow by registrantName/clientName/issue and inspect those nested entities. Retained for discoverability of the limitation."),
   issue: z
     .string()
     .min(1)
@@ -4895,6 +4895,21 @@ export const TOOLS: ToolDef[] = [
             "The organization-name filter is NOT supported by the keyless endpoint and was ignored (results are unfiltered on organization). Set SAM_GOV_API_KEY to filter by organization, or filter client-side on the returned `agency` field.",
           );
         }
+        // Pagination honesty: the keyless HAL endpoint pages by PAGE (page × size),
+        // not an arbitrary row offset, so a requested offset that is not a multiple
+        // of `limit` snaps DOWN to its page boundary. Disclose the snap so the AI
+        // never believes it read rows [offset..offset+limit) when it actually got
+        // the page-aligned window. (An aligned offset — the default paging pattern
+        // of incrementing by `limit` — is served exactly, so no note.)
+        {
+          const reqOffset = input.offset ?? 0;
+          const size = input.limit ?? 25;
+          if (reqOffset % size !== 0) {
+            notes.push(
+              `Keyless SAM pagination is page-based (page × ${size}); the requested offset ${reqOffset} was snapped DOWN to the page boundary ${Math.floor(reqOffset / size) * size}. Page through by incrementing offset in multiples of limit (set SAM_GOV_API_KEY for exact row offsets).`,
+            );
+          }
+        }
         notes.push(...enrichmentNotes);
 
         // freshness is surfaced structurally in `data` (the ResponseMeta type
@@ -5654,7 +5669,7 @@ export const TOOLS: ToolDef[] = [
   defineTool({
     name: "nist_800_53_controls",
     description:
-      "Look up NIST SP 800-53 Rev 5 security & privacy CONTROLS (keyless) — the requirement backbone for FedRAMP / CMMC / RMF compliance work. Retrieve a control by `controlId` (exact, e.g. 'AC-2', 'SC-7', 'AC-2(1)'), a `family` (2-letter code 'AC'/'SC'/'IA' or a name substring 'Access Control'), and/or a `keyword` (case-insensitive substring over title + statement); `limit`/`offset` pagination. Each row: { id (e.g. 'AC-2'), family (e.g. 'AC — Access Control'), title, statement (the labelled requirement prose), guidance (discussion), enhancements:[{id,title}] (e.g. AC-2(1)) }. Complements cve_lookup + cisa_kev_lookup (the vulnerability side) with the CONTROL/requirement side. HONESTY: source is NIST's OFFICIAL OSCAL catalog published at github.com/usnistgov/oscal-content (authoritative first-party data served from GitHub, not a .gov API host — provenance disclosed in _meta); the catalog has no query API so filtering is CLIENT-SIDE and totalAvailable is the EXACT match count; this is the REQUIREMENT text only — applicability depends on the system's FIPS-199 impact baseline (Low/Moderate/High), which the catalog does not encode (disclosed); a download failure or an implausibly-truncated catalog (< 15 families) THROWS (never a fake-empty 'control not found').",
+      "Look up NIST SP 800-53 Rev 5 security & privacy CONTROLS (keyless) — the requirement backbone for FedRAMP / CMMC / RMF compliance work. Retrieve a control by `controlId` (exact, e.g. 'AC-2', 'SC-7', 'AC-2(1)'), a `family` (2-letter code 'AC'/'SC'/'IA' or a name substring 'Access Control'), and/or a `keyword` (case-insensitive substring over title + statement); `limit`/`offset` pagination. Each row: { id (e.g. 'AC-2'), family (e.g. 'AC — Access Control'), title, status ('withdrawn' | null), statement (the labelled requirement prose; NULL for a WITHDRAWN control, never ''), guidance (discussion), incorporatedInto:[control ids that superseded a withdrawn control, e.g. AC-13 → ['AC-2','AU-6']], enhancements:[{id,title}] (e.g. AC-2(1)) }. Complements cve_lookup + cisa_kev_lookup (the vulnerability side) with the CONTROL/requirement side. HONESTY: source is NIST's OFFICIAL OSCAL catalog published at github.com/usnistgov/oscal-content (authoritative first-party data served from GitHub, not a .gov API host — provenance disclosed in _meta); the exact OSCAL version + last-modified are surfaced in _meta (the catalog is fetched live from the MOVING 'main' branch, so control text can shift between point releases, e.g. 5.1.1 → 5.2.0 — cite the version, not just 'Rev 5'); a WITHDRAWN control (status:'withdrawn') has statement:null and is NOT an active requirement (see incorporatedInto for what replaced it); the catalog has no query API so filtering is CLIENT-SIDE and totalAvailable is the EXACT match count; this is the REQUIREMENT text only — applicability depends on the system's FIPS-199 impact baseline (Low/Moderate/High), which the catalog does not encode (disclosed); a download failure or an implausibly-truncated catalog (< 15 families) THROWS (never a fake-empty 'control not found').",
     inputSchema: NistControlsInput,
     handler: (input) => nistControls.searchControls(input),
   }),
@@ -6446,7 +6461,7 @@ export const TOOLS: ToolDef[] = [
   defineTool({
     name: "cbp_border_wait_times",
     description:
-      "Live CBP land-border-port wait times — current commercial-vehicle (and passenger) crossing delays at every US Canadian- and Mexican-border port (keyless; bwt.cbp.gov). The FREIGHT / LOGISTICS situational-awareness lane: per-port commercial-vehicle standard + FAST lane delay (minutes), operational status, open-lane count, and maximum lanes. Filters (optional): `border` (case-insensitive substring, 'Canadian'/'Mexican'), `portName` (substring, e.g. 'Laredo'); `limit`/`offset` pagination. Each row: { portNumber, portName, crossingName, border, portStatus (Open/Closed), asOf, commercialVehicle:{ maxLanes, standard:{operationalStatus, delayMinutes, lanesOpen, updateTime}, fast:{…} } }. HONESTY: this is REAL-TIME operational data — each lane carries its own updateTime (surfaced verbatim; freshness never implied live-to-the-second); delayMinutes/lanesOpen are number|null (a real 0 stays 0; an empty/N/A value — e.g. a closed lane — is null, NEVER a fabricated 0, because a closed lane's delay is UNKNOWN, not zero); the API returns the WHOLE port set so totalAvailable is the EXACT matched-port count; an outage/4xx/timeout THROWS and a non-array body ⇒ schema_drift (never a fake empty).",
+      "Live CBP land-border-port wait times — current commercial-vehicle (freight-truck) crossing delays at every US Canadian- and Mexican-border port (keyless; bwt.cbp.gov). The FREIGHT / LOGISTICS situational-awareness lane: per-port commercial-vehicle standard + FAST lane delay (minutes), operational status, open-lane count, and maximum lanes — passenger/pedestrian lanes are NOT surfaced (freight lane only). Filters (optional, applied CLIENT-SIDE over the full fetched port set — the feed has NO server-side filter; an empty-string value is reported in _meta.filtersDropped, not applied): `border` (case-insensitive substring, 'Canadian'/'Mexican'), `portName` (substring, e.g. 'Laredo'); `limit`/`offset` pagination. Each row: { portNumber, portName, crossingName, border, portStatus (Open/Closed), asOf, commercialVehicle:{ maxLanes, standard:{operationalStatus, delayMinutes, lanesOpen, updateTime}, fast:{…} } }. HONESTY: this is REAL-TIME operational data — each lane carries its own updateTime (surfaced verbatim; freshness never implied live-to-the-second); delayMinutes/lanesOpen are number|null (a real 0 stays 0; an empty/N/A value — e.g. a closed lane — is null, NEVER a fabricated 0, because a closed lane's delay is UNKNOWN, not zero); the API returns the WHOLE port set so totalAvailable is the EXACT matched-port count; an outage/4xx/timeout THROWS and a non-array body ⇒ schema_drift (never a fake empty).",
     inputSchema: CbpBorderWaitInput,
     handler: (input) => cbpBorder.borderWaitTimes(input),
   }),
@@ -6475,7 +6490,7 @@ export const TOOLS: ToolDef[] = [
   defineTool({
     name: "gsa_perdiem_rates",
     description:
-      "Look up GSA Federal Travel PER-DIEM rates — the max lodging + Meals & Incidental Expenses (M&IE) reimbursement ceilings for official U.S. government travel (api.gsa.gov /travel/perdiem/v2, keyed — DATA_GOV_API_KEY or the shared DEMO_KEY). Input: EITHER `city` (e.g. 'Washington') + `state` (2-letter, e.g. 'DC') OR `zip` (5-digit) — supplying BOTH, or NEITHER, ⇒ invalid_input with 0 fetch; optional `year` (default '2025'). Returns { rates:[{ city, county, state, zip, year, isOconus, standardRate, mealsUsd, monthlyLodgingUsd:[{ month (1-12), monthName, lodgingUsd }] }] } + honest _meta. HONESTY: lodgingUsd (the API's monthly `value`) is the MAX nightly lodging ceiling for that month — it VARIES SEASONALLY (hence a per-month array), and mealsUsd is the daily M&IE ceiling; both are integer US dollars, null-when-withheld (NEVER 0 — a genuine 0 is preserved). standardRate/isOconus are booleans coerced from the API's string 'true'/'false' (an unrecognized value ⇒ null, never a fabricated false); the months array is preserved AS-IS (never padded to 12). The API returns the COMPLETE rate set (no pagination) ⇒ totalAvailable = the row count, complete:true. A genuine no-match (rates:[]/rate:[]) ⇒ honest empty (returned:0); the API's `errors` field non-null ⇒ invalid_input carrying the message (never a fake empty); a 429 (DEMO_KEY ~10 req/hr, hit quickly) ⇒ rate_limited THROWS; a 5xx/timeout ⇒ upstream_unavailable THROWS; a 200 non-JSON ⇒ schema_drift. DEMO_KEY ~10 req/hr shared ceiling — set DATA_GOV_API_KEY (free at api.data.gov/signup) for 1000/hr. The key rides ONLY in the X-Api-Key header (never the URL/_meta).",
+      "Look up GSA Federal Travel PER-DIEM rates — the max lodging + Meals & Incidental Expenses (M&IE) reimbursement ceilings for official U.S. government travel (api.gsa.gov /travel/perdiem/v2, keyed — DATA_GOV_API_KEY or the shared DEMO_KEY). Input: EITHER `city` (e.g. 'Washington') + `state` (2-letter, e.g. 'DC') OR `zip` (5-digit) — supplying BOTH, or NEITHER, ⇒ invalid_input with 0 fetch; optional `year` (default: the current U.S. federal fiscal year). Returns { rates:[{ city, county, state, zip, year, isOconus, standardRate, mealsUsd, monthlyLodgingUsd:[{ month (1-12), monthName, lodgingUsd }] }] } + honest _meta. HONESTY: lodgingUsd (the API's monthly `value`) is the MAX nightly lodging ceiling for that month — it VARIES SEASONALLY (hence a per-month array), and mealsUsd is the daily M&IE ceiling; both are integer US dollars, null-when-withheld (NEVER 0 — a genuine 0 is preserved). standardRate/isOconus are booleans coerced from the API's string 'true'/'false' (an unrecognized value ⇒ null, never a fabricated false); the months array is preserved AS-IS (never padded to 12). The API returns the COMPLETE rate set (no pagination) ⇒ totalAvailable = the row count, complete:true. A genuine no-match (rates:[]/rate:[]) ⇒ honest empty (returned:0); the API's `errors` field non-null ⇒ invalid_input carrying the message (never a fake empty); a 429 (DEMO_KEY ~10 req/hr, hit quickly) ⇒ rate_limited THROWS; a 5xx/timeout ⇒ upstream_unavailable THROWS; a 200 non-JSON ⇒ schema_drift. DEMO_KEY ~10 req/hr shared ceiling — set DATA_GOV_API_KEY (free at api.data.gov/signup) for 1000/hr. The key rides ONLY in the X-Api-Key header (never the URL/_meta).",
     inputSchema: GsaPerdiemRatesInput,
     handler: (input) => gsaPerdiem.perdiemRates(input),
   }),
@@ -6510,7 +6525,7 @@ export const TOOLS: ToolDef[] = [
   defineTool({
     name: "lda_search_filings",
     description:
-      "Search US Senate LDA (Lobbying Disclosure Act) filings — who is paid HOW MUCH to lobby WHICH federal agency on WHICH issue (lda.senate.gov/api/v1/filings, KEYLESS — anonymous access works; an optional free LDA_API_KEY only raises the rate limit). All inputs optional: `registrantName` (the lobbying firm/in-house filer), `clientName` (who it's for), `lobbyistName`, `filingYear` (4-digit), `filingType` (short code, e.g. 'Q1'/'RR'/'YE'), `agency` (the federal government_entity lobbied — the B2G signal), `issue` (specific lobbying issues text), `page` (1-based, default 1), `pageSize` (1..25, default 25). Returns { filings:[{ filingUuid, filingType, filingYear, filingPeriod, incomeUsd, expensesUsd, registrant, client, lobbyingActivities:[{ issueCode, description, governmentEntities:[names] }], documentUrl, postedDate, terminationDate }] } + honest _meta. HONESTY: totalAvailable is the API's REAL total match count (the corpus is ~1.95M filings) — NOT the rows on this page; pagination is page-based (pass the next page number when hasMore). incomeUsd/expensesUsd are parsed from the null-or-decimal-string income/expenses — null (not reported) ⇒ null, NEVER 0 (a genuine 0 stays 0); a filing reports EITHER income OR expenses, so the other is typically null. Missing lobbying_activities/government_entities ⇒ empty arrays (never fabricated). A genuine no-match (results:[]) ⇒ honest empty (returned:0); a 400 (bad filter) ⇒ invalid_input surfacing the API's message; a 429 ⇒ rate_limited THROWS (Retry-After honored, never routed around); a 5xx/timeout ⇒ upstream_unavailable THROWS; a 200 non-JSON / non-array results / non-number count ⇒ schema_drift. The optional key rides ONLY in the Authorization: Token header (never the URL/_meta).",
+      "Search US Senate LDA (Lobbying Disclosure Act) filings — who is paid HOW MUCH to lobby WHICH federal agency on WHICH issue (lda.senate.gov/api/v1/filings, KEYLESS — anonymous access works; an optional free LDA_API_KEY only raises the rate limit). All inputs optional: `registrantName` (the lobbying firm/in-house filer), `clientName` (who it's for), `lobbyistName`, `filingYear` (4-digit), `filingType` (short code, e.g. 'Q1'/'RR'/'YE'), `agency` (NOTE: /filings/ has NO server-side agency filter — the LDA API silently ignores it, so it is reported in _meta.filtersDropped and NOT applied; government entities are nested per activity in lobbyingActivities[].governmentEntities), `issue` (specific lobbying issues text), `page` (1-based, default 1), `pageSize` (1..25, default 25). Returns { filings:[{ filingUuid, filingType, filingYear, filingPeriod, incomeUsd, expensesUsd, registrant, client, lobbyingActivities:[{ issueCode, description, governmentEntities:[names] }], documentUrl, postedDate, terminationDate }] } + honest _meta. HONESTY: totalAvailable is the API's REAL total match count (the corpus is ~1.95M filings) — NOT the rows on this page; pagination is page-based (pass the next page number when hasMore). incomeUsd/expensesUsd are parsed from the null-or-decimal-string income/expenses — null (not reported) ⇒ null, NEVER 0 (a genuine 0 stays 0); a filing reports EITHER income OR expenses, so the other is typically null. Missing lobbying_activities/government_entities ⇒ empty arrays (never fabricated). A genuine no-match (results:[]) ⇒ honest empty (returned:0); a 400 (bad filter) ⇒ invalid_input surfacing the API's message; a 429 ⇒ rate_limited THROWS (Retry-After honored, never routed around); a 5xx/timeout ⇒ upstream_unavailable THROWS; a 200 non-JSON / non-array results / non-number count ⇒ schema_drift. The optional key rides ONLY in the Authorization: Token header (never the URL/_meta).",
     inputSchema: LdaSearchFilingsInput,
     handler: (input) => lda.searchFilings(input),
   }),
@@ -6796,7 +6811,7 @@ export async function runTool(
 /**
  * Hand-rolled Zod → JSON Schema converter (subset we use).
  */
-function zodToJsonSchema(schema: z.ZodTypeAny): Record<string, unknown> {
+export function zodToJsonSchema(schema: z.ZodTypeAny): Record<string, unknown> {
   const def = (schema as unknown as { _def: { typeName: string } })._def;
   const tn = def.typeName;
   const description = (schema as unknown as { description?: string }).description;
@@ -6848,6 +6863,19 @@ function zodToJsonSchema(schema: z.ZodTypeAny): Record<string, unknown> {
     const innerSchema = zodToJsonSchema(inner);
     return description ? { ...innerSchema, description } : innerSchema;
   }
+  if (tn === "ZodEffects") {
+    // .refine() / .superRefine() / .transform() wrap the REAL schema at
+    // `_def.schema` (used for cross-field rules like "npi OR state required").
+    // Without this branch these fall through to the {type:"string"} default,
+    // publishing a degenerate object-less inputSchema that a schema-driven MCP
+    // client cannot construct a call against — even though the runtime Zod still
+    // demands the full object. Unwrap so the published schema keeps its real
+    // properties / required / enums.
+    const inner = (schema as unknown as { _def: { schema: z.ZodTypeAny } })._def
+      .schema;
+    const innerSchema = zodToJsonSchema(inner);
+    return description ? { ...innerSchema, description } : innerSchema;
+  }
   return { type: "string", ...(description ? { description } : {}) };
 }
 

package/src/socrata.ts

@@ -194,6 +194,36 @@ export const SOCRATA_DOMAINS = [
   "data.lacity.org", // Los Angeles CA (.org, official) — e.g. hf3r-utnq (RAMP Open Bid Opportunities — live bids)
   "data.ramseycountymn.gov", // Ramsey County MN (.gov) — e.g. iu7r-dzmj (Solicitations & Addenda, with due_date/download_url ~516)
   "data.richmondgov.com", // Richmond VA (.com, official) — e.g. xqn7-jvv2 (City Contracts: contract_value/supplier/procurement_type ~1,387)
+  // ── County/city procurement sweep, wave 2 (loop cycle 19, 2026-07-20). Same
+  //    federated-catalog mining + host-scoped count(*) + /resource 200 bare-array
+  //    verification. All large real checkbook/PO/vendor-payment datasets. ──
+  "opendata.howardcountymd.gov", // Howard County MD (.gov) — e.g. mesh-jggc (Vendors Receiving Payments $30k+ ~35k)
+  "data.providenceri.gov", // Providence RI (.gov) — e.g. 425y-pm5m (City & School Dept Purchase Orders ~228k)
+  "fiscalfocus.pittsburghpa.gov", // Pittsburgh PA (.gov) — e.g. t8t2-4b5n (Checkbook Data ~1.01M)
+  "data.coloradosprings.gov", // Colorado Springs CO (.gov) — e.g. yn6y-xikx (Open Checkbook Vendors ~19k)
+  "data.framinghamma.gov", // Framingham MA (.gov) — e.g. cqve-ehkr (Checkbook ~324k)
+  "data.fultoncountyga.gov", // Fulton County GA (.gov) — e.g. mxhc-krcg (Vendor Payments/disbursements ~217k)
+  "atlanta.data.socrata.com", // City of Atlanta GA (Socrata-hosted official portal) — e.g. jmke-icfi (Open Checkbook Ledger ~1.78M)
+  "opendata.cityofmesquite.com", // Mesquite TX (.com, official) — e.g. 6tva-azs5 (Check Register ~144k)
+  // ── County/city procurement sweep, wave 3 (loop cycle 20, 2026-07-20). Expanded
+  //    federated-catalog queries (rfp/rfq/disbursement/expenditure/commodity) +
+  //    offset paging. Provenance confirmed via /api/views attribution or the gov
+  //    domain itself; hosts with only an individual-name attribution on a generic
+  //    *.data.socrata.com subdomain (washoe, newcastle) were DEFERRED. ──
+  "datahub.usac.org", // USAC E-Rate (.org, same org as opendata.usac.org) — e.g. 39tn-hjzv (E-Rate Open Competitive Bidding, FCC Form 470 — schools/libraries LIVE bids ~2.2M)
+  "performance.ci.janesville.wi.us", // City of Janesville WI (.us official municipal domain) — e.g. fd4q-2kma (Open Expenditures Ledger ~1.0M)
+  "datahub.austintexas.gov", // Austin TX secondary hub (.gov) — e.g. 3ebq-e9iz (Purchase Order Quantity/Price detail, commodity/goods procurements ~318k)
+  "cthru.data.socrata.com", // Commonwealth of Massachusetts, Office of the Comptroller — CTHRU statewide spending (attribution-confirmed) — e.g. kv7m-35wn (Budget/Actual + spending authorizations ~48k)
+  "data.macoupincountyil.gov", // Macoupin County IL (.gov) — e.g. wysn-7qcg (Open Expenditures Ledger ~156k)
+  "data.oaklandca.gov", // Oakland CA (.gov) — e.g. 4ewt-5m6f (budget expenditures ~66k)
+  "data.princegeorgescountymd.gov", // Prince George's County MD (.gov) — e.g. csi4-9jzc (Spending Information: payee_name/agency/amount ~62k)
+  "data.cstx.gov", // College Station TX (.gov) — e.g. i5cx-63zy (Open Budget Expenditures ~42k)
+  "datahub.transportation.gov", // US DOT secondary hub (.gov) — e.g. 255k-mnvp (Disbursements by States for Highways SF-2 ~15k)
+  // ── County/city procurement sweep, wave 4 / tail (loop cycle 21, 2026-07-20).
+  //    Diminishing returns (most remaining catalog hits are Canada/AU, demo/test,
+  //    duplicates, or off-theme); these two are the clean US wins. ──
+  "citydata.mesaaz.gov", // Mesa AZ secondary hub (.gov, attr "Office of Management and Budget") — e.g. vdg8-dx96 (City Expenditures ~15.4M)
+  "data.weho.org", // City of West Hollywood CA (.org, official portal) — e.g. atdr-sk64 (Active Contracts: contract_number/contractor_name/status/type — live/current ~1,030)
 ] as const;
 
 export type SocrataDomain = (typeof SOCRATA_DOMAINS)[number];