@pipeworx/mcp-grants-gov 0.1.3 → 0.1.5

package/README.md

@@ -2,16 +2,32 @@
 
 Grants.gov MCP — open federal funding opportunities, no auth.
 
-Part of [Pipeworx](https://pipeworx.io) — an MCP gateway connecting AI agents to 1683+ live data sources.
+Part of [Pipeworx](https://pipeworx.io) — an MCP gateway connecting AI agents to 1879+ live data sources. This is an independent, unofficial integration — not affiliated with, endorsed by, or published by the upstream provider.
 
 ## Tools
 
+Live (proxies Grants.gov's search2 API — currently-open coverage only):
+
 - `search_opportunities(keyword?, status?, agencies?, funding_categories?, aln?, limit?, offset?)` — filter open opportunities.
 - `get_opportunity(opportunity_id)` — full record with synopsis, eligibility, attachments.
 
-## Data source
+Local mirror (fleet #2512 — closed, archived and forecasted history the live API cannot serve):
+
+- `search_historical_opportunities(agency_code?, cfda?, keyword?, status?, opportunity_category?, post_date_from?, post_date_to?, limit?, offset?)` — search the FULL extract, every status, back to 2004.
+- `get_historical_opportunity(opportunity_id)` — one record, any status, with a computed `status`.
+- `agency_posting_trend(agency_code, years?)` — an agency's opportunity count / funding by federal fiscal year.
+
+## Data sources
 
-`https://api.grants.gov/v1/api/` — public POST-JSON API, no key required.
+- Live: `https://api.grants.gov/v1/api/` — public POST-JSON API, no key required.
+- Mirror: Grants.gov's own daily full-catalog XML extract (`grants.gov/xml-extract` ->
+  `prod-grants-gov-chatbot.s3.amazonaws.com/extracts/GrantsDBExtract<YYYYMMDD>v2.zip`,
+  no auth). Loaded into Supabase `grants_gov_opportunities`
+  (`supabase/migrations/226_grants_gov_extract.sql`) by
+  `scripts/grants-gov-upsert.sh` via `.github/workflows/grants-gov-refresh.yml`
+  (`workflow_dispatch` for now — see that workflow for why no `schedule:` is
+  wired yet). Every mirror response carries `data_as_of`, the last successful
+  refresh.
 
 ## Quick Start
 
@@ -57,7 +73,7 @@ directly, instead of just this one's:
 }
 ```
 
-Both URLs reach the same gateway and the same 1683+ data sources. The
+Both URLs reach the same gateway and the same 1879+ data sources. The
 only difference is which pack's tools are listed **directly**; `ask_pipeworx`
 reaches all of them from either one.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipeworx/mcp-grants-gov",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Grants.gov MCP — open federal grant opportunities (free, no auth)",
   "type": "module",
   "main": "src/index.ts",
@@ -26,7 +26,7 @@
     "@cloudflare/workers-types": "^4.20260405.1"
   },
   "pipeworx": {
-    "sourceHash": "v1-1846b32008e4c510b8dde408336adacab39956e0b19cc5caaf9c36b82d53b018",
-    "sourceCommit": "1064560fd9dcd0d6fc2606298c2ba244b3a1fbeb"
+    "sourceHash": "v1-f50f98d6224512fd191b91e450f154acc44d5a59caa5058a0e9f4c36e0854fbe",
+    "sourceCommit": "445bacc5596a845c12d6945fbfcdf330ef60dcc4"
   }
 }

package/server.json

@@ -3,7 +3,7 @@
   "name": "io.github.pipeworx-io/grants-gov",
   "title": "Grants Gov",
   "description": "Grants.gov MCP — open federal grant opportunities (free, no auth)",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "websiteUrl": "https://pipeworx.io/packs/grants-gov",
   "repository": {
     "url": "https://github.com/pipeworx-io/mcp-grants-gov",

package/src/index.ts

@@ -295,12 +295,28 @@ async function httpErrorMessage(res: Response, name: string): Promise<string> {
   // internal-host-class.ts; no-op for every third-party upstream, which is why
   // this touches 481 packs' error text and changes none of it.
   return markInternalOrigin(
-    `${name}: ${res.status}${detailSuffix(await readDetail(res))}`,
+    `${name}: ${res.status}${detailSuffix(await readDetail(res))}${responseHints(res)}`,
     res.url,
     res.status,
   );
 }
 
+/**
+ * Facts about a FAILED response that live in its headers, not its body, appended
+ * so the gateway's classifier (which only ever sees this string) can tell a limit
+ * from a wall from a refusal (fleet #3477). Both are standard, both are absent on
+ * almost every response, and neither is invented: `Retry-After` is the server
+ * saying "come back later" (the one thing that makes a 403 a throttle), and
+ * Cloudflare marks a bot challenge with `cf-mitigated: challenge`.
+ */
+function responseHints(res: Response): string {
+  const hints: string[] = [];
+  const retryAfter = res.headers.get('retry-after');
+  if (retryAfter) hints.push(`retry-after: ${retryAfter.slice(0, 40)}`);
+  if ((res.headers.get('cf-mitigated') ?? '').toLowerCase() === 'challenge') hints.push('Cloudflare bot challenge (cf-mitigated: challenge)');
+  return hints.length ? ` (${hints.join('; ')})` : '';
+}
+
 /**
  * Just the upstream's own explanation — no name, no status.
  *
@@ -670,6 +686,254 @@ function pickMessage(node: unknown, depth: number): string | null {
 function collapse(s: string): string {
   return s.replace(/\s+/g, ' ').trim();
 }
+
+// ── Bounded readers — peak memory, not just wall time (fleet #3322) ──────
+//
+// `parseJson` above does `await res.text()` then `JSON.parse(raw)`, so a
+// multi-megabyte upstream body is held TWICE at peak: once as the raw string
+// and once as the object graph (which for an array of small objects runs
+// several times the byte size). Nothing in `workers/gateway` can see that
+// peak — it happens inside the pack, before any gateway code holds a payload —
+// which is why the gateway-side payload caps of #3293 and #3320 did not move
+// the 503/1102 rate at all.
+//
+// Measured on the live gateway, 2026-10-19, sha fa5de5c34, direct tools/call:
+//   statcan_list_cubes  8,049,566 B   <- one upstream array, no limit argument
+//   gnomad_gene BRCA1   2,732,706 B
+// Five of those in flight at the quality runner's width is ~40 MB of parse
+// peak against a 128 MB isolate; Cloudflare kills the isolate and every
+// request in flight returns 503/1102, which is why the victims look random.
+//
+// `readJsonArrayPrefix` is the fix for the dominant shape: an upstream that
+// returns one enormous top-level JSON array with no server-side paging. It
+// parses elements off the stream and DISCARDS the consumed bytes, so peak
+// retained memory is one element plus one chunk rather than the whole body,
+// and it cancels the body as soon as it has what it was asked for.
+
+/** Default ceiling for a single upstream JSON body read into an isolate.
+ *  Chosen so the known tail (statcan 8.0 MB, gnomad 2.7 MB) is bounded well
+ *  below the point where a handful of concurrent reads exhausts 128 MB. */
+const DEFAULT_MAX_JSON_BYTES = 1_500_000;
+
+interface JsonArrayPrefix<T> {
+  /** The decoded elements, at most `maxItems` of them. */
+  items: T[];
+  /** True when the upstream array had more elements than we read. */
+  truncated: boolean;
+  /** Bytes actually pulled off the socket — the number that bounds memory. */
+  bytesRead: number;
+  /** `Content-Length` if the upstream declared one, else null. */
+  declaredBytes: number | null;
+}
+
+/**
+ * Read the FIRST `maxItems` elements of a top-level JSON array off a response
+ * stream without ever materialising the whole body.
+ *
+ * Use this instead of `parseJson` when the upstream returns one big array and
+ * offers no `limit`/`offset` of its own. Peak retained memory is one element
+ * plus the current chunk; the body is cancelled as soon as `maxItems` is
+ * reached, so the remaining megabytes are never transferred at all.
+ *
+ *     const { items, truncated } = await readJsonArrayPrefix<Cube>(
+ *       res, 'StatCan WDS', { maxItems: limit + offset });
+ *
+ * `maxBytes` is a second, independent backstop for a pathological element (a
+ * single 50 MB object): hitting it stops the read and sets `truncated`, rather
+ * than throwing, because a partial answer plus an explicit truncated flag is
+ * more useful to a caller than an error.
+ */
+async function readJsonArrayPrefix<T>(
+  res: Response,
+  name: string,
+  opts: { maxItems: number; maxBytes?: number },
+): Promise<JsonArrayPrefix<T>> {
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_JSON_BYTES;
+  const declaredRaw = Number(res.headers.get('content-length'));
+  const declaredBytes = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
+
+  // No streaming body available (a mock, or an already-buffered response):
+  // fall back to the whole-text path so behaviour stays correct, just unbounded.
+  if (!res.body) {
+    const all = await parseJson<T[]>(res, name);
+    if (!Array.isArray(all)) throw new Error(`upstream_down: ${name} returned ${typeof all} where a JSON array was expected.`);
+    return {
+      items: all.slice(0, opts.maxItems),
+      truncated: all.length > opts.maxItems,
+      bytesRead: declaredBytes ?? 0,
+      declaredBytes,
+    };
+  }
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  const items: T[] = [];
+
+  let buf = '';
+  let scan = 0;
+  let bytesRead = 0;
+  let started = false; // consumed the opening '['
+  let arrayEnded = false;
+  let depth = 0;
+  let inStr = false;
+  let esc = false;
+  let elemStart = -1;
+  let truncated = false;
+
+  const push = (text: string) => {
+    const t = text.trim();
+    if (!t) return;
+    items.push(JSON.parse(t) as T);
+  };
+
+  try {
+    outer: while (items.length < opts.maxItems && !arrayEnded) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      bytesRead += value.byteLength;
+      buf += decoder.decode(value, { stream: true });
+      if (bytesRead > maxBytes) { truncated = true; break; }
+
+      for (; scan < buf.length; scan++) {
+        const ch = buf[scan];
+
+        if (inStr) {
+          if (esc) { esc = false; continue; }
+          if (ch === '\\') { esc = true; continue; }
+          if (ch === '"') inStr = false;
+          continue;
+        }
+
+        if (!started) {
+          if (ch === '[') { started = true; continue; }
+          if (ch.trim() === '') continue;
+          throw new Error(
+            `upstream_down: ${name} returned a body that does not begin with a JSON array (saw ${JSON.stringify(ch)}). ` +
+              'This reader is only for upstreams that answer with one top-level array.',
+          );
+        }
+
+        if (ch === '"') { if (depth === 0 && elemStart < 0) elemStart = scan; inStr = true; continue; }
+
+        if (ch === '{' || ch === '[') {
+          if (depth === 0 && elemStart < 0) elemStart = scan;
+          depth++;
+          continue;
+        }
+
+        if (ch === '}' || ch === ']') {
+          if (depth > 0) {
+            depth--;
+            if (depth === 0 && elemStart >= 0) {
+              push(buf.slice(elemStart, scan + 1));
+              elemStart = -1;
+              if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
+            }
+            continue;
+          }
+          // depth 0 and a closing bracket: the end of the top-level array.
+          if (elemStart >= 0) { push(buf.slice(elemStart, scan)); elemStart = -1; }
+          arrayEnded = true;
+          break outer;
+        }
+
+        if (ch === ',' && depth === 0) {
+          if (elemStart >= 0) {
+            push(buf.slice(elemStart, scan));
+            elemStart = -1;
+            if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
+          }
+          continue;
+        }
+
+        // A bare scalar element (number / true / false / null).
+        if (depth === 0 && elemStart < 0 && ch.trim() !== '') elemStart = scan;
+      }
+
+      // Drop everything already consumed. THIS is what bounds memory: without
+      // it `buf` grows to the full body and the reader is streaming in name only.
+      if (elemStart >= 0) {
+        buf = buf.slice(elemStart);
+        scan -= elemStart;
+        elemStart = 0;
+      } else {
+        buf = buf.slice(scan);
+        scan = 0;
+      }
+    }
+  } finally {
+    // Cancel rather than drain: the point is that the remaining megabytes are
+    // never transferred. A cancel on an already-finished body is a no-op.
+    await reader.cancel().catch(() => {});
+  }
+
+  return { items, truncated, bytesRead, declaredBytes };
+}
+
+/**
+ * `parseJson`, with a hard ceiling on how many bytes are read.
+ *
+ * For an upstream whose body is not a single array (so `readJsonArrayPrefix`
+ * does not apply) but which can still answer with megabytes. Unlike the array
+ * reader this one THROWS when the cap is hit, because a truncated prefix of an
+ * arbitrary JSON document is not parseable and there is no partial answer to
+ * hand back. A declared `Content-Length` over the cap is refused before a
+ * single chunk is read.
+ */
+async function parseJsonBounded<T>(
+  res: Response,
+  name: string,
+  maxBytes: number = DEFAULT_MAX_JSON_BYTES,
+): Promise<T> {
+  const declaredRaw = Number(res.headers.get('content-length'));
+  const declared = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
+
+  const tooBig = (n: number) =>
+    new Error(
+      `upstream_down: ${name} answered with ${n.toLocaleString()} bytes, over this pack's ${maxBytes.toLocaleString()}-byte ceiling. ` +
+        'The body is not read past the ceiling on purpose — a few concurrent reads of a body that size exhaust the ' +
+        'Worker isolate and take every other in-flight request down with them. Narrow the request (a smaller range, ' +
+        'a specific id, or a lower limit) and retry.',
+    );
+
+  if (declared !== null && declared > maxBytes) {
+    await res.body?.cancel().catch(() => {});
+    throw tooBig(declared);
+  }
+  if (!res.body) return parseJson<T>(res, name);
+
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  let raw = '';
+  let bytesRead = 0;
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      bytesRead += value.byteLength;
+      if (bytesRead > maxBytes) throw tooBig(bytesRead);
+      raw += decoder.decode(value, { stream: true });
+    }
+    raw += decoder.decode();
+  } finally {
+    await reader.cancel().catch(() => {});
+  }
+
+  if (!raw.trim()) {
+    throw new Error(
+      `upstream_down: ${name} answered HTTP ${res.status} with an EMPTY body where JSON was expected. ` +
+        'Nothing about the request can cause this — it is an upstream fault, and the same call may well work on retry.',
+    );
+  }
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    throw new Error(
+      `upstream_down: ${name} answered HTTP ${res.status} with a body that is not valid JSON. ` +
+        `It begins: ${stripMarkup(raw).slice(0, 120) || '(unreadable)'}`,
+    );
+  }
+}
 /**
  * Grants.gov MCP — open federal grant opportunities (free, no auth)
  *
@@ -677,9 +941,38 @@ function collapse(s: string): string {
  * the federal-wide clearinghouse for currently-open and forecasted opportunities.
  *
  * API: https://www.grants.gov/web/grants/s2s/grantor/schemas/grants-gov-search2.html
- * Tools:
+ * Tools (live proxy, currently-open coverage only):
  * - search_opportunities: filter by keyword, status, agency, funding category, dates
  * - get_opportunity:      single full record with attachments and synopsis text
+ *
+ * Tools (local mirror, fleet #2512 — closed/archived/forecasted history the
+ * live API structurally cannot serve):
+ * - search_historical_opportunities: agency/CFDA/keyword/status (incl. closed
+ *   and forecasted)/date-range search over the FULL extract, back to 2004.
+ * - get_historical_opportunity:      one record from the mirror, any status.
+ * - agency_posting_trend:            an agency's opportunity count / funding
+ *   by fiscal year, derived from post_date.
+ *
+ * WHY A MIRROR FOR HISTORY: search2 (BASE_URL below) is Grants.gov's live
+ * index and only usefully answers "what is open or about to open right
+ * now" — there is no live endpoint for "what did this agency post five
+ * fiscal years ago" or "what was forecasted but never opened". Per Bruce's
+ * 2026-09-23 ruling ("build a copy when live doesn't serve"), the historical
+ * tools below read migration 226's grants_gov_opportunities, loaded from
+ * Grants.gov's own daily full-catalog XML export
+ * (grants.gov/xml-extract -> prod-grants-gov-chatbot.s3.amazonaws.com,
+ * verified live 2026-09-30: 83,497 records, no auth) by
+ * scripts/grants-gov-upsert.sh via .github/workflows/grants-gov-refresh.yml.
+ * `data_as_of` is on every mirror response (the last successful refresh,
+ * read from grants_gov_ingest_runs via the grants_gov_coverage() RPC).
+ * The live tools (search_opportunities/get_opportunity) are UNCHANGED —
+ * they still proxy search2 for the "what's open right now" case, which the
+ * mirror (refreshed once daily) is never quite fresh enough for.
+ *
+ * NO NEW PERSONAL DATA: the mirror carries GrantorContactName/Text, an
+ * agency's published point-of-contact for the opportunity — the same class
+ * of disclosure get_opportunity's live GrantorContact* fields already
+ * surface; see the migration file for the full reasoning.
  */
 
 
@@ -709,7 +1002,8 @@ const tools: McpToolExport['tools'] = [
         },
         status: {
           type: 'string',
-          description: 'forecasted | posted | closed | archived (default posted; use a pipe to combine, e.g. "posted|forecasted")',
+          enum: ['posted', 'closed', 'archived', 'forecasted'],
+          description: 'posted (open for applications now) | forecasted (announced, not yet posted) | closed | archived. Default posted. There is no "open" or "active" status: use "posted". For posted plus forecasted together pass open_only: true.',
         },
         open_only: { type: 'boolean', description: 'true restricts to currently-open or forthcoming opportunities (same as status "posted|forecasted"); omit or false leaves status at its default. Alias for status when a caller only knows open/not-open, not the exact status vocabulary.' },
         agencies: { type: 'string', description: 'Comma-separated agency codes (e.g., "EPA,USDA" or a bare department like "HHS" or "NIH"). Grants.gov itself indexes opportunities under a FULLY-QUALIFIED sub-agency code ("HHS-NIH11", "HHS-CDC1", ...), and its own filter returns ZERO hits for a bare department code (verified live 2026-09-25, fleet #2450) — this tool detects a bare code automatically and matches it client-side against every hit\'s sub-agency code instead of sending it upstream, so "HHS" still finds NIH/CDC/FDA opportunities. A code you already have in the full "DEPT-SUBAGENCY#" form is sent upstream unchanged.' },
@@ -740,6 +1034,64 @@ const tools: McpToolExport['tools'] = [
       required: ['opportunity_id'],
     },
   },
+  {
+    name: 'search_historical_opportunities',
+    description:
+      'Search Grants.gov\'s FULL opportunity history (closed and archived opportunities back to 2004, plus forecasted opportunities not yet posted) — coverage search_opportunities\' live API cannot serve, since search2 only usefully answers "what is open right now". Covers Grants.gov\'s full catalog, refreshed once a day (see `data_as_of`). Filter by agency code, CFDA/Assistance Listing number, keyword (full-text over title + description), status, opportunity category, or post-date range. Unlike search_opportunities, status here defaults to EVERY status, not just "posted" — pass status explicitly to narrow. Answers "what did this agency fund last fiscal year", "what opportunities under CFDA 93.351 have ever closed", "what is forecasted for next year that hasn\'t posted yet".',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        agency_code: {
+          type: 'string',
+          description: 'Agency or sub-agency code, e.g. "NIH" or "HHS-NIH11" or "DOS". Matches as a prefix/segment against the fully-qualified codes on file (same bare-vs-qualified forgiveness as search_opportunities\' `agencies`). Alias: agency.',
+        },
+        agency: { type: 'string', description: 'Alias for agency_code.' },
+        cfda: { type: 'string', description: 'Assistance Listing Number / CFDA number, exact match, e.g. "93.351". Alias: aln.' },
+        aln: { type: 'string', description: 'Alias for cfda.' },
+        keyword: { type: 'string', description: 'Full-text search over the opportunity title and description (PostgreSQL websearch syntax — quote a phrase, "-word" to exclude). More forgiving than the live tool\'s exact-phrase keyword match.' },
+        status: {
+          type: 'string',
+          enum: ['posted', 'closed', 'archived', 'forecasted'],
+          description: 'posted (open now) | closed | archived | forecasted (not yet posted). One value. There is no "open" or "active" status: use "posted" for opportunities open for applications. Omit to search every status, unlike the live tool (whose default is "posted").',
+        },
+        opportunity_category: { type: 'string', description: 'Single-letter Grants.gov category code: D discretionary, M mandatory, C continuation, E earmark, O other.' },
+        post_date_from: { type: 'string', description: 'Only opportunities posted on/after this date, YYYY-MM-DD.' },
+        post_date_to: { type: 'string', description: 'Only opportunities posted on/before this date, YYYY-MM-DD.' },
+        limit: { type: 'number', description: 'Rows to return, 1-500 (default 25).' },
+        offset: { type: 'number', description: '0-based start record, for paging.' },
+      },
+      required: [],
+    },
+  },
+  {
+    name: 'get_historical_opportunity',
+    description:
+      'Fetch one Grants.gov opportunity by its numeric opportunity ID, in ANY status (open, closed, archived, or forecasted) — unlike get_opportunity, which calls the live API and only reliably resolves currently-open opportunities. Returns the full record plus a computed `status` (derived from close/archive dates against today, the same vocabulary search_historical_opportunities uses) and `data_as_of`.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        opportunity_id: {
+          type: 'number',
+          description: 'Numeric opportunity ID (returned by search_historical_opportunities or search_opportunities as "id").',
+        },
+      },
+      required: ['opportunity_id'],
+    },
+  },
+  {
+    name: 'agency_posting_trend',
+    description:
+      'An agency\'s opportunity-posting trend by federal fiscal year (Oct-Sep): how many opportunities it posted, total estimated program funding, and average award ceiling, per year. Answers "is this agency posting more or fewer opportunities than last year" and "how has this agency\'s funding trended". Fiscal year is derived from each opportunity\'s post_date.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        agency_code: { type: 'string', description: 'Agency or sub-agency code, e.g. "EPA", "NIH", "HHS-CDC1". Matches as a prefix/segment, same forgiveness as search_historical_opportunities. Alias: agency.' },
+        agency: { type: 'string', description: 'Alias for agency_code.' },
+        years: { type: 'number', description: 'Number of most-recent fiscal years to return, 1-40 (default 10).' },
+      },
+      required: [],
+    },
+  },
 ];
 
 function reqNum(args: Record<string, unknown>, key: string, example: string): number {
@@ -751,7 +1103,13 @@ function reqNum(args: Record<string, unknown>, key: string, example: string): nu
   return n;
 }
 
+const GRANTS_STATUSES = ['posted', 'closed', 'archived', 'forecasted'];
+const MIRROR_TOOLS = new Set(['search_historical_opportunities', 'get_historical_opportunity', 'agency_posting_trend']);
+
 async function callTool(name: string, args: Record<string, unknown>): Promise<unknown> {
+  if (MIRROR_TOOLS.has(name)) {
+    return callMirrorTool(name, args);
+  }
   switch (name) {
     case 'search_opportunities':
       return searchOpps(args);
@@ -776,7 +1134,21 @@ async function grantsPost<T>(path: string, body: unknown): Promise<T> {
 }
 
 async function searchOpps(args: Record<string, unknown>) {
-  const status = (args.status as string) ?? (args.open_only === true ? 'posted|forecasted' : 'posted');
+  // fleet #3471: an off-vocabulary status ("open", "active") was sent upstream,
+  // matched nothing, and came back as total 0 with empty_reason no_match -- a
+  // confident claim the catalog held nothing. Refuse it before any query.
+  // A pipe-joined list of legal values is still accepted (the upstream syntax).
+  const statusArg = args.status == null || String(args.status).trim() === '' ? null : String(args.status).trim().toLowerCase();
+  if (statusArg !== null) {
+    const parts = statusArg.split('|').map((x) => x.trim());
+    if (parts.some((x) => !GRANTS_STATUSES.includes(x))) {
+      throw new Error(
+        `user_error: status "${String(args.status)}" is not a Grants.gov status. Use exactly one of: ${GRANTS_STATUSES.join(', ')}. ` +
+          '"posted" is what is open for applications now; "forecasted" is announced but not yet posted; use open_only: true for posted plus forecasted.',
+      );
+    }
+  }
+  const status = statusArg ?? (args.open_only === true ? 'posted|forecasted' : 'posted');
   const requestedRows = Math.min(1000, Math.max(1, (args.limit as number) ?? 25));
   const requestedOffset = Math.max(0, (args.offset as number) ?? 0);
 
@@ -829,10 +1201,16 @@ async function searchOpps(args: Record<string, unknown>) {
   const rawHits = data.data?.oppHits ?? [];
 
   if (!needsClientSideAgencyFilter) {
+    const total = data.data?.hitCount ?? 0;
     return {
-      total: data.data?.hitCount ?? 0,
+      total,
       returned: rawHits.length,
       opportunities: rawHits.map(normalizeHit),
+      // fleet #3471 audit: an unrecognised funding_categories code matches nothing
+      // upstream and read as "no opportunities" (BOGUS -> total 0, no_match).
+      ...(total === 0 && args.funding_categories
+        ? { hint: `funding_categories "${String(args.funding_categories)}" was sent upstream as given; a code Grants.gov does not recognise matches nothing, which looks identical to no opportunities. Codes are short upper-case ones such as AG, ED, ENV, HL, HU, ST, T; retry without funding_categories to confirm.` }
+        : {}),
     };
   }
 
@@ -911,7 +1289,17 @@ async function getOpp(opportunityId: number) {
   }
   const d = data.data;
   if (!d) throw new Error(`Grants.gov: no data for opportunity ${opportunityId}`);
-
+  // fleet #3471 audit: an id the live API does not hold (closed/archived, or never
+  // existed) came back as a record of 25 nulls. That is a "not found" wearing the
+  // shape of a found record, so say so.
+  if (d.id == null && d.opportunityNumber == null && d.opportunityTitle == null) {
+    return {
+      found: false,
+      reason: 'not_found_in_live_api',
+      opportunity_id: opportunityId,
+      hint: 'The live Grants.gov API does not hold this opportunity (it may be closed or archived, or the id is wrong). Try get_historical_opportunity, which resolves an opportunity in any status.',
+    };
+  }
   return normalizeDetail(d);
 }
 
@@ -1010,4 +1398,254 @@ function normalizeDetail(d: OpportunityDetail) {
   };
 }
 
+/* ----------------------------------------------------- mirror (local DB) */
+
+interface DbConfig {
+  url: string;
+  key: string;
+}
+
+async function sbRpc(cfg: DbConfig, fn: string, body: Record<string, unknown>): Promise<unknown[]> {
+  const url = `${cfg.url}/rest/v1/rpc/${fn}`;
+  const res = await pwFetch(url, {
+    method: 'POST',
+    headers: {
+      apikey: cfg.key,
+      Authorization: `Bearer ${cfg.key}`,
+      'Content-Type': 'application/json',
+      Prefer: 'params=single-object',
+    },
+    body: JSON.stringify(body),
+  });
+  if (!res.ok) throw await httpError(res, 'Grants.gov historical data');
+  const parsed = await res.json();
+  if (!Array.isArray(parsed)) {
+    throw new Error('Grants.gov historical data: RPC did not return a row array — treating this as a failure.');
+  }
+  return parsed;
+}
+
+let cachedDataAsOf: { value: string | null; coverage: Record<string, unknown> | null } | undefined;
+
+/** The last successful refresh + coverage stats, read once per Worker instance and reused across calls. */
+async function mirrorCoverage(cfg: DbConfig): Promise<{ value: string | null; coverage: Record<string, unknown> | null }> {
+  if (cachedDataAsOf !== undefined) return cachedDataAsOf;
+  try {
+    const rows = (await sbRpc(cfg, 'grants_gov_coverage', {})) as Record<string, unknown>[];
+    const row = rows[0] ?? null;
+    cachedDataAsOf = { value: (row?.data_as_of as string | undefined) ?? null, coverage: row };
+  } catch {
+    cachedDataAsOf = { value: null, coverage: null }; // freshness is metadata, never the reason a call fails
+  }
+  return cachedDataAsOf;
+}
+
+function withDataAsOf<T extends Record<string, unknown>>(result: T, asOf: string | null): T & { data_as_of: string | null } {
+  return {
+    ...result,
+    data_as_of: asOf,
+    ...(asOf
+      ? {}
+      : {
+          coverage_status:
+            'The Grants.gov historical dataset has not completed its first load yet — results may be empty or incomplete. This is a deployment state, not a query problem.',
+        }),
+  };
+}
+
+function agencyArg(args: Record<string, unknown>): string | undefined {
+  const raw = (args.agency_code ?? args.agency) as string | undefined;
+  return raw ? String(raw).trim() : undefined;
+}
+
+interface MirrorSearchRow {
+  opportunity_id: number;
+  opportunity_title: string | null;
+  opportunity_number: string | null;
+  agency_code: string | null;
+  agency_name: string | null;
+  status: string | null;
+  post_date: string | null;
+  close_date: string | null;
+  fiscal_year: number | null;
+  cfda_numbers: string[] | null;
+  award_ceiling: number | null;
+  estimated_total_program_funding: number | null;
+  total_count: number;
+}
+
+function shapeSearchRow(r: MirrorSearchRow) {
+  return {
+    id: r.opportunity_id,
+    title: r.opportunity_title,
+    number: r.opportunity_number,
+    agency_code: r.agency_code,
+    agency: r.agency_name,
+    status: r.status,
+    post_date: r.post_date,
+    close_date: r.close_date,
+    fiscal_year: r.fiscal_year,
+    assistance_listing_numbers: r.cfda_numbers ?? [],
+    award_ceiling: r.award_ceiling,
+    estimated_total_program_funding: r.estimated_total_program_funding,
+    grants_gov_url: `https://www.grants.gov/search-results-detail/${r.opportunity_id}`,
+  };
+}
+
+async function searchHistorical(cfg: DbConfig, args: Record<string, unknown>) {
+  const limit = Math.min(500, Math.max(1, Number(args.limit) || 25));
+  const offset = Math.max(0, Number(args.offset) || 0);
+  const HISTORICAL_STATUSES = GRANTS_STATUSES;
+  const status = args.status == null || String(args.status).trim() === '' ? null : String(args.status).trim().toLowerCase();
+  // fleet #3404: an off-vocabulary status ("open", "active") matched nothing in
+  // the RPC and came back as a clean total:0. Refuse it, naming the legal four.
+  if (status !== null && !HISTORICAL_STATUSES.includes(status)) {
+    throw new Error(
+      `user_error: status "${String(args.status)}" is not a Grants.gov status. Use exactly one of: ${HISTORICAL_STATUSES.join(', ')}. ` +
+        '"posted" is what is open for applications now; "forecasted" is announced but not yet posted; omit status to search all four.',
+    );
+  }
+  const agencyIn = agencyArg(args);
+  const rows = (await sbRpc(cfg, 'grants_gov_search', {
+    p_agency_code: agencyIn ?? null,
+    p_cfda: (args.cfda ?? args.aln) ? String(args.cfda ?? args.aln) : null,
+    p_keyword: args.keyword ? String(args.keyword) : null,
+    p_status: status,
+    p_opportunity_category: args.opportunity_category ? String(args.opportunity_category) : null,
+    p_post_date_from: args.post_date_from ? String(args.post_date_from) : null,
+    p_post_date_to: args.post_date_to ? String(args.post_date_to) : null,
+    p_limit: limit,
+    p_offset: offset,
+  })) as MirrorSearchRow[];
+
+  const agencyLooksLikeName = agencyIn !== undefined && (/\s/.test(agencyIn) || agencyIn.length > 16);
+  return {
+    total: rows.length ? rows[0].total_count : 0,
+    returned: rows.length,
+    offset,
+    opportunities: rows.map(shapeSearchRow),
+    source: 'Grants.gov\'s own full-catalog extract, refreshed daily — covers every status, not just currently-open (see search_opportunities for the live-only view).',
+    ...(rows.length === 0 && offset === 0
+      ? {
+          empty_reason: agencyLooksLikeName ? 'agency_not_a_code' : 'no_match',
+          hint: agencyLooksLikeName
+            ? `agency_code takes an agency CODE such as "NIH", "HHS-NIH11" or "DOS", not a name ("${agencyIn}"). Pass the code, or put the name words in \`keyword\`.`
+            : 'No opportunity in the catalog matches every filter given. Loosen one: drop status, use a bare department code (e.g. "HHS") or a shorter keyword.',
+        }
+      : {}),
+  };
+}
+
+async function getHistorical(cfg: DbConfig, opportunityId: number) {
+  const rows = (await sbRpc(cfg, 'grants_gov_get', { p_opportunity_id: opportunityId })) as Record<string, unknown>[];
+  const row = rows[0];
+  if (!row) {
+    return {
+      found: false,
+      reason: `No opportunity with id ${opportunityId} in Grants.gov's historical dataset.`,
+      hint: 'Check the id with search_historical_opportunities, or the id may belong to an opportunity type not yet loaded.',
+    };
+  }
+  const status =
+    row.record_type === 'forecast'
+      ? 'forecasted'
+      : !row.close_date || (row.close_date as string) >= new Date().toISOString().slice(0, 10)
+        ? 'posted'
+        : !row.archive_date || (row.archive_date as string) >= new Date().toISOString().slice(0, 10)
+          ? 'closed'
+          : 'archived';
+  return {
+    found: true,
+    id: row.opportunity_id,
+    record_type: row.record_type,
+    status,
+    title: row.opportunity_title,
+    number: row.opportunity_number,
+    category: row.opportunity_category,
+    funding_instrument_type: row.funding_instrument_type,
+    funding_activity_categories: row.category_of_funding_activity ?? [],
+    assistance_listing_numbers: row.cfda_numbers ?? [],
+    eligible_applicant_codes: row.eligible_applicant_codes ?? [],
+    eligibility_note: row.additional_information_on_eligibility,
+    agency_code: row.agency_code,
+    agency: row.agency_name,
+    post_date: row.post_date,
+    close_date: row.close_date,
+    archive_date: row.archive_date,
+    last_updated_date: row.last_updated_date,
+    award_ceiling: row.award_ceiling,
+    award_floor: row.award_floor,
+    estimated_total_program_funding: row.estimated_total_program_funding,
+    expected_number_of_awards: row.expected_number_of_awards,
+    description: row.description,
+    cost_sharing_required: row.cost_sharing_or_matching_requirement,
+    contact_name: row.grantor_contact_name,
+    contact_email: row.grantor_contact_email,
+    contact_phone: row.grantor_contact_phone_number,
+    fiscal_year: row.fiscal_year,
+    estimated_synopsis_post_date: row.estimated_synopsis_post_date,
+    estimated_synopsis_close_date: row.estimated_synopsis_close_date,
+    estimated_award_date: row.estimated_award_date,
+    grants_gov_url: `https://www.grants.gov/search-results-detail/${row.opportunity_id}`,
+    source: 'Grants.gov\'s own full-catalog extract, refreshed daily.',
+  };
+}
+
+interface TrendRow {
+  fiscal_year: number;
+  opportunities_posted: number;
+  total_estimated_funding: number | null;
+  avg_award_ceiling: number | null;
+}
+
+async function agencyTrend(cfg: DbConfig, args: Record<string, unknown>) {
+  const agency = agencyArg(args);
+  if (!agency) {
+    throw new Error('agency_posting_trend needs an agency code. Pass agency_code (or its alias agency), e.g. "EPA" or "HHS-NIH11".');
+  }
+  const years = Math.min(40, Math.max(1, Number(args.years) || 10));
+  const rows = (await sbRpc(cfg, 'grants_gov_agency_trend', { p_agency_code: agency, p_years: years })) as TrendRow[];
+  return {
+    agency_code: agency,
+    years_returned: rows.length,
+    trend: rows.map((r) => ({
+      fiscal_year: r.fiscal_year,
+      opportunities_posted: r.opportunities_posted,
+      total_estimated_funding: r.total_estimated_funding,
+      avg_award_ceiling: r.avg_award_ceiling,
+    })),
+    source: 'Grants.gov\'s own full-catalog extract, refreshed daily; fiscal year derived from each opportunity\'s post_date (federal FY = Oct-Sep).',
+  };
+}
+
+async function callMirrorTool(name: string, args: Record<string, unknown>): Promise<unknown> {
+  const url = (args._supabaseUrl as string | undefined)?.trim();
+  const key = (args._supabaseKey as string | undefined)?.trim();
+  if (!url || !key) {
+    throw new Error(
+      'The Grants.gov historical dataset is not configured on this deployment — an operator must enable its data credentials. This is a setup problem, not your arguments. search_opportunities/get_opportunity (the live tools) do not need this and still work.',
+    );
+  }
+  const cfg: DbConfig = { url, key };
+  const asOf = await mirrorCoverage(cfg);
+
+  let result: unknown;
+  switch (name) {
+    case 'search_historical_opportunities':
+      result = await searchHistorical(cfg, args);
+      break;
+    case 'get_historical_opportunity':
+      result = await getHistorical(cfg, reqNum(args, 'opportunity_id', '363992 (numeric ID)'));
+      break;
+    case 'agency_posting_trend':
+      result = await agencyTrend(cfg, args);
+      break;
+    default:
+      throw new Error(`Unknown tool: ${name}`);
+  }
+  return result && typeof result === 'object' && !Array.isArray(result)
+    ? withDataAsOf(result as Record<string, unknown>, asOf.value)
+    : result;
+}
+
 export default { tools, callTool, meter: { credits: 1 } } satisfies McpToolExport;

package/src/server.ts

@@ -8,8 +8,9 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import pack from './index.js';
 
+
 const server = new Server(
-  { name: '@pipeworx/mcp-grants-gov', version: '0.1.3' },
+  { name: '@pipeworx/mcp-grants-gov', version: '0.1.5' },
   { capabilities: { tools: {} } },
 );
 
@@ -24,7 +25,9 @@ server.setRequestHandler(ListToolsRequestSchema, async () => ({
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   const { name, arguments: args } = request.params;
   try {
-    const result = await pack.callTool(name, (args ?? {}) as Record<string, unknown>);
+    const callArgs = { ...((args ?? {}) as Record<string, unknown>) };
+    
+    const result = await pack.callTool(name, callArgs);
     return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
   } catch (err) {
     return {