@cliwant/mcp-sam-gov 1.11.0 → 1.13.0

package/src/fdic.ts

@@ -523,7 +523,7 @@ export async function searchInstitutions(args: {
   const records = env.records.map(mapInstitution);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = [];
@@ -618,7 +618,7 @@ export async function institutionFinancials(args: {
   const records = env.records.map(mapFinancials);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const notes: string[] = [freshnessNote(env.indexName, env.indexCreated), ASSET_NOTE];
@@ -862,7 +862,7 @@ export async function bankFailures(args: {
   const records = env.records.map(mapFailure);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = [];
@@ -1184,7 +1184,7 @@ export async function institutionHistory(args: {
   const records = env.records.map(mapHistory);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = [];
@@ -1524,7 +1524,7 @@ export async function industrySummary(args: {
   const records = env.records.map(mapSummary);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = [];
@@ -1820,7 +1820,7 @@ export async function riskRatios(args: {
   const records = env.records.map(mapRiskRatios);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = ["cert"];
@@ -2013,7 +2013,7 @@ export async function branchDeposits(args: {
   const records = env.records.map(mapBranchDeposit);
   const returned = records.length;
   const totalAvailable = env.totalAvailable;
-  const hasMore = offset + returned < totalAvailable;
+  const hasMore = returned > 0 && offset + returned < totalAvailable;
   const nextOffset = hasMore ? offset + returned : null;
 
   const filtersApplied: string[] = [];

package/src/fema.ts

@@ -407,8 +407,14 @@ function shapeResponse(args: {
 
   const rows = coerceAmounts(rawRows as FemaRow[], def.amountFields);
   const returned = rows.length;
+  // `returned > 0` guard (matches nvd/usaspending): an EMPTY page MUST terminate the
+  // walk. OpenFEMA's metadata.count can EXCEED the rows it will actually serve via
+  // $skip/$top (live: the ~822k-row public_assistance set stops serving rows well
+  // before its count), so without this guard a near-end offset yields returned:0 while
+  // offset < count ⇒ hasMore:true, nextOffset === offset — a non-advancing cursor an
+  // agent following nextOffset re-requests forever (empty page, re-scan, repeat).
   const hasMore =
-    totalAvailable !== null && args.offset + returned < totalAvailable;
+    returned > 0 && totalAvailable !== null && args.offset + returned < totalAvailable;
   const nextOffset = hasMore ? args.offset + returned : null;
 
   const notes: string[] = [SHAPE_NOTE];
@@ -421,6 +427,14 @@ function shapeResponse(args: {
       "This page returned fewer rows than the requested limit while more remain (OpenFEMA byte-truncates a wide page below $top); metadata.count is authoritative — page with a larger offset ($skip).",
     );
   }
+  // Phantom tail: OpenFEMA returned 0 rows though metadata.count is higher — its count
+  // can exceed the rows it will serve via pagination, so the walk is COMPLETE here even
+  // though returned:0 < count. Disclose it so the count is not read as a reachable target.
+  if (returned === 0 && totalAvailable !== null && args.offset > 0 && args.offset < totalAvailable) {
+    notes.push(
+      `OpenFEMA served 0 rows at offset ${args.offset} although metadata.count is ${totalAvailable} — its count can exceed the rows actually pageable via $skip/$top, so pagination is COMPLETE here (do not treat count as a reachable row target; narrow the filter for an exact set).`,
+    );
+  }
   // Deep-offset caveat on the ~800k PA set (ADR-0016 OQ2).
   if (args.offset > 100000) {
     notes.push(

package/src/grants.ts

@@ -45,13 +45,17 @@ export async function searchGrants(args: {
   oppStatuses?: GrantStatus[];
   rows?: number;
 }) {
+  // The tool ALWAYS sends oppStatuses — default forecasted|posted when the caller
+  // omits it — so it is ALWAYS an applied server-side filter (disclosed below).
+  const effOppStatuses = args.oppStatuses ?? ["forecasted", "posted"];
+  const statusDefaulted = args.oppStatuses == null;
   const body: Record<string, unknown> = {
     rows: args.rows ?? 10,
     keyword: args.keyword ?? "",
     cfda: args.cfda ?? "",
     agencies: args.agency ?? "",
     oppNum: args.oppNum ?? "",
-    oppStatuses: (args.oppStatuses ?? ["forecasted", "posted"]).join("|"),
+    oppStatuses: effOppStatuses.join("|"),
   };
   type Resp = {
     errorcode?: number;
@@ -135,8 +139,17 @@ export async function searchGrants(args: {
   if (args.cfda) sent.push("cfda");
   if (args.agency) sent.push("agency");
   if (args.oppNum) sent.push("oppNum");
-  if (args.oppStatuses?.length) sent.push("oppStatuses");
+  // oppStatuses is ALWAYS sent (default or caller-supplied), so ALWAYS disclose the
+  // EFFECTIVE set — omitting it when defaulted made totalAvailable (a status-filtered
+  // subset) read as unfiltered (live: keyword=cybersecurity → 235 forecasted|posted vs
+  // 2008 all-statuses, a 1773-record closed/archived bucket hidden).
+  sent.push(`oppStatuses(${effOppStatuses.join("|") || "all-statuses"})`);
   const notes: string[] = [];
+  if (statusDefaulted) {
+    notes.push(
+      "No oppStatuses supplied — DEFAULTED to forecasted|posted, so CLOSED and ARCHIVED opportunities are EXCLUDED from both the results AND totalAvailable (this total is the open/forecasted subset, not all statuses). To count or return every status, pass oppStatuses including 'closed'/'archived'.",
+    );
+  }
   if (args.agency || args.cfda) {
     notes.push(
       "Grants.gov applies the agency/CFDA filter server-side (live-verified 2026-07-20): a bogus or misspelled value returns 0 results, NOT an error and NOT the unfiltered set — so an unexpectedly EMPTY filtered search most often means the agency/CFDA value is invalid, not that no grants exist. `filtersApplied` reflects that the filter was sent. Verify the agency code / CFDA number if a filtered result is surprisingly empty.",

package/src/gsa-csv.ts

@@ -57,10 +57,12 @@
 import { createReadStream, createWriteStream } from "node:fs";
 import { mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
 import { createInterface } from "node:readline";
+import { once } from "node:events";
 import { pipeline } from "node:stream/promises";
 import * as os from "node:os";
 import * as path from "node:path";
 import { fetchWithRetry, ToolErrorCarrier } from "./errors.js";
+import { driftError } from "./datasource.js";
 import { withMeta } from "./meta.js";
 
 // ─── Source + config ─────────────────────────────────────────────
@@ -309,20 +311,34 @@ function fieldsFromRecord(rec: string[]): NoticeFields {
  * loads the whole file into memory — readline yields one physical line at a
  * time and the assembler holds at most one in-progress (quote-spanning) record.
  */
-async function buildIndexFromFile(
+export async function buildIndexFromFile(
   csvPath: string,
 ): Promise<{ notices: Record<string, NoticeFields>; rowCount: number }> {
   const notices: Record<string, NoticeFields> = Object.create(null);
   let headerSeen = false;
   let rowCount = 0;
 
-  const rl = createInterface({
-    input: createReadStream(csvPath, { encoding: "utf8" }),
-    crlfDelay: Infinity,
-  });
+  const stream = createReadStream(csvPath, { encoding: "utf8" });
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
   const asm = makeRecordAssembler((rec) => {
     if (!headerSeen) {
       headerSeen = true; // first logical record is the 47-column header
+      // [P4] The column contract is POSITIONAL (fixed COL indices). If GSA
+      // reorders / inserts / renames a column, trusting the old positions would
+      // serve WRONG-POSITION data as authoritative (or, if NoticeId shifts, skip
+      // every row → a false "not in the current CSV snapshot"). Neither throws
+      // today. Assert the expected header NAME at each COL index before indexing
+      // any data row — a mismatch is schema drift, never a fake-empty (mirrors the
+      // census-economic.ts header guard). Each COL key IS its expected header name.
+      for (const [expectedName, idx] of Object.entries(COL)) {
+        const got = (rec[idx] ?? "").trim();
+        if (got !== expectedName) {
+          throw driftError(
+            "gsa:csv",
+            `GSA Contract-Opportunities CSV header drift — column ${idx} is ${JSON.stringify(got)}, expected ${JSON.stringify(expectedName)}. The positional column contract changed; refusing to index (a silent column shift would serve wrong-position data as authoritative, or skip every row as a false "not in the current snapshot").`,
+          );
+        }
+      }
       return;
     }
     rowCount++;
@@ -333,8 +349,23 @@ async function buildIndexFromFile(
     notices[noticeId.toLowerCase()] = fieldsFromRecord(rec);
   });
 
-  for await (const line of rl) asm.push(line);
-  asm.flush();
+  try {
+    for await (const line of rl) asm.push(line);
+    asm.flush();
+  } finally {
+    // Release the OS file handle on ALL exit paths (including the header-drift
+    // throw, which aborts mid-stream). destroy() is ASYNC — the fd is not freed
+    // until the 'close' event — so AWAIT it; otherwise a caller deleting the file
+    // immediately after (Windows holds a lock on an open read stream) hits
+    // ENOTEMPTY/EBUSY. On the normal EOF path the stream auto-destroys, so guard
+    // on `.destroyed` and set the listener BEFORE destroy() to catch its 'close'.
+    rl.close();
+    if (!stream.destroyed) {
+      const closed = once(stream, "close");
+      stream.destroy();
+      await closed.catch(() => {});
+    }
+  }
 
   return { notices, rowCount };
 }

package/src/nppes.ts

@@ -793,16 +793,29 @@ export async function lookupProvider(
   const candidateNext = skip + returned;
   const nextSkip = pageFull && candidateNext <= NPPES_MAX_SKIP ? candidateNext : null;
   const hasMore = nextSkip !== null;
-  const totalAvailable = skip + returned;
+  // P1: NPPES exposes no grand total. `skip + returned` is the total ONLY when the
+  // page actually held rows — a partial/last page ⇒ EXACT, a full page ⇒ a lower
+  // bound (flagged below). An EMPTY page past skip 0 proves only that the true total
+  // is ≤ skip (an UPPER bound), NEVER a count: reporting skip + 0 = skip would
+  // fabricate an arbitrary total (e.g. skip 56 over a 6-match query ⇒ a false "56").
+  // At skip 0 an empty page is the genuine zero.
+  const overSkipped = returned === 0 && skip > 0;
+  const totalAvailable = overSkipped ? null : skip + returned;
 
   const notes: string[] = [];
   if (pageFull) {
-    notes.push(lowerBoundNote(totalAvailable));
+    // pageFull ⇒ returned === limit ≥ 1 ⇒ overSkipped false ⇒ totalAvailable is the
+    // numeric lower bound skip + returned (pass it directly so the type stays number).
+    notes.push(lowerBoundNote(skip + returned));
     if (nextSkip === null) {
       notes.push(
         `A full page was returned but the next page would exceed the skip ≤ ${NPPES_MAX_SKIP} policy cap — additional matches exist but are NOT reachable via this tool. Narrow your filters for a complete set.`,
       );
     }
+  } else if (overSkipped) {
+    notes.push(
+      `No providers at skip=${skip}: the skip offset is past the end of the result set. An empty page proves only that the true total is ≤ ${skip} (an upper bound), not a count — so totalAvailable is unknown (null). Page back (a lower skip) to reach the last populated page and its exact total.`,
+    );
   } else if (returned === 0) {
     notes.push(
       `No NPPES providers matched (found:false / empty — an honest zero, distinct from an outage or a body-level error). Not a fitness determination.`,
@@ -813,8 +826,9 @@ export async function lookupProvider(
   const meta: Partial<ResponseMeta> = {
     source: SOURCE,
     keylessMode: true,
-    // A full page is never complete (more may exist, reachable or not).
-    truncated: pageFull,
+    // A full page (more may exist) OR an over-skip empty page (matches exist at a
+    // LOWER skip, not here) is not complete; only skip-0 / a partial last page is.
+    truncated: pageFull || overSkipped,
     returned,
     totalAvailable,
     filtersApplied,

package/src/nsf.ts

@@ -158,6 +158,13 @@ const AMOUNTS_NOTE =
 const DATA_CURRENCY_NOTE =
   "NSF updates awards on a rolling basis; per-record refresh lag is not API-verifiable.";
 
+/** Unstable-order disclosure: NSF exposes no stable server-side sort/tiebreaker, so
+ *  paging by offset can overlap (duplicate) or skip rows across pages. Fires whenever
+ *  the result set spans more than one page (hasMore, or the caller is already past
+ *  offset 0). Live-verified: repeated identical requests reorder; adjacent windows overlap. */
+const NSF_ORDER_UNSTABLE_NOTE =
+  "NSF returns results in an order that is NOT stably sorted (no server-side stable sort/tiebreaker), so paging by offset across multiple pages can DUPLICATE or SKIP rows (live-verified: repeated identical requests reorder, and adjacent offset windows overlap). totalAvailable is the reliable COUNT — do NOT enumerate by walking offsets; to list an exact set, narrow the query (state / keyword / PI / date / UEI) so the whole result fits one page (offset 0).";
+
 /** Live-verified date-filter semantics (Open-Q6 — do not assume). */
 const DATE_SEMANTICS_NOTE =
   "dateStart / dateEnd filter on the award ACTION date (the initial award / obligation date — the `date` / initAmendmentDate field, live-verified 2026-07-12), NOT the project startDate or expDate. Format is STRICT mm/dd/yyyy; a yyyy-mm-dd (or any other format) is silently mis-parsed by NSF (not an error), so it is rejected client-side.";
@@ -618,6 +625,9 @@ export async function searchAwards(args: NsfSearchArgs): Promise<MetaBundle> {
   const notes: string[] = [NSF_GRANT_CAVEAT, UEI_JOIN_NOTE, AMOUNTS_NOTE];
   if (multiWordKeyword) notes.push(orSemanticsNote(multiWordKeyword)); // M1
   if (totalIsLowerBound) notes.push(LOWER_BOUND_NOTE);
+  // Unstable-order disclosure: fire whenever the result spans >1 page (offset paging is
+  // in play), so an agent never silently double-counts/skips rows by walking offsets.
+  if (hasMore || offset > 0) notes.push(NSF_ORDER_UNSTABLE_NOTE);
   if (rpp < limit) notes.push(clampNote(rpp));
   if (args.dateStart !== undefined || args.dateEnd !== undefined)
     notes.push(DATE_SEMANTICS_NOTE);

package/src/open-checkbook.ts

@@ -0,0 +1,183 @@
+/**
+ * open-checkbook.ts — Socrata "Open Expenditures / Open Checkbook" row-level
+ * vendor-payment API, a keyless-first SLED source (loop cycle 76, 2026-07-24 —
+ * South Dakota dark-state closure, the LAST dark state).
+ *
+ * WHAT IT ADDS: some US governments run Socrata's "Open Expenditures" product,
+ * whose public dashboard fronts a KEYLESS app-proxy JSON API at
+ *   `https://{host}/api/checkbook_data.json?year=…&{filters}&page=P&limit=L`
+ * → `{ data:[…row-level payments…], count, total_amount }`. First (and, as of
+ * this writing, only live-verified) portal: **South Dakota Open Checkbook**
+ * (740,980 vendor-payment rows, ~$8.41B, the ~3 most-recent fiscal years).
+ *
+ * ★ KEYLESS- vs-GATED HONESTY (load-bearing): the app-proxy above is anonymous
+ *   and public. The UNDERLYING Socrata SODA dataset (7uwr-juaf on
+ *   southdakota.data.socrata.com) is **403 login-gated** — this module NEVER
+ *   touches it and NEVER presents it as reachable. We only call the public
+ *   /api/checkbook_data.json surface the dashboard itself uses anonymously.
+ *
+ * ★ CURATED allowlist (SSRF core): each portal is a FIXED, live-verified host;
+ *   the `portal` enum in server.ts is built FROM the keys. Host asserted before
+ *   fetch (redirect:"error").
+ *
+ * ★ HONESTY PILLARS:
+ *   P1: totalAvailable = the API's own `count` (the REAL filtered total — it
+ *     matches the product's totals.json exactly; e.g. 740,980 unfiltered,
+ *     109,887 for org1=TRANSPORTATION), NEVER the page length.
+ *   P2: getJson THROWS on 429/5xx/timeout — NEVER a fake empty. A bogus filter
+ *     ⇒ honest count:0/empty; a deep offset past the end ⇒ returned:0 with the
+ *     real count preserved (an honest tail, not an outage).
+ *   P3: `amount` ⇒ number|null (a real $0 is 0, an absent value is null, never a
+ *     fabricated 0); all other fields via str (null-never-empty). Dates verbatim.
+ *   P4: a body that is not `{data:[…], count:<number>}` ⇒ schema_drift.
+ *   ★ Coverage disclosure (P5): only the ~3 most-recent fiscal years are exposed
+ *     by the product (NOT full history) — disclosed every response.
+ */
+
+import { ToolErrorCarrier } from "./errors.js";
+import { getJson, driftError } from "./datasource.js";
+import { num, str } from "./coerce.js";
+import { withMeta, type MetaBundle, type ResponseMeta } from "./meta.js";
+
+export { num };
+
+// ─── Curated portal allowlist (SSRF core) — LIVE-VERIFIED 2026-07-24 ──
+export type OpenCheckbookPortal = { key: string; host: string; label: string; note: string };
+export const OPEN_CHECKBOOK_PORTALS: readonly OpenCheckbookPortal[] = [
+  {
+    key: "sd",
+    host: "southdakota.spending.socrata.com",
+    label: "South Dakota — Open Checkbook",
+    note: "State of South Dakota vendor-payment checkbook (row fields: expense_category, description, fund, payment_date, vendor, org1=department, amount, custom_checkbook_field7=invoice ref, payment_id). ~740,980 rows / ~$8.41B across the ~3 most-recent fiscal years (NOT full history). The underlying Socrata SODA dataset is login-gated; this public app-proxy is the keyless door.",
+  },
+] as const;
+
+const PORTAL_BY_KEY: ReadonlyMap<string, OpenCheckbookPortal> = new Map(OPEN_CHECKBOOK_PORTALS.map((p) => [p.key, p]));
+
+// Sort fields the product supports (validated — an SSRF/injection + silent-noop guard).
+const SORT_FIELDS = new Set(["amount", "payment_date", "vendor", "org1", "expense_category"]);
+
+// ─── SSRF-guarded fetch (fixed allowlist host + assertion) ──
+async function getCheckbook(portal: OpenCheckbookPortal, params: URLSearchParams): Promise<unknown> {
+  const url = `https://${portal.host}/api/checkbook_data.json?${params.toString()}`;
+  const built = new URL(url);
+  if (built.hostname !== portal.host || built.protocol !== "https:") {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Constructed Open-Checkbook URL host ${JSON.stringify(built.hostname)} (${built.protocol}) does not match the allowlisted portal host ${JSON.stringify(portal.host)} over https — refusing to fetch (SSRF safety).`,
+      retryable: false,
+      upstreamEndpoint: `open-checkbook:${portal.key}`,
+    });
+  }
+  return getJson(url, { label: `open-checkbook:${portal.key}`, redirect: "error" });
+}
+
+// ─── Tool: open_checkbook_search ──────────────────────────────────
+export type OpenCheckbookSearchArgs = {
+  portal: string;
+  year?: string;
+  vendor?: string;
+  org?: string;
+  expenseCategory?: string;
+  sortBy?: string;
+  sortOrder?: string;
+  limit?: number;
+  offset?: number;
+};
+
+/**
+ * Row-level vendor-payment search over a curated Socrata Open-Expenditures
+ * checkbook portal (keyless). `portal` is an allowlist enum; `year`/`vendor`/
+ * `org`/`expenseCategory` are EXACT-match filters; `sortBy`/`sortOrder` sort;
+ * `limit`/`offset` page. Returns { portal, rows:[…] } + honest _meta
+ * (totalAvailable = the API's real count, NOT a page length).
+ */
+export async function openCheckbookSearch(args: OpenCheckbookSearchArgs): Promise<MetaBundle> {
+  const portal = PORTAL_BY_KEY.get(args.portal);
+  if (!portal) {
+    throw new ToolErrorCarrier({
+      kind: "invalid_input",
+      message: `Unknown Open-Checkbook portal ${JSON.stringify(args.portal)}. Allowed: ${OPEN_CHECKBOOK_PORTALS.map((p) => p.key).join(", ")}.`,
+      retryable: false,
+    });
+  }
+  const limit = args.limit ?? 25;
+  const offset = args.offset ?? 0;
+  // The product paginates by 1-based `page` + `limit`. Map offset→page and snap
+  // offset to the page boundary (disclosing the served offset when it differs).
+  const page = Math.floor(offset / limit) + 1;
+  const servedOffset = (page - 1) * limit;
+
+  const filtersApplied: string[] = ["portal"];
+  const params = new URLSearchParams();
+  params.set("year", args.year && args.year.trim() ? args.year.trim() : "All Years");
+  if (args.year && args.year.trim()) filtersApplied.push("year");
+  if (args.vendor && args.vendor.trim()) { params.set("vendor", args.vendor.trim()); filtersApplied.push("vendor"); }
+  if (args.org && args.org.trim()) { params.set("org1", args.org.trim()); filtersApplied.push("org"); }
+  if (args.expenseCategory && args.expenseCategory.trim()) { params.set("expense_category", args.expenseCategory.trim()); filtersApplied.push("expenseCategory"); }
+  if (args.sortBy && args.sortBy.trim()) {
+    const sf = args.sortBy.trim();
+    if (!SORT_FIELDS.has(sf)) {
+      throw new ToolErrorCarrier({
+        kind: "invalid_input",
+        message: `Invalid sortBy ${JSON.stringify(sf)}. Allowed: ${[...SORT_FIELDS].join(", ")}.`,
+        retryable: false,
+      });
+    }
+    params.set("sort_field", sf);
+    params.set("sort_order", args.sortOrder === "asc" ? "asc" : "desc");
+    filtersApplied.push("sortBy");
+  }
+  params.set("page", String(page));
+  params.set("limit", String(limit));
+
+  const body = await getCheckbook(portal, params);
+  const b = (body ?? {}) as { data?: unknown; count?: unknown; total_amount?: unknown };
+  // P4: the shape MUST be { data:[…], count:<number> }.
+  if (!Array.isArray(b.data)) throw driftError(`open-checkbook:${portal.key}`, "Open-Checkbook shape drift — response.data must be an array.");
+  const totalAvailable = num(b.count);
+  if (totalAvailable === null) throw driftError(`open-checkbook:${portal.key}`, "Open-Checkbook shape drift — response.count must be a number.");
+
+  const rows = (b.data as unknown[]).map((r) => {
+    const o = (r ?? {}) as Record<string, unknown>;
+    return {
+      vendor: str(o.vendor),
+      amount: num(o.amount), // P3: a real $0 is 0; absent ⇒ null (never a fabricated 0)
+      payment_date: str(o.payment_date),
+      org1: str(o.org1),
+      expense_category: str(o.expense_category),
+      description: str(o.description),
+      fund: str(o.fund),
+      invoice: str(o.custom_checkbook_field7),
+      payment_id: str(o.payment_id),
+    };
+  });
+  const returned = rows.length;
+  const hasMore = servedOffset + returned < totalAvailable;
+  const nextOffset = hasMore ? servedOffset + returned : null;
+
+  const notes: string[] = [
+    `Source: ${portal.label} (Socrata Open Expenditures app-proxy /api/checkbook_data.json, keyless). ${portal.note}`,
+    "totalAvailable = the API's exact match count (matches the product's totals.json), NOT the page length.",
+    "Filters (year/vendor/org/expenseCategory) are EXACT-match — a partial/misspelled value returns an honest count:0, not an error. amount is number|null (a real $0 is 0, an absent value is null, never a fabricated 0).",
+    "COVERAGE: only the ~3 most-recent fiscal years are exposed by this product — this is NOT the state's full payment history.",
+    "FRESHNESS is set by the publisher: the portal says it refreshes each payment cycle, but refreshes can lag by weeks. To check recency, sort by payment_date (sortBy='payment_date', sortOrder='desc') and read the newest date.",
+  ];
+  if (servedOffset !== offset)
+    notes.push(`offset ${offset} was snapped to ${servedOffset} (the product paginates by fixed page×limit); pass an offset that is a multiple of limit to avoid snapping.`);
+
+  return withMeta(
+    { portal: portal.key, rows },
+    {
+      source: `${portal.host} via Socrata Open Expenditures (keyless app-proxy)`,
+      keylessMode: true,
+      returned,
+      totalAvailable,
+      filtersApplied,
+      filtersDropped: [],
+      fieldsUnavailable: [],
+      pagination: { offset: servedOffset, limit, hasMore, nextOffset },
+      notes,
+    } satisfies Partial<ResponseMeta>,
+  );
+}

package/src/openfda-device.ts

@@ -53,6 +53,10 @@ import {
   readOpenfdaError,
   luceneQuote,
   openfdaApiKey,
+  openfdaPageMeta,
+  openfdaEmptyTotal,
+  OPENFDA_CEILING_NOTE,
+  OPENFDA_OVERSKIP_NOTE,
 } from "./openfda.js";
 
 // state filter charclass (a 2-letter US state/territory postal code).
@@ -282,10 +286,10 @@ export async function deviceClearances(
   const rawTotal = metaResults.total;
   const totalAvailable =
     typeof rawTotal === "number" && Number.isFinite(rawTotal) ? rawTotal : null;
-  const hasMore = totalAvailable !== null && skip + returned < totalAvailable;
-  const nextOffset = hasMore ? skip + returned : null;
+  const { hasMore, nextOffset, ceilingHit } = openfdaPageMeta(skip, returned, totalAvailable);
 
   const notes: string[] = [NOT_DETERMINATION_NOTE, keyNote(key !== undefined)];
+  if (ceilingHit) notes.push(OPENFDA_CEILING_NOTE);
   if (filtersApplied.length === 0) notes.push(NO_FILTER_NOTE);
 
   return withMeta(
@@ -327,13 +331,15 @@ function emptyResult(
       })`,
       keylessMode: true,
       returned: 0,
-      totalAvailable: 0,
+      totalAvailable: openfdaEmptyTotal(skip),
       filtersApplied,
       filtersDropped: [],
       fieldsUnavailable: [],
       pagination: { offset: skip, limit, hasMore: false, nextOffset: null },
       notes: [
-        "No 510(k) device clearances matched this query (openFDA returned HTTP 404 NOT_FOUND — the source's honest no-match). This is an exact empty, not an error.",
+        skip === 0
+          ? "No 510(k) device clearances matched this query (openFDA returned HTTP 404 NOT_FOUND at skip 0 — the source's honest no-match). This is an exact empty (total 0), not an error."
+          : OPENFDA_OVERSKIP_NOTE,
         NOT_DETERMINATION_NOTE,
         keyNote(hasKey),
       ],

package/src/openfda-drugsfda.ts

@@ -37,6 +37,10 @@ import {
   readOpenfdaError,
   luceneQuote,
   openfdaApiKey,
+  openfdaPageMeta,
+  openfdaEmptyTotal,
+  OPENFDA_CEILING_NOTE,
+  OPENFDA_OVERSKIP_NOTE,
 } from "./openfda.js";
 
 const DEFAULT_LIMIT = 25;
@@ -241,10 +245,10 @@ export async function drugApprovals(args: DrugApprovalsArgs): Promise<MetaBundle
   const rawTotal = metaResults.total;
   const totalAvailable =
     typeof rawTotal === "number" && Number.isFinite(rawTotal) ? rawTotal : null;
-  const hasMore = totalAvailable !== null && skip + returned < totalAvailable;
-  const nextOffset = hasMore ? skip + returned : null;
+  const { hasMore, nextOffset, ceilingHit } = openfdaPageMeta(skip, returned, totalAvailable);
 
   const notes: string[] = [NOT_DETERMINATION_NOTE, MARKETING_NOTE, keyNote(key !== undefined)];
+  if (ceilingHit) notes.push(OPENFDA_CEILING_NOTE);
   if (filtersApplied.length === 0) notes.push(NO_FILTER_NOTE);
 
   return withMeta(
@@ -284,13 +288,15 @@ function emptyResult(
       })`,
       keylessMode: true,
       returned: 0,
-      totalAvailable: 0,
+      totalAvailable: openfdaEmptyTotal(skip),
       filtersApplied,
       filtersDropped: [],
       fieldsUnavailable: [],
       pagination: { offset: skip, limit, hasMore: false, nextOffset: null },
       notes: [
-        "No Drugs@FDA applications matched this query (openFDA returned HTTP 404 NOT_FOUND — the source's honest no-match). This is an exact empty, not an error.",
+        skip === 0
+          ? "No Drugs@FDA applications matched this query (openFDA returned HTTP 404 NOT_FOUND at skip 0 — the source's honest no-match). This is an exact empty (total 0), not an error."
+          : OPENFDA_OVERSKIP_NOTE,
         NOT_DETERMINATION_NOTE,
         keyNote(hasKey),
       ],

package/src/openfda.ts

@@ -435,11 +435,10 @@ export async function enforcement(
   const rawTotal = metaResults.total;
   const totalAvailable =
     typeof rawTotal === "number" && Number.isFinite(rawTotal) ? rawTotal : null;
-  const hasMore =
-    totalAvailable !== null && skip + returned < totalAvailable;
-  const nextOffset = hasMore ? skip + returned : null;
+  const { hasMore, nextOffset, ceilingHit } = openfdaPageMeta(skip, returned, totalAvailable);
 
   const notes: string[] = [NOT_DETERMINATION_NOTE, keyNote(key !== undefined)];
+  if (ceilingHit) notes.push(OPENFDA_CEILING_NOTE);
   if (!hasStructuredFilter) notes.push(NO_FILTER_NOTE);
   if (categoryDefaulted) notes.push(CATEGORY_DEFAULT_NOTE);
 
@@ -476,8 +475,11 @@ function emptyResult(
   hasKey: boolean,
   categoryDefaulted: boolean,
 ): MetaBundle {
+  const emptyTotal = openfdaEmptyTotal(skip);
   const notes = [
-    "No recall/enforcement records matched this query (openFDA returned HTTP 404 NOT_FOUND — the source's honest no-match). This is an exact empty, not an error.",
+    skip === 0
+      ? "No recall/enforcement records matched this query (openFDA returned HTTP 404 NOT_FOUND at skip 0 — the source's honest no-match). This is an exact empty (total 0), not an error."
+      : OPENFDA_OVERSKIP_NOTE,
     NOT_DETERMINATION_NOTE,
     keyNote(hasKey),
   ];
@@ -493,7 +495,7 @@ function emptyResult(
       })`,
       keylessMode: true,
       returned: 0,
-      totalAvailable: 0,
+      totalAvailable: emptyTotal,
       filtersApplied,
       filtersDropped: [],
       fieldsUnavailable: [],
@@ -516,3 +518,43 @@ function clampSkip(v: unknown): number {
   const n = Math.floor(v);
   return n < 0 ? 0 : n;
 }
+
+// ─── Shared pagination-honesty helpers (used by all three openFDA modules) ──
+/** openFDA caps `skip` at 25000 (skip>25000 → HTTP 400 "Skip value must 25000 or less").
+ *  The last reachable page is skip=25000, so at most ~25000+limit records are enumerable
+ *  via offset paging — datasets larger than that have an UNREACHABLE tail. */
+export const OPENFDA_MAX_SKIP = 25000;
+
+/** Ceiling- and empty-aware pagination. `hasMore` is false on an empty page (returned:0,
+ *  the fema/nvd guard) OR when the next offset would exceed OPENFDA_MAX_SKIP (advertising a
+ *  skip>25000 nextOffset is a poison cursor — following it 400s). `ceilingHit` is true when
+ *  more rows exist upstream but lie beyond the reachable skip window (must be disclosed). */
+export function openfdaPageMeta(
+  skip: number,
+  returned: number,
+  totalAvailable: number | null,
+): { hasMore: boolean; nextOffset: number | null; ceilingHit: boolean } {
+  const candidateNext = skip + returned;
+  const moreUpstream = totalAvailable !== null && candidateNext < totalAvailable;
+  const withinWindow = candidateNext <= OPENFDA_MAX_SKIP;
+  const hasMore = returned > 0 && moreUpstream && withinWindow;
+  return {
+    hasMore,
+    nextOffset: hasMore ? candidateNext : null,
+    ceilingHit: returned > 0 && moreUpstream && !withinWindow,
+  };
+}
+
+/** The ceiling disclosure (records beyond the skip window are permanently unreachable). */
+export const OPENFDA_CEILING_NOTE =
+  `openFDA caps pagination at skip=${OPENFDA_MAX_SKIP}: records beyond ~${OPENFDA_MAX_SKIP}+limit are NOT reachable via this API, so totalAvailable exceeds the paginable window here and the tail is unreachable. Narrow the query (add filters or a date range) to bring the matching set within the first ${OPENFDA_MAX_SKIP} records.`;
+
+/** Skip-aware total for a 404 NOT_FOUND: openFDA returns the SAME 404 for a genuine
+ *  zero-match (only provable at skip 0) and an OVER-SKIP (skip past the end). So skip===0
+ *  ⇒ a true total of 0; a skip>0 404 is AMBIGUOUS ⇒ totalAvailable unknown (null). */
+export function openfdaEmptyTotal(skip: number): number | null {
+  return skip === 0 ? 0 : null;
+}
+/** Disclosure for a skip>0 404 (ambiguous over-skip vs no-match; total unknown). */
+export const OPENFDA_OVERSKIP_NOTE =
+  "Returned 0 at this offset: the skip is at/past the end of the result set (or a genuine no-match). openFDA returns an IDENTICAL HTTP 404 for both, so the true total is UNKNOWN at a non-zero skip (totalAvailable is null, not 0) — re-query at skip:0 for the exact total. This is NOT a confirmed empty result set.";

package/src/opengov.ts

@@ -238,10 +238,14 @@ export async function searchSolicitations(
     });
   }
 
-  // The API paginates by 0-based `page` + `limit`. Map our offset → page.
-  const page = Math.floor(offset / limit);
-  const servedOffset = page * limit;
-  const payload = JSON.stringify({ governmentCode: code, publicView: true, page, limit });
+  // The OpenGov /project/list `page` param is 1-BASED — `page=0` CLAMPS to page 1
+  // (live-verified 2026-07-20: phoenix offset 0 and offset<limit both returned the
+  // SAME first page → a 0-based send duplicates page 1 and shifts every later page,
+  // silently dropping the tail while totalAvailable still claims the full count).
+  // Map our row `offset` → the 1-based page; servedOffset stays offset-aligned.
+  const apiPage = Math.floor(offset / limit) + 1;
+  const servedOffset = (apiPage - 1) * limit;
+  const payload = JSON.stringify({ governmentCode: code, publicView: true, page: apiPage, limit });
 
   let body: unknown;
   try {