@cliwant/mcp-sam-gov 1.15.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.ja.md +11 -11
  2. package/README.ko.md +11 -11
  3. package/README.md +18 -17
  4. package/dist/bonfire.d.ts +1 -1
  5. package/dist/bonfire.d.ts.map +1 -1
  6. package/dist/bonfire.js +7 -9
  7. package/dist/bonfire.js.map +1 -1
  8. package/dist/ckan.d.ts +6 -4
  9. package/dist/ckan.d.ts.map +1 -1
  10. package/dist/ckan.js +17 -4
  11. package/dist/ckan.js.map +1 -1
  12. package/dist/courtlistener.d.ts +32 -0
  13. package/dist/courtlistener.d.ts.map +1 -1
  14. package/dist/courtlistener.js +180 -0
  15. package/dist/courtlistener.js.map +1 -1
  16. package/dist/data-map.d.ts.map +1 -1
  17. package/dist/data-map.js +56 -1
  18. package/dist/data-map.js.map +1 -1
  19. package/dist/fpds.d.ts.map +1 -1
  20. package/dist/fpds.js +3 -2
  21. package/dist/fpds.js.map +1 -1
  22. package/dist/gao.d.ts.map +1 -1
  23. package/dist/gao.js +17 -6
  24. package/dist/gao.js.map +1 -1
  25. package/dist/gsa-perdiem.d.ts +5 -0
  26. package/dist/gsa-perdiem.d.ts.map +1 -1
  27. package/dist/gsa-perdiem.js +6 -1
  28. package/dist/gsa-perdiem.js.map +1 -1
  29. package/dist/keys.d.ts.map +1 -1
  30. package/dist/keys.js +1 -0
  31. package/dist/keys.js.map +1 -1
  32. package/dist/open-checkbook.d.ts +8 -0
  33. package/dist/open-checkbook.d.ts.map +1 -1
  34. package/dist/open-checkbook.js +9 -1
  35. package/dist/open-checkbook.js.map +1 -1
  36. package/dist/pricing.d.ts +1 -0
  37. package/dist/pricing.d.ts.map +1 -1
  38. package/dist/pricing.js +135 -50
  39. package/dist/pricing.js.map +1 -1
  40. package/dist/server.d.ts.map +1 -1
  41. package/dist/server.js +93 -19
  42. package/dist/server.js.map +1 -1
  43. package/dist/socrata.d.ts +16 -1
  44. package/dist/socrata.d.ts.map +1 -1
  45. package/dist/socrata.js +31 -1
  46. package/dist/socrata.js.map +1 -1
  47. package/dist/toolsets.d.ts.map +1 -1
  48. package/dist/toolsets.js +1 -0
  49. package/dist/toolsets.js.map +1 -1
  50. package/package.json +1 -1
  51. package/src/bonfire.ts +7 -9
  52. package/src/ckan.ts +17 -4
  53. package/src/courtlistener.ts +244 -1
  54. package/src/data-map.ts +58 -1
  55. package/src/fpds.ts +3 -2
  56. package/src/gao.ts +20 -6
  57. package/src/gsa-perdiem.ts +12 -1
  58. package/src/keys.ts +1 -0
  59. package/src/open-checkbook.ts +26 -3
  60. package/src/pricing.ts +142 -48
  61. package/src/server.ts +101 -19
  62. package/src/socrata.ts +37 -1
  63. package/src/toolsets.ts +1 -0
package/src/data-map.ts CHANGED
@@ -41,6 +41,36 @@ export interface DataMapEntry {
41
41
  */
42
42
  export const DATA_MAP_ENTRIES: DataMapEntry[] = [
43
43
  // ── State-level ──────────────────────────────────────────────────────────
44
+ {
45
+ state: "CA",
46
+ jurisdiction: "California",
47
+ dataLabel: "DGS Purchase Order Data 2012–2015",
48
+ tool: "ckan_query",
49
+ keyArgs: "host=data.ca.gov, resourceId=bb82edc5-9c78-44e2-8947-68ece26197c5",
50
+ rows: 344504,
51
+ approximate: false,
52
+ notNote: "live Cal eProcure portal (WAF-403); FY2012–2015 only — no post-2015 rows",
53
+ },
54
+ {
55
+ state: "CA",
56
+ jurisdiction: "California",
57
+ dataLabel: "DGS-Approved Non-Competitive Bids",
58
+ tool: "ckan_query",
59
+ keyArgs: "host=data.ca.gov, resourceId=14932789-485b-481b-910a-dafb40d3471c",
60
+ rows: 480,
61
+ approximate: false,
62
+ notNote: "open competitive solicitations; sole-source/non-competitive award register",
63
+ },
64
+ {
65
+ state: "OK",
66
+ jurisdiction: "Oklahoma",
67
+ dataLabel: "Vendor Payments FY2019 Q1 (OMES)",
68
+ tool: "ckan_query",
69
+ keyArgs: "host=data.ok.gov, resourceId=cc443616-15eb-4a1f-8d87-93e5711ac43c",
70
+ rows: 286185,
71
+ approximate: false,
72
+ notNote: "bids or awards; vendor PAYMENTS — per-quarter resources, one fiscal year = 4 calls",
73
+ },
44
74
  {
45
75
  state: "VA",
46
76
  jurisdiction: "Virginia",
@@ -133,6 +163,18 @@ export const DATA_MAP_ENTRIES: DataMapEntry[] = [
133
163
  approximate: true,
134
164
  notNote: "an award register; only ~3 most-recent FYs, exact-match filters",
135
165
  },
166
+ // ── Alaska — verified 2026-09-21 ─────────────────────────────────────────
167
+ {
168
+ state: "AK",
169
+ jurisdiction: "Alaska",
170
+ dataLabel: "Open Checkbook vendor payments (FY2026 ONLY)",
171
+ tool: "open_checkbook_search",
172
+ keyArgs: "portal=ak, year=2026",
173
+ rows: 41751,
174
+ approximate: false,
175
+ notNote:
176
+ "historical FY coverage — ONLY FY2026 is published by this portal. FY2019–FY2025 return count:0, meaning 'not published', NOT 'no spending'. Probed years 2019–2026 and no-year on 2026-09-21: 2019–2025 = 0 rows each; 2026 = 41,751 rows ($1,179,091,896.12); no-year = 0 rows. Always pass year='2026' to get results.",
177
+ },
136
178
  {
137
179
  state: "IL",
138
180
  jurisdiction: "Illinois",
@@ -141,7 +183,7 @@ export const DATA_MAP_ENTRIES: DataMapEntry[] = [
141
183
  keyArgs: "domain=data.illinois.gov, datasetId=6rb8-ntpm",
142
184
  rows: 48,
143
185
  approximate: false,
144
- notNote: "a comprehensive solicitation feed; only ~48 upcoming CDB capital bids",
186
+ notNote: "a comprehensive solicitation feed, and NOT currently open bids — these are ~48 CDB capital solicitations the publisher describes as \"anticipated for a future date, but have not been posted yet\" (estimated_bid_date field)",
145
187
  },
146
188
  {
147
189
  state: "IL",
@@ -270,6 +312,18 @@ export const DATA_MAP_ENTRIES: DataMapEntry[] = [
270
312
  notNote:
271
313
  "Colorado STATE procurement — this is City of Denver checkbook data hosted on the Colorado state portal (data.colorado.gov). An agent that reads the domain as the jurisdiction gets this wrong. Updated 2026-09-20.",
272
314
  },
315
+ // ── Fulton County GA (second portal) — verified 2026-09-21 ───────────────
316
+ {
317
+ state: "GA",
318
+ jurisdiction: "Fulton County GA",
319
+ dataLabel: "Vendor Payments 2014–present (sharefulton portal)",
320
+ tool: "socrata_query",
321
+ keyArgs: "domain=sharefulton.fultoncountyga.gov, datasetId=kp4p-scak",
322
+ rows: 226797,
323
+ approximate: false,
324
+ notNote:
325
+ "data.fultoncountyga.gov (a DIFFERENT host that is also allowlisted — both are official Fulton County portals). Fields: fiscal_year, fy_period, disb_date, dept, department_name, unit, unit_name, object, object_name, fund, fund_name, amount, vendor_code, vendor_legal_name. Updated 2026-09-14, attribution: 'Fulton County Government (GA)', 226,797 rows spanning 2014-01-01 to present.",
326
+ },
273
327
  ];
274
328
 
275
329
  /**
@@ -325,5 +379,8 @@ ${renderStateTableMarkdown()}
325
379
  - Login-gated or WAF-blocked live-bid portals: CA (Cal eProcure), TX ESBD/TxSmartBuy non-TxDOT (TxDOT lettings ARE available via qh8x-rm8r above), OH, NC, MI, and Periscope-based portals for IL/MA/NJ live bids.
326
380
  - Portal exists but carries no procurement datasets: PA (data.pa.gov is live; scoped catalog has 0 bid/vendor/procurement datasets matching). Michigan (data.michigan.gov) was checked 2026-09-21 and carries only NIGP commodity code reference tables (w3u3-uptp 9,333 rows; jv5q-yp8x 235 rows) — not procurement transactions.
327
381
  - No state-level open-data portal: FL (data.fl.gov NXDOMAIN), GA (data.georgia.gov NXDOMAIN). These states have no keyless state procurement data source — this is a measured absence, not a connectivity block.
382
+
383
+ **City-level measured absences (portals exist but carry no procurement data):**
384
+ - New Orleans LA (rank 46, 383,997 people): data.nola.gov exists but has NO procurement dataset. Full scan across 9 terms (procurement, solicitation, bid opportunities, vendor payments, purchase orders, contracts awarded, checkbook, supplier diversity, expenditure) on 2026-09-21 found only a 626-row DBE directory last updated 2019-11-11. Largest datasets are permits (347,839) and traffic citations (4M+) — not procurement. Use opengov_search_solicitations or bonfire_search_opportunities for New Orleans live bids.
328
385
  `;
329
386
  }
package/src/fpds.ts CHANGED
@@ -48,9 +48,10 @@ const FPDS_HOST = "www.fpds.gov";
48
48
  const FPDS_PATH = "/ezsearch/FEEDS/ATOM";
49
49
  const FPDS_ORIGIN_PATH = `https://${FPDS_HOST}${FPDS_PATH}`;
50
50
  const FPDS_LABEL = "www.fpds.gov";
51
- // WAF-friendly browser-ish UA (mirrors gao.ts convention).
51
+ // Identify this client honestly — same string used by every other module in the
52
+ // project. Live-verified 2026-09-22: FPDS returns 200 to this UA and with no UA.
52
53
  const FPDS_UA =
53
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
54
+ "Mozilla/5.0 (compatible; @cliwant/mcp-sam-gov; +https://github.com/cliwant/mcp-sam-gov)";
54
55
 
55
56
  const PAGE_SIZE = 10;
56
57
  /** Hard cap on entries sliced per feed (page size is 10; anything past a small
package/src/gao.ts CHANGED
@@ -42,11 +42,12 @@ import { withMeta } from "./meta.js";
42
42
 
43
43
  // ─── Shared HTTP ─────────────────────────────────────────────────
44
44
 
45
- // GAO's edge (Cloudflare/WAF) will 403 a bare client — always send a realistic
46
- // browser User-Agent. Mirror the shape the rest of the server uses for GAO-ish
47
- // public HTML/RSS scraping so behavior is consistent.
45
+ // Identify this client honestly — same string used by every other module in the
46
+ // project. Live-verified 2026-09-22: GAO's RSS feed returns 200 to this UA, to
47
+ // the formerly-used fake Chrome UA, and even with no UA at all. GAO per-decision
48
+ // pages 403 from Akamai regardless of UA, so spoofing a browser gains nothing.
48
49
  const GAO_UA =
49
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
50
+ "Mozilla/5.0 (compatible; @cliwant/mcp-sam-gov; +https://github.com/cliwant/mcp-sam-gov)";
50
51
 
51
52
  const RSS_URL = "https://www.gao.gov/rss/reportslegal.xml";
52
53
  const PRODUCT_BASE = "https://www.gao.gov/products/";
@@ -58,9 +59,9 @@ const SOURCE = "gao.gov Legal Products RSS + decision pages (keyless)";
58
59
  * in `data` and inside `_meta.notes` so no consumer can miss the scope boundary.
59
60
  */
60
61
  const ACCESS_NOTE =
61
- "Keyless GAO access covers only RECENT decisions from the public Legal-Products RSS feed (a rolling ~25-item window). GAO's faceted historical protest search (by protester/agency/outcome/date across all years) is WAF-blocked to automated clients and available only via a paid third-party API. Do NOT treat these results as the complete protest history.";
62
+ "Keyless GAO access covers only RECENT decisions from the public Legal-Products RSS feed (a rolling ~25-item window). GAO's faceted historical protest search (by protester/agency/outcome/date across all years) is WAF-blocked to automated clients — keyless programmatic access is unavailable (a human can search at https://www.gao.gov/search). Do NOT treat these results as the complete protest history.";
62
63
 
63
- // Thin LOCAL wrapper (ADR-0013) that injects GAO's WAF-friendly UA + RSS Accept
64
+ // Thin LOCAL wrapper (ADR-0013) that injects the project's honest UA + RSS Accept
64
65
  // for the tool's two call sites, then delegates to the shared `getText` port
65
66
  // (retry defaults true → fetchWithRetry, byte-identical to the former
66
67
  // hand-rolled fetcher). `timeoutMs` is preserved as a param default (never
@@ -710,6 +711,19 @@ export async function gaoProtestLookup(args: {
710
711
  );
711
712
  }
712
713
 
714
+ // When a protester/agency/solicitationNumber filter was given and the window
715
+ // contains no matching decisions, give the caller a precise next step: GAO's
716
+ // own search pre-filtered to bid-protest decisions with the term filled in.
717
+ // (This is NOT a reading list — it is a single direct link to the full
718
+ // historical database that a human can open immediately.)
719
+ const searchTerm = (args.protester ?? args.agency ?? args.solicitationNumber)?.trim();
720
+ if (decisions.length === 0 && filtersApplied.length > 0 && searchTerm) {
721
+ const encoded = encodeURIComponent(searchTerm);
722
+ notes.push(
723
+ `"${searchTerm}" does not appear in the current ~25-decision window. GAO's full protest database is searchable at https://www.gao.gov/search?f%5B0%5D=ctype_search%3ABid%20Protest%20Decision&keyword=${encoded} — each result shows parties, agency, date and outcome. Keyless programmatic access to the historical archive is unavailable; the link above goes directly to the relevant results.`,
724
+ );
725
+ }
726
+
713
727
  return withMeta(
714
728
  { accessNote: ACCESS_NOTE, decisions },
715
729
  {
@@ -93,6 +93,10 @@ const STANDARD_RATE_NOTE =
93
93
  "standardRate:true means this location falls under the CONUS STANDARD rate (not an individually-set non-standard rate). standardRate/isOconus are booleans coerced from the API's string 'true'/'false'.";
94
94
  const NO_PAGINATION_NOTE =
95
95
  "The per-diem API returns the COMPLETE rate set for the lookup (no pagination); totalAvailable equals the number of rows returned.";
96
+ const FISCAL_YEAR_NOTE =
97
+ "fiscalYear (= year) is the U.S. FEDERAL FISCAL YEAR (Oct 1–Sep 30). " +
98
+ "Example: October 2026 falls in FY2027 (FY runs Oct 2026–Sep 2027); September 2026 falls in FY2026. " +
99
+ "To look up October 2026 rates, pass year='2027'. To look up September 2026 rates, pass year='2026'.";
96
100
 
97
101
  // ─── STRING-boolean coercion (null-never-fabricate) ───────────────
98
102
  /** Coerce the API's string 'true'/'false' → a real boolean; anything else ⇒ null. */
@@ -119,6 +123,11 @@ export type PerdiemRate = {
119
123
  state: string | null;
120
124
  zip: string | null;
121
125
  year: number | null;
126
+ /** Explicit U.S. federal fiscal year label (Oct 1–Sep 30). Same numeric value as
127
+ * `year` (the GSA API's year field IS the FY number). Surfaced as a separate named
128
+ * field so callers never confuse it with a calendar year. E.g. fiscalYear:2027 means
129
+ * rates run Oct 2026–Sep 2027; October 2026 travel uses FY2027 rates. */
130
+ fiscalYear: number | null;
122
131
  isOconus: boolean | null; // OCONUS (outside-CONUS) flag — coerced from string boolean
123
132
  standardRate: boolean | null; // CONUS standard-rate flag — coerced from string boolean
124
133
  mealsUsd: number | null; // M&IE ceiling ($) — null-never-0
@@ -350,6 +359,8 @@ export async function perdiemRates(
350
359
  state: gState,
351
360
  zip: str(r.zip),
352
361
  year: gYear,
362
+ fiscalYear: gYear, // same numeric value as year; surfaced separately so callers
363
+ // never confuse it with a calendar year (GSA year IS the FY)
353
364
  isOconus: gOconus,
354
365
  standardRate: strBool(r.standardRate),
355
366
  mealsUsd: num(r.meals),
@@ -359,7 +370,7 @@ export async function perdiemRates(
359
370
  }
360
371
 
361
372
  const returned = rows.length;
362
- const notes: string[] = [RATE_MEANING_NOTE, STANDARD_RATE_NOTE, NO_PAGINATION_NOTE];
373
+ const notes: string[] = [RATE_MEANING_NOTE, STANDARD_RATE_NOTE, NO_PAGINATION_NOTE, FISCAL_YEAR_NOTE];
363
374
  if (yearWasDefaulted) {
364
375
  notes.push(
365
376
  `No \`year\` was supplied, so it defaulted to the CURRENT U.S. federal fiscal year (FY${year}). GSA per-diem ceilings are set per fiscal year (Oct 1–Sep 30); pass an explicit \`year\` for a prior or upcoming FY.`,
package/src/keys.ts CHANGED
@@ -159,6 +159,7 @@ export const KEY_REGISTRY: readonly KeyRegistryEntry[] = [
159
159
  envVar: "COURTLISTENER_API_TOKEN",
160
160
  sources: [
161
161
  "CourtListener federal court opinions (courtlistener_search_opinions)",
162
+ "CourtListener RECAP dockets (courtlistener_search_dockets)",
162
163
  ],
163
164
  required: false,
164
165
  signupUrl: "https://www.courtlistener.com/help/api/rest/",
@@ -41,14 +41,37 @@ import { withMeta, type MetaBundle, type ResponseMeta } from "./meta.js";
41
41
 
42
42
  export { num };
43
43
 
44
- // ─── Curated portal allowlist (SSRF core) — LIVE-VERIFIED 2026-07-24 ──
45
- export type OpenCheckbookPortal = { key: string; host: string; label: string; note: string };
44
+ // ─── Curated portal allowlist (SSRF core) — LIVE-VERIFIED 2026-07-24/2026-09-21 ──
45
+ export type OpenCheckbookPortal = {
46
+ key: string;
47
+ host: string;
48
+ label: string;
49
+ note: string;
50
+ /**
51
+ * What fiscal-year span this portal actually publishes, as MEASURED (not assumed).
52
+ * Portals differ: SD exposes roughly the three most recent FYs, AK exposes exactly
53
+ * one. Emitting a single hardcoded coverage sentence for both made the envelope
54
+ * contradict itself — the generic line claimed "~3 most-recent fiscal years" while
55
+ * the AK portal note in the same response said FY2026 only.
56
+ */
57
+ coverageNote: string;
58
+ };
46
59
  export const OPEN_CHECKBOOK_PORTALS: readonly OpenCheckbookPortal[] = [
47
60
  {
48
61
  key: "sd",
49
62
  host: "southdakota.spending.socrata.com",
50
63
  label: "South Dakota — Open Checkbook",
51
64
  note: "State of South Dakota vendor-payment checkbook (row fields: expense_category, description, fund, payment_date, vendor, org1=department, amount, custom_checkbook_field7=invoice ref, payment_id). ~740,980 rows / ~$8.41B across the ~3 most-recent fiscal years (NOT full history). The underlying Socrata SODA dataset is login-gated; this public app-proxy is the keyless door.",
65
+ coverageNote:
66
+ "COVERAGE: this portal exposes only the ~3 most-recent fiscal years — NOT South Dakota's full payment history.",
67
+ },
68
+ {
69
+ key: "ak",
70
+ host: "checkbook.alaska.gov",
71
+ label: "Alaska — Open Checkbook",
72
+ note: "State of Alaska vendor-payment checkbook (official .gov CNAME to alaska-state.spending.socrata.com). Verified live 2026-09-21: 41,751 rows / $1,179,091,896.12 for FY2026. ★ FY COVERAGE CAVEAT: ONLY FY2026 is published by this portal — years 2019–2025 and no-year all return count:0 / empty data. A count:0 result for a non-2026 year means 'this portal does not publish that year', NOT 'Alaska spent nothing'. Do NOT interpret zero results for FY2019–FY2025 as an absence of spending.",
73
+ coverageNote:
74
+ "COVERAGE: this portal publishes ONLY FY2026 (measured 2026-09-21: FY2019-FY2025 and an omitted year each return count:0). A count:0 for any other year means the portal does not publish that year — it does NOT mean Alaska made no payments.",
52
75
  },
53
76
  ] as const;
54
77
 
@@ -160,7 +183,7 @@ export async function openCheckbookSearch(args: OpenCheckbookSearchArgs): Promis
160
183
  `Source: ${portal.label} (Socrata Open Expenditures app-proxy /api/checkbook_data.json, keyless). ${portal.note}`,
161
184
  "totalAvailable = the API's exact match count (matches the product's totals.json), NOT the page length.",
162
185
  "Filters (year/vendor/org/expenseCategory) are EXACT-match — a partial/misspelled value returns an honest count:0, not an error. amount is number|null (a real $0 is 0, an absent value is null, never a fabricated 0).",
163
- "COVERAGE: only the ~3 most-recent fiscal years are exposed by this product — this is NOT the state's full payment history.",
186
+ portal.coverageNote,
164
187
  "FRESHNESS is set by the publisher: the portal says it refreshes each payment cycle, but refreshes can lag by weeks. To check recency, sort by payment_date (sortBy='payment_date', sortOrder='desc') and read the newest date.",
165
188
  ];
166
189
  if (servedOffset !== offset)
package/src/pricing.ts CHANGED
@@ -164,10 +164,26 @@ function epochToIso(v: number | string | undefined): string | null {
164
164
  return null;
165
165
  }
166
166
 
167
+ // Maximum pages to scan when a client-side filter (county / constructionType)
168
+ // requires a full-state sweep. 10 pages × 50 rows = 500 WDs — well above any
169
+ // real state's active DBA stock (IL has 70 as of 2026-09-22).
170
+ const WD_SCAN_PAGE_CAP = 10;
171
+ const WD_SCAN_SIZE = 50; // max the SGS API accepts per request
172
+
173
+ /** Extract constructionTypes as a lowercase string array (handles string[] or unknown). */
174
+ function resultConstructionTypes(r: SgsResult): string[] {
175
+ const ct = r.constructionTypes;
176
+ if (!ct) return [];
177
+ if (Array.isArray(ct)) return ct.map((v) => String(v).toLowerCase());
178
+ if (typeof ct === "string") return [ct.toLowerCase()];
179
+ return [];
180
+ }
181
+
167
182
  export async function searchWageDeterminations(args: {
168
183
  coverage: string;
169
184
  state?: string;
170
185
  county?: string;
186
+ constructionType?: string;
171
187
  query?: string;
172
188
  activeOnly?: boolean;
173
189
  standardOnly?: boolean;
@@ -181,35 +197,47 @@ export async function searchWageDeterminations(args: {
181
197
  const page = Math.max(0, Math.floor(args.page ?? 0));
182
198
  const stateFilter = args.state?.trim().toUpperCase() || undefined;
183
199
  const countyFilter = args.county?.trim().toLowerCase() || undefined;
200
+ const ctFilter = args.constructionType?.trim().toLowerCase() || undefined;
201
+
202
+ // When county or constructionType is given, we must scan ALL pages for the
203
+ // state to avoid missing matching WDs (e.g. IL has 70 DBA WDs — the target
204
+ // Cook County Building WD is on page 1). We use max size=50 per fetch and
205
+ // cap at WD_SCAN_PAGE_CAP pages to keep the request count bounded.
206
+ const needsFullScan = Boolean(countyFilter || ctFilter);
184
207
 
185
- const params = new URLSearchParams({
186
- index,
187
- size: String(limit),
188
- page: String(page),
189
- mode: "search",
190
- sort: "-modifiedDate",
191
- });
192
- if (activeOnly) params.set("is_active", "true");
193
- if (standardOnly) params.set("is_standard", "true");
194
- // `q` matches WD number/title ONLY (NOT occupation) — verified q=guard→0.
195
- if (args.query) params.set("q", args.query);
196
- // `state` IS honored server-side (LIVE-VERIFIED 2026-07-03: index=sca&state=VA
197
- // → 30 vs 1028 unfiltered, and every returned WD's location contains VA;
198
- // state=ZZ → 0). It wants the 2-letter USPS code (a full name like "Virginia"
199
- // → 0). This CORRECTS the earlier brief which assumed state was ignored.
200
208
  const filtersApplied: string[] = [`coverage(${index})`];
201
209
  const filtersDropped: string[] = [];
202
210
  const notes: string[] = [];
203
211
  if (activeOnly) filtersApplied.push("activeOnly");
204
212
  if (standardOnly) filtersApplied.push("standardOnly");
205
213
  if (args.query) filtersApplied.push("query(WD number/title only)");
214
+
215
+ // Build the base URLSearchParams that are shared across every page fetch.
216
+ function buildParams(pg: number, sz: number): URLSearchParams {
217
+ const p = new URLSearchParams({
218
+ index,
219
+ size: String(sz),
220
+ page: String(pg),
221
+ mode: "search",
222
+ sort: "-modifiedDate",
223
+ });
224
+ if (activeOnly) p.set("is_active", "true");
225
+ if (standardOnly) p.set("is_standard", "true");
226
+ // `q` matches WD number/title ONLY (NOT occupation) — verified q=guard→0.
227
+ if (args.query) p.set("q", args.query);
228
+ return p;
229
+ }
230
+
231
+ // `state` IS honored server-side (LIVE-VERIFIED 2026-07-03: index=sca&state=VA
232
+ // → 30 vs 1028 unfiltered, and every returned WD's location contains VA;
233
+ // state=ZZ → 0). It wants the 2-letter USPS code (a full name like "Virginia"
234
+ // → 0). This CORRECTS the earlier brief which assumed state was ignored.
235
+ const needsClientState =
236
+ stateFilter !== undefined && !/^[A-Z]{2}$/.test(stateFilter);
206
237
  if (stateFilter) {
207
- if (/^[A-Z]{2}$/.test(stateFilter)) {
208
- params.set("state", stateFilter);
238
+ if (!needsClientState) {
209
239
  filtersApplied.push("state(server-side)");
210
240
  } else {
211
- // Not a 2-letter code → the server would return 0; apply client-side
212
- // instead so a full name still works, and disclose it.
213
241
  filtersDropped.push("state(server-side; not a 2-letter code)");
214
242
  notes.push(
215
243
  `The state value '${args.state}' is not a 2-letter USPS code; the SGS 'state' param only matches 2-letter codes (a full name returns 0), so it was applied CLIENT-SIDE over the fetched page instead. Pass a 2-letter code (e.g. 'VA') for a precise server-side filter.`,
@@ -217,30 +245,75 @@ export async function searchWageDeterminations(args: {
217
245
  }
218
246
  }
219
247
 
220
- const url = `${SGS_BASE}?${params.toString()}`;
221
- const json = await getJson<SgsSearchResp>(url, SAM_HAL_HEADERS, `sam:sgs:${index}`);
222
- const rawResults = json._embedded?.results ?? [];
223
- const serverTotal = json.page?.totalElements ?? null;
248
+ // ── Fetch: one page (no client filters) OR full-state scan ──────
249
+ let allRaw: SgsResult[];
250
+ let serverTotal: number | null;
251
+ let scanCapHit = false;
252
+
253
+ if (!needsFullScan) {
254
+ // Original single-page path (no county/constructionType filter).
255
+ const params = buildParams(page, limit);
256
+ if (!needsClientState && stateFilter) params.set("state", stateFilter);
257
+ const url = `${SGS_BASE}?${params.toString()}`;
258
+ const json = await getJson<SgsSearchResp>(url, SAM_HAL_HEADERS, `sam:sgs:${index}`);
259
+ allRaw = json._embedded?.results ?? [];
260
+ serverTotal = json.page?.totalElements ?? null;
261
+ } else {
262
+ // Full-state scan: fetch all pages with size=50 until exhausted or cap.
263
+ const scanParams = buildParams(0, WD_SCAN_SIZE);
264
+ if (!needsClientState && stateFilter) scanParams.set("state", stateFilter);
265
+ const firstUrl = `${SGS_BASE}?${scanParams.toString()}`;
266
+ const firstJson = await getJson<SgsSearchResp>(firstUrl, SAM_HAL_HEADERS, `sam:sgs:${index}`);
267
+ allRaw = firstJson._embedded?.results ?? [];
268
+ serverTotal = firstJson.page?.totalElements ?? null;
269
+ const totalPages = firstJson.page?.totalPages ?? 1;
270
+
271
+ const pagesToFetch = Math.min(totalPages, WD_SCAN_PAGE_CAP);
272
+ if (totalPages > WD_SCAN_PAGE_CAP) scanCapHit = true;
273
+
274
+ for (let pg = 1; pg < pagesToFetch; pg++) {
275
+ const p = buildParams(pg, WD_SCAN_SIZE);
276
+ if (!needsClientState && stateFilter) p.set("state", stateFilter);
277
+ const pgJson = await getJson<SgsSearchResp>(
278
+ `${SGS_BASE}?${p.toString()}`,
279
+ SAM_HAL_HEADERS,
280
+ `sam:sgs:${index}`,
281
+ );
282
+ allRaw = allRaw.concat(pgJson._embedded?.results ?? []);
283
+ }
284
+ }
224
285
 
225
- // Client-side filtering: county is NOT a documented SGS server param, so it
226
- // is applied here over the fetched page. A non-2-letter state also lands here.
227
- const needsClientState = filtersDropped.some((f) => f.startsWith("state"));
228
- let filtered = rawResults;
286
+ // ── Client-side filtering ────────────────────────────────────────
287
+ let filtered = allRaw;
229
288
  if (needsClientState && stateFilter) {
230
289
  filtered = filtered.filter((r) =>
231
- resultStateCodes(r).some((c) => c === stateFilter || r.location?.state?.name?.toUpperCase() === stateFilter),
290
+ resultStateCodes(r).some(
291
+ (c) => c === stateFilter || r.location?.state?.name?.toUpperCase() === stateFilter,
292
+ ),
232
293
  );
233
294
  }
234
295
  if (countyFilter) {
235
- filtersApplied.push("county(client-side)");
296
+ filtersApplied.push("county(client-side, full-state scan)");
236
297
  filtered = filtered.filter((r) =>
237
298
  resultCounties(r).some((c) => c.name.toLowerCase().includes(countyFilter)),
238
299
  );
239
- notes.push(
240
- "County is filtered CLIENT-SIDE over the fetched page only (the SGS API has no county filter). A county filter combined with a small limit can miss WDs on later pages — raise `limit` or narrow by `state` (server-side) to be sure.",
300
+ }
301
+ if (ctFilter) {
302
+ filtersApplied.push(`constructionType(client-side, value=${args.constructionType})`);
303
+ filtered = filtered.filter((r) =>
304
+ resultConstructionTypes(r).includes(ctFilter),
241
305
  );
242
306
  }
243
307
 
308
+ // ── Specificity ranking: single-county WDs before multi-county ──
309
+ if (countyFilter) {
310
+ filtered.sort((a, b) => {
311
+ const aCount = resultCounties(a).length;
312
+ const bCount = resultCounties(b).length;
313
+ return aCount - bCount; // 1-county first, then 2-county, etc.
314
+ });
315
+ }
316
+
244
317
  const determinations = filtered.map((r) => {
245
318
  const coverageCode = r.type?.code ?? (index === "sca" ? "SCA" : "DBA");
246
319
  return {
@@ -273,27 +346,48 @@ export async function searchWageDeterminations(args: {
273
346
  "The `q` parameter matches the WD number/title only — it does NOT search by occupation or job title (e.g. q=guard returns 0). To find rates for a specific occupation, open the WD and read its rate table.",
274
347
  );
275
348
 
276
- // totalAvailable: the server total is REAL for the coverage/active/standard/
277
- // query/state filters (state is server-side). But when we additionally filter
278
- // client-side (county, or a non-code state), the returned page count no longer
279
- // reflects a full server total for THAT combined filter → null it out and say so.
280
- const clientFiltered = Boolean(countyFilter) || needsClientState;
281
- const totalAvailable = clientFiltered ? null : serverTotal;
349
+ // ── Honest meta ──────────────────────────────────────────────────
350
+ const clientFiltered = Boolean(countyFilter) || Boolean(ctFilter) || needsClientState;
351
+ const scannedCount = allRaw.length;
282
352
  const returned = determinations.length;
283
- const truncated = clientFiltered
284
- ? true // page-bounded client filter → can't prove completeness
285
- : serverTotal !== null && page * limit + returned < serverTotal;
286
353
 
287
- if (clientFiltered) {
354
+ // When we did a full-state scan, totalAvailable is the REAL count of matching
355
+ // WDs across the scanned pages (not null). Only null it when the scan cap was
356
+ // hit (we didn't see all pages) or when there was no client filter.
357
+ let totalAvailable: number | null;
358
+ let truncated: boolean;
359
+
360
+ if (!clientFiltered) {
361
+ // No client filter: server total is accurate for this result set.
362
+ totalAvailable = serverTotal;
363
+ truncated = serverTotal !== null && page * limit + returned < serverTotal;
364
+ } else if (needsFullScan && !scanCapHit) {
365
+ // Full scan completed: real match count is known.
366
+ totalAvailable = returned;
367
+ truncated = false;
368
+ } else {
369
+ // Scan cap hit or no full scan (shouldn't happen but be safe).
370
+ totalAvailable = null;
371
+ truncated = true;
372
+ }
373
+
374
+ if (needsFullScan) {
375
+ const stateLabel = stateFilter ?? "all states";
376
+ const capNote = scanCapHit
377
+ ? ` (scan cap of ${WD_SCAN_PAGE_CAP * WD_SCAN_SIZE} WDs reached — some WDs may have been missed)`
378
+ : "";
288
379
  notes.push(
289
- "totalAvailable is null because a client-side filter (county and/or a non-code state) was applied over just the fetched page — the true match count for the combined filter is unknown. The server-side total for the coverage/state/active filters was " +
290
- (serverTotal ?? "unknown") +
291
- ".",
380
+ `Scanned ${scannedCount} of ${serverTotal ?? scannedCount} ${stateLabel} ${index.toUpperCase()} WDs across all pages${capNote}. ` +
381
+ (countyFilter ? `County filter applied across all scanned WDs. ` : "") +
382
+ (ctFilter ? `constructionType filter (${args.constructionType}) applied. ` : "") +
383
+ (countyFilter
384
+ ? "Single-county WDs ranked first — they are almost always the most specific match for a given locality. "
385
+ : ""),
292
386
  );
293
387
  }
294
388
 
295
389
  return withMeta(
296
- { determinations, coverageIndex: index, page, limit },
390
+ { determinations, coverageIndex: index, page: needsFullScan ? 0 : page, limit: needsFullScan ? returned : limit },
297
391
  {
298
392
  source: WD_SEARCH_SOURCE,
299
393
  keylessMode: true,
@@ -301,9 +395,9 @@ export async function searchWageDeterminations(args: {
301
395
  totalAvailable,
302
396
  truncated,
303
397
  pagination: {
304
- offset: page * limit,
305
- limit,
306
- nextOffset: truncated ? (page + 1) * limit : null,
398
+ offset: 0,
399
+ limit: needsFullScan ? returned : limit,
400
+ nextOffset: truncated && !needsFullScan ? (page + 1) * limit : null,
307
401
  hasMore: truncated,
308
402
  },
309
403
  filtersApplied,