@cliwant/mcp-sam-gov 0.3.0 → 1.1.0

package/src/disclosure.ts

@@ -0,0 +1,61 @@
+/**
+ * disclosure.ts — the single audited disclosure-note TOKENIZER shared across the
+ * keyless DataSources (ADR-0022: an honesty-layer primitive, split into its own
+ * tiny single-concern module — the sibling of coerce.ts's map-layer primitives).
+ *
+ * WHY IT EXISTS: two independent sources (NSF Awards C108 = the OR-note,
+ * ClinicalTrials.gov C109 = the AND-note) each built a mandatory multi-token
+ * disclosure note from the SAME character-for-character delimiter class, and
+ * adversarial verification caught the SAME latent gap on BOTH — a whitespace-only
+ * `.split(/\s+/)` misses the API's PUNCTUATION delimiters, so a compound that only
+ * LOOKS like one token ("coral-reef" = coral OR reef; "Sanofi-Aventis" = Sanofi
+ * AND Aventis) leaks through and the mandatory note is silently SKIPPED. Hoisting
+ * ONE audited tokenizer removes the drift risk (a class regression now fails NSF
+ * AND ClinicalTrials suites at once instead of silently in one) and gives the lint
+ * guardrail (lint-invariants.mjs) + the parity fault test a single home to point
+ * at: "the ONLY sanctioned way to tokenize a disclosure value lives here."
+ *
+ * THE CLASS (`DISCLOSURE_SPLIT_RE`) is the PRECISE ES/Essie confirmed-splitter set,
+ * live-verified byte-identically on BOTH sources 2026-07-12:
+ *   - SPLIT (→ multi-token, the note fires): whitespace + `- , / ; + & | @ # =`
+ *   - DO NOT split (→ one token, no note):   `. : _ '`  (NSF also `\` `*`)
+ * It is deliberately NOT a `[^A-Za-z0-9]+` superset — that would over-disclose a
+ * split the APIs did NOT make on `.`/`_`/`'` (a fabricated union/conjunction). `-`
+ * is placed LAST (a literal, not a range); `/` is escaped for the regex delimiter.
+ *
+ * NOTE-AGNOSTIC BY DESIGN: this returns ONLY the token array. The `.length > 1`
+ * decision, the per-source note WORDING (NSF = OR-union, ClinicalTrials = AND
+ * co-occurrence), and any downstream suppression (ClinicalTrials' sponsor
+ * broadening-note) stay in each CALLER — they are genuinely source-specific and do
+ * NOT belong to the tokenizer.
+ *
+ * THE OPTIONAL `splitRe` PARAM is the sanctioned, honest escape hatch: a FUTURE
+ * source whose analyzer genuinely splits on a DIFFERENT class routes through THIS
+ * helper with its own `splitRe` (explicit, greppable, reviewable) rather than
+ * re-inlining a raw `.split(/\s+/)`. That is exactly what lets the lint ban the
+ * bare whitespace split outright (there is one sanctioned tokenizer, not many).
+ *
+ * No new dep, no I/O, pure function — the exact shape of a coerce.ts primitive.
+ */
+
+/**
+ * The shared ES/Essie confirmed-splitter class (byte-identical to NSF's former
+ * `NSF_KEYWORD_SPLIT_RE` and ClinicalTrials' former `CT_TOKEN_SPLIT_RE`). Stateless
+ * (no `/g`), so a single module-level RegExp is safe to share across callers.
+ */
+export const DISCLOSURE_SPLIT_RE = /[\s,;+&|@#=\/-]+/;
+
+/**
+ * Tokenize a disclosure value for a multi-token honesty note: trim, split on the
+ * confirmed-splitter class (default `DISCLOSURE_SPLIT_RE`), and drop empty tokens.
+ * Returns ONLY the token array — the caller owns the `.length > 1` gate + note
+ * wording. `tokenizeForDisclosure("coral-reef")` → `["coral", "reef"]`;
+ * `tokenizeForDisclosure("web_service")` → `["web_service"]` (the class does NOT
+ * split `_`); `tokenizeForDisclosure("robotics")` → `["robotics"]`.
+ */
+export function tokenizeForDisclosure(
+  value: string,
+  splitRe: RegExp = DISCLOSURE_SPLIT_RE,
+): string[] {
+  return value.trim().split(splitRe).filter((t) => t.length > 0);
+}

package/src/ecfr.ts

@@ -14,10 +14,18 @@
  */
 
 import { fetchWithRetry } from "./errors.js";
+import { driftError } from "./datasource.js";
 import { memoize } from "./cache.js";
+import { withMeta } from "./meta.js";
 
 const ECFR = "https://www.ecfr.gov/api";
 
+// eCFR's /search/v1/results `meta.total_count` is capped by Elasticsearch's
+// `index.max_result_window` (live-verified 10,000). A total AT OR ABOVE this
+// sentinel is a LOWER BOUND, not an exact count — see the totalIsLowerBound
+// wiring in `search` (D1). A genuine count below this stays exact.
+const ECFR_TOTAL_COUNT_CAP = 10000;
+
 async function fetchJson<T>(url: string): Promise<T> {
   const r = await fetchWithRetry(
     url,
@@ -60,6 +68,7 @@ export async function listTitles() {
 export async function search(args: {
   query: string;
   titleNumber?: number;
+  chapter?: number;
   perPage?: number;
 }) {
   const url = new URL(`${ECFR}/search/v1/results`);
@@ -70,6 +79,13 @@ export async function search(args: {
     // an "unpermitted parameter" error from the eCFR API).
     url.searchParams.set("hierarchy[title]", String(args.titleNumber));
   }
+  // Optional chapter filter (additive; existing ecfr_search callers pass none).
+  // Within Title 48: chapter 1 = FAR, chapter 2 = DFARS, chapter 5 = GSAM, etc.
+  // This is what lets far_search scope to FAR/DFARS and keep GSAM/agency
+  // supplements out server-side. Same hierarchy[…] contract as the title filter.
+  if (args.chapter !== undefined) {
+    url.searchParams.set("hierarchy[chapter]", String(args.chapter));
+  }
 
   type Resp = {
     results?: {
@@ -89,9 +105,35 @@ export async function search(args: {
       full_text_excerpt?: string;
       score?: number;
     }[];
+    meta?: {
+      current_page?: number;
+      total_pages?: number;
+      total_count?: number;
+      max_score?: number;
+      description?: string;
+    };
   };
   const json = await fetchJson<Resp>(url.toString());
-  return {
+  // F6 (P2 empty-vs-outage): a 200 whose `results` is PRESENT-but-non-array — or
+  // a body carrying NEITHER a `results` array NOR a `meta` object — is drift / an
+  // unexpected shape, NOT a genuine no-match. Throw (schema_drift) rather than
+  // letting `(json.results ?? []).map` coalesce it into a fake AUTHORITATIVE empty.
+  // A GENUINE empty (results:[] with meta.total_count:0) flows through honestly
+  // below. (eCFR normally signals errors via HTTP status caught by fetchWithRetry;
+  // this closes the previously-unhandled + untested 200-body drift path.)
+  if (json.results !== undefined && !Array.isArray(json.results)) {
+    throw driftError(
+      "ecfr.gov",
+      "eCFR search returned HTTP 200 but `results` is not an array — treating it as schema drift, NOT an empty result set.",
+    );
+  }
+  if (json.results === undefined && json.meta === undefined) {
+    throw driftError(
+      "ecfr.gov",
+      "eCFR search returned HTTP 200 with neither a `results` array nor a `meta` object — an unexpected shape; treating it as schema drift, NOT an empty result set.",
+    );
+  }
+  const data = {
     results: (json.results ?? []).map((r) => ({
       type: r.type ?? "",
       title: r.hierarchy?.title ?? "",
@@ -109,8 +151,63 @@ export async function search(args: {
         ? buildEcfrUrl(r.hierarchy)
         : "",
       effectiveOn: r.starts_on ?? "",
+      // Additive: the version's end date. null = the CURRENT (in-force) version;
+      // a non-null date = a HISTORICAL version. eCFR search returns ~5 versions
+      // per section; existing ecfr_search callers simply ignore this extra field,
+      // while far_search uses it to collapse historical dups to the current one.
+      endsOn: r.ends_on ?? null,
     })),
   };
+
+  // Truthful `_meta` (spec §1.2 A6, §2.3). eCFR returns a hit count in
+  // `meta.total_count`, BUT that count is capped by Elasticsearch's
+  // `index.max_result_window` at 10,000 (a real ceiling — unlike a genuine
+  // small count, a value at/above the cap is a LOWER BOUND, not an exact
+  // total). Below the cap `totalAvailable` is exact (the AI can tell a top-N
+  // slice from the full match set); at/above the cap we flag
+  // `totalIsLowerBound:true` + a note so a broad query that truly matches
+  // >10,000 sections is NOT reported as if exactly 10,000 (D1). A6: echo the
+  // applied title scope so the AI can VERIFY it searched the intended corpus —
+  // Title 48 (FAR) vs every CFR title — rather than silently trusting a filter
+  // that could return cross-title results if the eCFR param contract ever changes.
+  const returned = data.results.length;
+  const totalAvailable =
+    typeof json.meta?.total_count === "number" ? json.meta.total_count : null;
+  // eCFR's search total_count saturates at the Elasticsearch max_result_window
+  // (live-verified 10,000). Use `>= cap` (not `=== cap`) so a total AT OR ABOVE
+  // the ceiling is treated as a lower bound — strictly safe if the window were
+  // ever configured higher. Mirrors federal-register.ts's FR_COUNT_CAP and
+  // edgar.ts's FTS_WINDOW.
+  const totalIsLowerBound =
+    totalAvailable !== null && totalAvailable >= ECFR_TOTAL_COUNT_CAP;
+  const scopeNote =
+    args.titleNumber !== undefined
+      ? `searched CFR Title ${args.titleNumber}${
+          args.titleNumber === 48 ? " (FAR — Federal Acquisition Regulation)" : ""
+        } only`
+      : "searched all CFR titles (no title filter applied)";
+  const notes = [scopeNote];
+  if (totalIsLowerBound) {
+    notes.push(
+      `eCFR caps total_count at ${ECFR_TOTAL_COUNT_CAP} (Elasticsearch index.max_result_window); totalAvailable is a LOWER BOUND — the true match count may be higher and is UNKNOWN. See totalIsLowerBound. Narrow by title/chapter/date for an exact count.`,
+    );
+  }
+  return withMeta(data, {
+    source: "ecfr.gov/api (search/v1)",
+    keylessMode: true,
+    returned,
+    totalAvailable,
+    // Explicit boolean (not conditional): a below-cap count is DEFINITIVELY
+    // exact (totalIsLowerBound:false), an at/above-cap count is a lower bound
+    // (true). The AI can trust `false` as "this is the real total".
+    totalIsLowerBound,
+    truncated:
+      totalAvailable !== null ? returned < totalAvailable : undefined,
+    filtersApplied: args.titleNumber !== undefined ? ["titleNumber"] : [],
+    filtersDropped: [],
+    fieldsUnavailable: [],
+    notes,
+  });
 }
 
 function stripHtml(s: string): string {