@cliwant/mcp-sam-gov 0.2.1 → 1.0.0

package/src/disclosure.ts

@@ -0,0 +1,61 @@
+/**
+ * disclosure.ts — the single audited disclosure-note TOKENIZER shared across the
+ * keyless DataSources (ADR-0022: an honesty-layer primitive, split into its own
+ * tiny single-concern module — the sibling of coerce.ts's map-layer primitives).
+ *
+ * WHY IT EXISTS: two independent sources (NSF Awards C108 = the OR-note,
+ * ClinicalTrials.gov C109 = the AND-note) each built a mandatory multi-token
+ * disclosure note from the SAME character-for-character delimiter class, and
+ * adversarial verification caught the SAME latent gap on BOTH — a whitespace-only
+ * `.split(/\s+/)` misses the API's PUNCTUATION delimiters, so a compound that only
+ * LOOKS like one token ("coral-reef" = coral OR reef; "Sanofi-Aventis" = Sanofi
+ * AND Aventis) leaks through and the mandatory note is silently SKIPPED. Hoisting
+ * ONE audited tokenizer removes the drift risk (a class regression now fails NSF
+ * AND ClinicalTrials suites at once instead of silently in one) and gives the lint
+ * guardrail (lint-invariants.mjs) + the parity fault test a single home to point
+ * at: "the ONLY sanctioned way to tokenize a disclosure value lives here."
+ *
+ * THE CLASS (`DISCLOSURE_SPLIT_RE`) is the PRECISE ES/Essie confirmed-splitter set,
+ * live-verified byte-identically on BOTH sources 2026-07-12:
+ *   - SPLIT (→ multi-token, the note fires): whitespace + `- , / ; + & | @ # =`
+ *   - DO NOT split (→ one token, no note):   `. : _ '`  (NSF also `\` `*`)
+ * It is deliberately NOT a `[^A-Za-z0-9]+` superset — that would over-disclose a
+ * split the APIs did NOT make on `.`/`_`/`'` (a fabricated union/conjunction). `-`
+ * is placed LAST (a literal, not a range); `/` is escaped for the regex delimiter.
+ *
+ * NOTE-AGNOSTIC BY DESIGN: this returns ONLY the token array. The `.length > 1`
+ * decision, the per-source note WORDING (NSF = OR-union, ClinicalTrials = AND
+ * co-occurrence), and any downstream suppression (ClinicalTrials' sponsor
+ * broadening-note) stay in each CALLER — they are genuinely source-specific and do
+ * NOT belong to the tokenizer.
+ *
+ * THE OPTIONAL `splitRe` PARAM is the sanctioned, honest escape hatch: a FUTURE
+ * source whose analyzer genuinely splits on a DIFFERENT class routes through THIS
+ * helper with its own `splitRe` (explicit, greppable, reviewable) rather than
+ * re-inlining a raw `.split(/\s+/)`. That is exactly what lets the lint ban the
+ * bare whitespace split outright (there is one sanctioned tokenizer, not many).
+ *
+ * No new dep, no I/O, pure function — the exact shape of a coerce.ts primitive.
+ */
+
+/**
+ * The shared ES/Essie confirmed-splitter class (byte-identical to NSF's former
+ * `NSF_KEYWORD_SPLIT_RE` and ClinicalTrials' former `CT_TOKEN_SPLIT_RE`). Stateless
+ * (no `/g`), so a single module-level RegExp is safe to share across callers.
+ */
+export const DISCLOSURE_SPLIT_RE = /[\s,;+&|@#=\/-]+/;
+
+/**
+ * Tokenize a disclosure value for a multi-token honesty note: trim, split on the
+ * confirmed-splitter class (default `DISCLOSURE_SPLIT_RE`), and drop empty tokens.
+ * Returns ONLY the token array — the caller owns the `.length > 1` gate + note
+ * wording. `tokenizeForDisclosure("coral-reef")` → `["coral", "reef"]`;
+ * `tokenizeForDisclosure("web_service")` → `["web_service"]` (the class does NOT
+ * split `_`); `tokenizeForDisclosure("robotics")` → `["robotics"]`.
+ */
+export function tokenizeForDisclosure(
+  value: string,
+  splitRe: RegExp = DISCLOSURE_SPLIT_RE,
+): string[] {
+  return value.trim().split(splitRe).filter((t) => t.length > 0);
+}

package/src/ecfr.ts

@@ -1,127 +1,231 @@
-/**
- * eCFR (Electronic Code of Federal Regulations) wrappers (keyless).
- *
- * eCFR is the up-to-date version of the CFR — Title 48 = FAR (Federal
- * Acquisition Regulation), Title 2 = Federal financial assistance, etc.
- * For a federal contractor, eCFR is the primary source for regulation
- * text the agent should quote when answering compliance questions.
- *
- * Endpoints:
- *   - /versioner/v1/titles.json — list 50 CFR titles + last-amended dates
- *   - /search/v1/results — full-text search across the entire CFR
- *
- * Both keyless. Documented at https://www.ecfr.gov/developers/.
- */
-
-const ECFR = "https://www.ecfr.gov/api";
-
-async function fetchJson<T>(url: string): Promise<T> {
-  const r = await fetch(url, {
-    headers: { Accept: "application/json" },
-    signal: AbortSignal.timeout(15_000),
-  });
-  if (!r.ok) {
-    throw new Error(`eCFR ${url} returned ${r.status}`);
-  }
-  return (await r.json()) as T;
-}
-
-export async function listTitles() {
-  type Resp = {
-    titles?: {
-      number?: number;
-      name?: string;
-      latest_amended_on?: string;
-      latest_issue_date?: string;
-      up_to_date_as_of?: string;
-      reserved?: boolean;
-    }[];
-  };
-  const json = await fetchJson<Resp>(`${ECFR}/versioner/v1/titles.json`);
-  return {
-    titles: (json.titles ?? []).map((t) => ({
-      number: t.number ?? 0,
-      name: t.name ?? "",
-      latestAmendedOn: t.latest_amended_on,
-      latestIssueDate: t.latest_issue_date,
-      upToDateAsOf: t.up_to_date_as_of,
-      reserved: !!t.reserved,
-    })),
-  };
-}
-
-export async function search(args: {
-  query: string;
-  titleNumber?: number;
-  perPage?: number;
-}) {
-  const url = new URL(`${ECFR}/search/v1/results`);
-  url.searchParams.set("query", args.query);
-  url.searchParams.set("per_page", String(args.perPage ?? 5));
-  if (args.titleNumber) {
-    // eCFR search filter: hierarchy[title]=N (NOT just title=N — that's
-    // an "unpermitted parameter" error from the eCFR API).
-    url.searchParams.set("hierarchy[title]", String(args.titleNumber));
-  }
-
-  type Resp = {
-    results?: {
-      starts_on?: string;
-      ends_on?: string | null;
-      type?: string;
-      hierarchy?: {
-        title?: string;
-        chapter?: string;
-        subchapter?: string;
-        part?: string;
-        subpart?: string;
-        section?: string;
-      };
-      hierarchy_headings?: Record<string, string | null>;
-      headings?: Record<string, string | null>;
-      full_text_excerpt?: string;
-      score?: number;
-    }[];
-  };
-  const json = await fetchJson<Resp>(url.toString());
-  return {
-    results: (json.results ?? []).map((r) => ({
-      type: r.type ?? "",
-      title: r.hierarchy?.title ?? "",
-      chapter: r.hierarchy?.chapter,
-      part: r.hierarchy?.part,
-      subpart: r.hierarchy?.subpart,
-      section: r.hierarchy?.section,
-      headingPath: Object.values(r.hierarchy_headings ?? {})
-        .filter(Boolean)
-        .join(" › "),
-      excerpt: stripHtml(r.full_text_excerpt ?? ""),
-      score: r.score ?? 0,
-      // Stable ecfr.gov URL pattern from the hierarchy
-      ecfrUrl: r.hierarchy
-        ? buildEcfrUrl(r.hierarchy)
-        : "",
-      effectiveOn: r.starts_on ?? "",
-    })),
-  };
-}
-
-function stripHtml(s: string): string {
-  return s
-    .replace(/<[^>]+>/g, "")
-    .replace(/\s+/g, " ")
-    .trim();
-}
-
-function buildEcfrUrl(h: {
-  title?: string;
-  chapter?: string;
-  part?: string;
-  section?: string;
-}): string {
-  const base = `https://www.ecfr.gov/current/title-${h.title}`;
-  if (h.section) return `${base}/section-${h.section}`;
-  if (h.part) return `${base}/part-${h.part}`;
-  if (h.chapter) return `${base}/chapter-${h.chapter}`;
-  return base;
-}
+/**
+ * eCFR (Electronic Code of Federal Regulations) wrappers (keyless).
+ *
+ * eCFR is the up-to-date version of the CFR — Title 48 = FAR (Federal
+ * Acquisition Regulation), Title 2 = Federal financial assistance, etc.
+ * For a federal contractor, eCFR is the primary source for regulation
+ * text the agent should quote when answering compliance questions.
+ *
+ * Endpoints:
+ *   - /versioner/v1/titles.json — list 50 CFR titles + last-amended dates
+ *   - /search/v1/results — full-text search across the entire CFR
+ *
+ * Both keyless. Documented at https://www.ecfr.gov/developers/.
+ */
+
+import { fetchWithRetry } from "./errors.js";
+import { driftError } from "./datasource.js";
+import { memoize } from "./cache.js";
+import { withMeta } from "./meta.js";
+
+const ECFR = "https://www.ecfr.gov/api";
+
+// eCFR's /search/v1/results `meta.total_count` is capped by Elasticsearch's
+// `index.max_result_window` (live-verified 10,000). A total AT OR ABOVE this
+// sentinel is a LOWER BOUND, not an exact count — see the totalIsLowerBound
+// wiring in `search` (D1). A genuine count below this stays exact.
+const ECFR_TOTAL_COUNT_CAP = 10000;
+
+async function fetchJson<T>(url: string): Promise<T> {
+  const r = await fetchWithRetry(
+    url,
+    {
+      headers: { Accept: "application/json" },
+      signal: AbortSignal.timeout(15_000),
+    },
+    `ecfr:${url.split("/api/")[1] ?? url}`,
+  );
+  return (await r.json()) as T;
+}
+
+export async function listTitles() {
+  // 50 CFR titles change very infrequently. Cache aggressively (5 min).
+  return memoize("ecfr:titles", async () => {
+    type Resp = {
+      titles?: {
+        number?: number;
+        name?: string;
+        latest_amended_on?: string;
+        latest_issue_date?: string;
+        up_to_date_as_of?: string;
+        reserved?: boolean;
+      }[];
+    };
+    const json = await fetchJson<Resp>(`${ECFR}/versioner/v1/titles.json`);
+    return {
+      titles: (json.titles ?? []).map((t) => ({
+        number: t.number ?? 0,
+        name: t.name ?? "",
+        latestAmendedOn: t.latest_amended_on,
+        latestIssueDate: t.latest_issue_date,
+        upToDateAsOf: t.up_to_date_as_of,
+        reserved: !!t.reserved,
+      })),
+    };
+  });
+}
+
+export async function search(args: {
+  query: string;
+  titleNumber?: number;
+  chapter?: number;
+  perPage?: number;
+}) {
+  const url = new URL(`${ECFR}/search/v1/results`);
+  url.searchParams.set("query", args.query);
+  url.searchParams.set("per_page", String(args.perPage ?? 5));
+  if (args.titleNumber) {
+    // eCFR search filter: hierarchy[title]=N (NOT just title=N — that's
+    // an "unpermitted parameter" error from the eCFR API).
+    url.searchParams.set("hierarchy[title]", String(args.titleNumber));
+  }
+  // Optional chapter filter (additive; existing ecfr_search callers pass none).
+  // Within Title 48: chapter 1 = FAR, chapter 2 = DFARS, chapter 5 = GSAM, etc.
+  // This is what lets far_search scope to FAR/DFARS and keep GSAM/agency
+  // supplements out server-side. Same hierarchy[…] contract as the title filter.
+  if (args.chapter !== undefined) {
+    url.searchParams.set("hierarchy[chapter]", String(args.chapter));
+  }
+
+  type Resp = {
+    results?: {
+      starts_on?: string;
+      ends_on?: string | null;
+      type?: string;
+      hierarchy?: {
+        title?: string;
+        chapter?: string;
+        subchapter?: string;
+        part?: string;
+        subpart?: string;
+        section?: string;
+      };
+      hierarchy_headings?: Record<string, string | null>;
+      headings?: Record<string, string | null>;
+      full_text_excerpt?: string;
+      score?: number;
+    }[];
+    meta?: {
+      current_page?: number;
+      total_pages?: number;
+      total_count?: number;
+      max_score?: number;
+      description?: string;
+    };
+  };
+  const json = await fetchJson<Resp>(url.toString());
+  // F6 (P2 empty-vs-outage): a 200 whose `results` is PRESENT-but-non-array — or
+  // a body carrying NEITHER a `results` array NOR a `meta` object — is drift / an
+  // unexpected shape, NOT a genuine no-match. Throw (schema_drift) rather than
+  // letting `(json.results ?? []).map` coalesce it into a fake AUTHORITATIVE empty.
+  // A GENUINE empty (results:[] with meta.total_count:0) flows through honestly
+  // below. (eCFR normally signals errors via HTTP status caught by fetchWithRetry;
+  // this closes the previously-unhandled + untested 200-body drift path.)
+  if (json.results !== undefined && !Array.isArray(json.results)) {
+    throw driftError(
+      "ecfr.gov",
+      "eCFR search returned HTTP 200 but `results` is not an array — treating it as schema drift, NOT an empty result set.",
+    );
+  }
+  if (json.results === undefined && json.meta === undefined) {
+    throw driftError(
+      "ecfr.gov",
+      "eCFR search returned HTTP 200 with neither a `results` array nor a `meta` object — an unexpected shape; treating it as schema drift, NOT an empty result set.",
+    );
+  }
+  const data = {
+    results: (json.results ?? []).map((r) => ({
+      type: r.type ?? "",
+      title: r.hierarchy?.title ?? "",
+      chapter: r.hierarchy?.chapter,
+      part: r.hierarchy?.part,
+      subpart: r.hierarchy?.subpart,
+      section: r.hierarchy?.section,
+      headingPath: Object.values(r.hierarchy_headings ?? {})
+        .filter(Boolean)
+        .join(" › "),
+      excerpt: stripHtml(r.full_text_excerpt ?? ""),
+      score: r.score ?? 0,
+      // Stable ecfr.gov URL pattern from the hierarchy
+      ecfrUrl: r.hierarchy
+        ? buildEcfrUrl(r.hierarchy)
+        : "",
+      effectiveOn: r.starts_on ?? "",
+      // Additive: the version's end date. null = the CURRENT (in-force) version;
+      // a non-null date = a HISTORICAL version. eCFR search returns ~5 versions
+      // per section; existing ecfr_search callers simply ignore this extra field,
+      // while far_search uses it to collapse historical dups to the current one.
+      endsOn: r.ends_on ?? null,
+    })),
+  };
+
+  // Truthful `_meta` (spec §1.2 A6, §2.3). eCFR returns a hit count in
+  // `meta.total_count`, BUT that count is capped by Elasticsearch's
+  // `index.max_result_window` at 10,000 (a real ceiling — unlike a genuine
+  // small count, a value at/above the cap is a LOWER BOUND, not an exact
+  // total). Below the cap `totalAvailable` is exact (the AI can tell a top-N
+  // slice from the full match set); at/above the cap we flag
+  // `totalIsLowerBound:true` + a note so a broad query that truly matches
+  // >10,000 sections is NOT reported as if exactly 10,000 (D1). A6: echo the
+  // applied title scope so the AI can VERIFY it searched the intended corpus —
+  // Title 48 (FAR) vs every CFR title — rather than silently trusting a filter
+  // that could return cross-title results if the eCFR param contract ever changes.
+  const returned = data.results.length;
+  const totalAvailable =
+    typeof json.meta?.total_count === "number" ? json.meta.total_count : null;
+  // eCFR's search total_count saturates at the Elasticsearch max_result_window
+  // (live-verified 10,000). Use `>= cap` (not `=== cap`) so a total AT OR ABOVE
+  // the ceiling is treated as a lower bound — strictly safe if the window were
+  // ever configured higher. Mirrors federal-register.ts's FR_COUNT_CAP and
+  // edgar.ts's FTS_WINDOW.
+  const totalIsLowerBound =
+    totalAvailable !== null && totalAvailable >= ECFR_TOTAL_COUNT_CAP;
+  const scopeNote =
+    args.titleNumber !== undefined
+      ? `searched CFR Title ${args.titleNumber}${
+          args.titleNumber === 48 ? " (FAR — Federal Acquisition Regulation)" : ""
+        } only`
+      : "searched all CFR titles (no title filter applied)";
+  const notes = [scopeNote];
+  if (totalIsLowerBound) {
+    notes.push(
+      `eCFR caps total_count at ${ECFR_TOTAL_COUNT_CAP} (Elasticsearch index.max_result_window); totalAvailable is a LOWER BOUND — the true match count may be higher and is UNKNOWN. See totalIsLowerBound. Narrow by title/chapter/date for an exact count.`,
+    );
+  }
+  return withMeta(data, {
+    source: "ecfr.gov/api (search/v1)",
+    keylessMode: true,
+    returned,
+    totalAvailable,
+    // Explicit boolean (not conditional): a below-cap count is DEFINITIVELY
+    // exact (totalIsLowerBound:false), an at/above-cap count is a lower bound
+    // (true). The AI can trust `false` as "this is the real total".
+    totalIsLowerBound,
+    truncated:
+      totalAvailable !== null ? returned < totalAvailable : undefined,
+    filtersApplied: args.titleNumber !== undefined ? ["titleNumber"] : [],
+    filtersDropped: [],
+    fieldsUnavailable: [],
+    notes,
+  });
+}
+
+function stripHtml(s: string): string {
+  return s
+    .replace(/<[^>]+>/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function buildEcfrUrl(h: {
+  title?: string;
+  chapter?: string;
+  part?: string;
+  section?: string;
+}): string {
+  const base = `https://www.ecfr.gov/current/title-${h.title}`;
+  if (h.section) return `${base}/section-${h.section}`;
+  if (h.part) return `${base}/part-${h.part}`;
+  if (h.chapter) return `${base}/chapter-${h.chapter}`;
+  return base;
+}