2sagaco48 1.0.0

package/src/lib/sage.ts

@@ -0,0 +1,166 @@
+/**
+ * Type B shim — SAGE barrel with a PROMPT OVERRIDE.
+ * ---------------------------------------------------------------------------
+ * WHY THIS FILE IS A WRAPPER AND NOT A PURE RE-EXPORT
+ *
+ * The packaged fiction runner (`src/components/ChatApp.tsx`) builds
+ * every chapter prompt with:
+ *
+ *     import { compileCompactChapterPrompt } from "@/lib/sage";
+ *     ...
+ *     compileCompactChapterPrompt(bp, i, priorSummary, bp.cadence)
+ *
+ * `@/lib/sage` resolves to THIS file. So overriding the export here is the one
+ * durable, non-DOM seam that puts SAGA-OS v2 + reality grounding into the text
+ * that is actually sent to the model — and therefore into the main output box.
+ *
+ * A previous turn attempted this with a capture-phase `submit` listener that
+ * mutated the textarea and gated on `getV15Enabled()`. That was wrong twice
+ * over: V15 is a different toggle from Fiction mode, and the fiction runner
+ * never re-reads the textarea (it uses the frozen `blueprint` memo). That
+ * bridge is REMOVED and replaced by this override.
+ *
+ * INVARIANTS
+ *  - Everything the package exports is still exported (`export *`).
+ *  - The override is ADDITIVE: it calls the original compiler first and appends.
+ *  - The unified block is derived from `bp.seed`, so SAGE, the Genesis-Grid,
+ *    KALLOS and the reality level all ride the SAME seed as the blueprint the
+ *    user previewed. No second roll, no divergence.
+ *  - If anything throws, the ORIGINAL prompt is returned unchanged. The fiction
+ *    engine can never be broken by this layer.
+ */
+
+export * from '@/lib/sage/index.ts';
+
+// Net-new workspace subsystems, re-exported through the same barrel so any
+// consumer importing "@/lib/sage" sees one coherent surface.
+export * from "./sage/reality-grounding";
+export * from "./sage/saga-os-v2-engines";
+export * from "./sage/unified-roll";
+export * from "./sage/saga-os-v3-core";
+export * from "./sage/saga-os-v3-release";
+export * from "./sage/saga-os-v4-aureate";
+
+import {
+  compileCompactChapterPrompt as _origCompileCompactChapterPrompt,
+  scrubLeakedDirective as _origScrubLeakedDirective,
+  type SageBlueprint, type SageProseCadence,
+} from '@/lib/sage/engine.ts';
+import { unifiedRoll } from "./sage/unified-roll";
+import {
+  loadRealityLevel, auditRealityGrounding, type RealityAuditReport,
+} from "./sage/reality-grounding";
+import {
+  evaluateKallosS, buildStyleGenome, applyStylePack, loadStylePack,
+  type KallosSReport, type AureateContext,
+} from "./sage/saga-os-v4-aureate";
+
+/** Last block actually injected — surfaced in the UI so the user can read it. */
+let _lastInjected: { seed: number; chapterIndex: number; block: string; at: number } | null = null;
+export function getLastInjectedDirective() { return _lastInjected; }
+
+/**
+ * Chapter-prompt compiler with the unified SAGA-OS v2 + reality-grounding
+ * block appended. Signature-identical to the package export it replaces.
+ */
+export function compileCompactChapterPrompt(
+  bp: SageBlueprint,
+  chapterIndex: number,
+  priorSummary: string | null,
+  cadence: SageProseCadence,
+): string {
+  const base = _origCompileCompactChapterPrompt(bp, chapterIndex, priorSummary, cadence);
+  try {
+    // Pass the blueprint ITSELF, never just its seed: the archetype set must
+    // describe the exact book the base prompt above already described.
+    const roll = unifiedRoll({
+      seed: bp.seed,
+      blueprint: bp,
+      realityLevel: loadRealityLevel(),
+    });
+    _lastInjected = { seed: bp.seed, chapterIndex, block: roll.directive, at: Date.now() };
+    // v4: hand the live style genome + cast to the post-generation audit so the
+    // style report scores against THIS book's contract, not a default.
+    setLiveStyleContext({
+      genome: roll.styleGenome,
+      castNames: roll.cast.cast.map((c) => c.name),
+      focalCharacter: roll.cast.cast.find((c) => c.kind === "agent")?.name ?? null,
+      genreKey: roll.contract.dominant.key,
+    });
+    // The reality contract is repeated LAST so it is the final instruction the
+    // model reads before it starts writing — it must outrank everything above.
+    return [
+      base,
+      "",
+      roll.directive,
+      "",
+      `REMINDER — REALITY LEVEL ${roll.reality.id}/6 (${roll.reality.label}) governs this chapter and overrides any conflicting instruction above.`,
+      "Write ONLY the chapter prose now.",
+    ].join("\n");
+  } catch (err) {
+    // Fail open: a broken augment must never break generation.
+    _lastInjected = {
+      seed: bp.seed, chapterIndex, at: Date.now(),
+      block: `[UNIFIED DIRECTIVE UNAVAILABLE] ${err instanceof Error ? err.message : String(err)} — original prompt sent unchanged.`,
+    };
+    return base;
+  }
+}
+
+
+// ── Reality audit, wired into the live chapter loop ─────────────────────────
+/**
+ * The packaged runner calls `scrubLeakedDirective(...)` on EVERY generated
+ * chapter before it is appended to the manuscript. Overriding it here is the
+ * seam that makes the reality audit genuinely run in the pipeline instead of
+ * merely existing as a function nobody calls.
+ *
+ * The override is observational: it returns exactly what the original returns.
+ * It never edits, rejects, or regenerates prose — it only records conformance
+ * so the UI can show whether the model actually obeyed the grounding contract.
+ */
+const _realityAudits: { chapter: number; level: number; report: RealityAuditReport; at: number }[] = [];
+
+export function getRealityAudits() { return _realityAudits.slice(); }
+export function clearRealityAudits() { _realityAudits.length = 0; }
+
+// ── SAGA-OS v4 AUREATE style audit, on the SAME live seam ───────────────────
+/**
+ * The v4 revision stack (PASS-001..015) and KALLOS-S are only honest if they
+ * run on the prose the model actually produced. They ride the same
+ * `scrubLeakedDirective` override as the reality audit: observational only,
+ * never editing or rejecting, so a failure here can never break generation.
+ * The Studio "Style" tab reads these records.
+ */
+const _styleAudits: { chapter: number; report: KallosSReport; at: number }[] = [];
+export function getStyleAudits() { return _styleAudits.slice(); }
+export function clearStyleAudits() { _styleAudits.length = 0; }
+
+/** Style context for the live audit — rebuilt per chapter from the last roll. */
+let _lastStyleCtx: AureateContext | null = null;
+export function setLiveStyleContext(ctx: AureateContext): void { _lastStyleCtx = ctx; }
+
+export function scrubLeakedDirective(text: string): string {
+  const cleaned = _origScrubLeakedDirective(text);
+  try {
+    const level = loadRealityLevel();
+    const words = (cleaned.trim().match(/\S+/g) || []).length;
+    if (words >= 40) {
+      _realityAudits.push({
+        chapter: _realityAudits.length + 1,
+        level,
+        report: auditRealityGrounding(cleaned, level),
+        at: Date.now(),
+      });
+      if (_realityAudits.length > 200) _realityAudits.shift();
+
+      // v4: style audit over the same finished chapter.
+      const ctx = _lastStyleCtx ?? {
+        genome: applyStylePack(buildStyleGenome("literary"), loadStylePack()),
+      };
+      _styleAudits.push({ chapter: _styleAudits.length + 1, report: evaluateKallosS(cleaned, ctx), at: Date.now() });
+      if (_styleAudits.length > 200) _styleAudits.shift();
+    }
+  } catch { /* observation must never break generation */ }
+  return cleaned;
+}

package/src/lib/scraper-debug-bus.ts

@@ -0,0 +1,53 @@
+/**
+ * scraper-debug-bus.ts
+ * ============================================================================
+ * Lightweight, additive, dependency-free pub/sub bus so every scraper lane
+ * (accelerator, hydra, nexus, sibyl, strata, palisade, arbiter, academic
+ * sources, enhanced scraper, OG scraper, package original groundQuestion,
+ * VNext terminal/portfolio stack) can emit a visible, timestamped log line
+ * that the V15 Rigor Guard UI renders live AND that always echoes to the
+ * browser console via console.log — closing the "I still don't see them
+ * log" gap. Purely additive; nothing existing is modified.
+ * ============================================================================ */
+
+export interface ScraperLogLine {
+  ts: number;
+  lane: string;
+  message: string;
+}
+
+const listeners = new Set<(line: ScraperLogLine) => void>();
+const HISTORY_MAX = 300;
+let history: ScraperLogLine[] = [];
+
+export function emitScraperDebug(lane: string, message: string): void {
+  const line: ScraperLogLine = { ts: Date.now(), lane, message };
+  history.push(line);
+  if (history.length > HISTORY_MAX) history = history.slice(-HISTORY_MAX);
+  try {
+    // eslint-disable-next-line no-console
+    console.log(`[scraper:${lane}] ${message}`);
+  } catch {
+    /* console unavailable */
+  }
+  for (const cb of listeners) {
+    try {
+      cb(line);
+    } catch {
+      /* fail-open per subscriber */
+    }
+  }
+}
+
+export function subscribeScraperDebug(cb: (line: ScraperLogLine) => void): () => void {
+  listeners.add(cb);
+  return () => listeners.delete(cb);
+}
+
+export function getScraperDebugHistory(): ScraperLogLine[] {
+  return history.slice();
+}
+
+export function clearScraperDebugHistory(): void {
+  history = [];
+}

package/src/lib/scraper-enhanced.orig.ts

@@ -0,0 +1,153 @@
+/**
+ * Enhanced Scraper — patched with preserveSourceDuplicates to allow RRF accumulation.
+ * This is a verbatim copy of the package's implementation with the single additive option.
+ */
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+  source: string;
+  relevanceScore?: number;
+  publishedAt?: string;
+  domain?: string;
+}
+export interface ScraperResult {
+  url: string;
+  title: string;
+  content: string;
+  extractedData: { dates?: string[]; numbers?: string[]; entities?: string[]; citations?: string[]; emails?: string[]; urls?: string[] };
+  qualityScore: number;
+  contentLength: number;
+}
+
+const SEARCH_ENGINES = {
+  google: (q: string) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
+  bing: (q: string) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+  duckduckgo: (q: string) => `https://duckduckgo.com/html/?q=${encodeURIComponent(q)}`,
+  wikipedia: (q: string) => `https://en.wikipedia.org/w/index.php?search=${encodeURIComponent(q)}`,
+  scholar: (q: string) => `https://scholar.google.com/scholar?q=${encodeURIComponent(q)}`,
+  arxiv: (q: string) => `https://arxiv.org/search/?query=${encodeURIComponent(q)}&searchtype=all`,
+};
+
+const NICHE_APIS = {
+  hackernews: (q: string) => `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(q)}&hitsPerPage=10`,
+  stackexchange: (q: string, site = "stackoverflow") => `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${encodeURIComponent(q)}&site=${site}&pagesize=10`,
+  reddit: (q: string) => `https://www.reddit.com/search.json?q=${encodeURIComponent(q)}&limit=10&sort=relevance`,
+  semanticscholar: (q: string) => `https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(q)}&limit=10&fields=title,abstract,authors,year,url,venue,citationCount`,
+  crossref: (q: string) => `https://api.crossref.org/works?query=${encodeURIComponent(q)}&rows=10`,
+  openalex: (q: string) => `https://api.openalex.org/works?search=${encodeURIComponent(q)}&per-page=10`,
+  pubmed: (q: string) => `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=${encodeURIComponent(q)}&retmax=10&retmode=json`,
+  sec_edgar: (q: string) => `https://efts.sec.gov/LATEST/search-index?q=${encodeURIComponent(q)}&dateRange=custom&forms=10-K,10-Q,8-K`,
+  patents: (q: string) => `https://api.patentsview.org/patents/query?q={"_text_any":{"patent_title":"${encodeURIComponent(q)}"}}&f=["patent_number","patent_title","patent_date","patent_abstract"]&o={"per_page":10}`,
+  wayback: (url: string) => `https://archive.org/wayback/available?url=${encodeURIComponent(url)}`,
+  github: (q: string) => `https://api.github.com/search/repositories?q=${encodeURIComponent(q)}&per_page=10`,
+};
+
+const STACK_EXCHANGE_SITES = ["stackoverflow","stats","math","physics","chemistry","biology","engineering","medicalsciences","law","money","quant","datascience","ai","electronics","workplace","academia"];
+
+const PROXY_FLEET = [
+  { name: "corsproxy.io", build: (u: string) => `https://corsproxy.io/?url=${encodeURIComponent(u)}` },
+  { name: "allorigins-raw", build: (u: string) => `https://api.allorigins.win/raw?url=${encodeURIComponent(u)}` },
+  { name: "codetabs", build: (u: string) => `https://api.codetabs.com/v1/proxy/?quest=${encodeURIComponent(u)}` },
+  { name: "cors.sh", build: (u: string) => `https://proxy.cors.sh/${u}` },
+  { name: "yacdn", build: (u: string) => `https://yacdn.org/proxy/${u}` },
+  { name: "thingproxy", build: (u: string) => `https://thingproxy.freeboard.io/fetch/${u}` },
+  { name: "cors-anywhere", build: (u: string) => `https://cors-anywhere.herokuapp.com/${u}` },
+  { name: "corsproxy.org", build: (u: string) => `https://corsproxy.org/?${encodeURIComponent(u)}` },
+];
+
+const TRUSTED_DOMAINS: Record<string, number> = {
+  "nature.com":0.3,"science.org":0.3,"cell.com":0.3,"nejm.org":0.3,"thelancet.com":0.3,"bmj.com":0.25,
+  "ieee.org":0.25,"acm.org":0.25,"springer.com":0.2,"sciencedirect.com":0.2,"wiley.com":0.2,"jstor.org":0.2,
+  "sec.gov":0.3,"federalreserve.gov":0.3,"bls.gov":0.3,"census.gov":0.3,"who.int":0.3,"cdc.gov":0.3,"nih.gov":0.3,"fda.gov":0.3,
+  "arxiv.org":0.2,"biorxiv.org":0.15,"ssrn.com":0.15,"reuters.com":0.15,"ft.com":0.2,"bloomberg.com":0.2,"wsj.com":0.2,
+  "economist.com":0.15,"hbr.org":0.15,"mckinsey.com":0.15,"bain.com":0.15,"bcg.com":0.15,"deloitte.com":0.1,"pwc.com":0.1,"kpmg.com":0.1,
+  "gartner.com":0.15,"forrester.com":0.15,"idc.com":0.15,"github.com":0.1,"stackoverflow.com":0.1,
+};
+
+function domainOf(url: string): string { try{ return new URL(url).hostname.replace(/^www\./,""); }catch{ return ""; } }
+function trustBoost(url: string): number { const d=domainOf(url); for (const [k,v] of Object.entries(TRUSTED_DOMAINS)) if (d.endsWith(k)) return v; return 0; }
+
+export async function enhancedSearch(
+  query: string,
+  options?: {
+    engines?: Array<keyof typeof SEARCH_ENGINES>;
+    maxResults?: number;
+    includeAcademic?: boolean;
+    includeIndustry?: boolean;
+    includeForums?: boolean;
+    stackExchangeSites?: string[];
+    preserveSourceDuplicates?: boolean;
+  }
+): Promise<SearchResult[]> {
+  const engines = options?.engines || ["google","bing","duckduckgo"];
+  const maxResults = options?.maxResults || 20;
+  const includeAcademic = options?.includeAcademic !== false;
+  const includeIndustry = options?.includeIndustry !== false;
+  const includeForums = options?.includeForums !== false;
+  const seSites = options?.stackExchangeSites || ["stackoverflow","stats","engineering"];
+  const promises: Promise<SearchResult[]>[] = [];
+  for (const engine of engines) {
+    promises.push((async ()=>{ try{ const r=await fetch(SEARCH_ENGINES[engine](query)); if (!r.ok) return []; const html=await r.text(); return parseSearchResults(html,engine); }catch{ return []; } })());
+  }
+  if (includeAcademic){ promises.push(fetchSemanticScholar(query)); promises.push(fetchCrossref(query)); promises.push(fetchOpenAlex(query)); promises.push(fetchPubMed(query)); }
+  if (includeForums){ promises.push(fetchHackerNews(query)); for (const site of seSites) promises.push(fetchStackExchange(query,site)); promises.push(fetchReddit(query)); }
+  if (includeIndustry){ promises.push(fetchGitHub(query)); }
+  const all=(await Promise.allSettled(promises)).flatMap((p)=> p.status==="fulfilled" ? p.value : []);
+  const seen=new Set<string>();
+  const deduped=all.filter((result)=>{
+    if (!result.url) return false;
+    const dedupeKey = options?.preserveSourceDuplicates ? `${result.source||"web"}\u0000${result.url}` : result.url;
+    if (seen.has(dedupeKey)) return false;
+    seen.add(dedupeKey);
+    result.domain=domainOf(result.url);
+    result.relevanceScore=(result.relevanceScore ?? 0.5)+trustBoost(result.url);
+    return true;
+  });
+  deduped.sort((a,b)=>(b.relevanceScore ?? 0)-(a.relevanceScore ?? 0));
+  return deduped.slice(0,maxResults);
+}
+
+async function fetchHackerNews(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.hackernews(q)); if (!r.ok) return []; const j=await r.json(); return (j.hits||[]).slice(0,10).map((h:any)=>({ title:h.title||h.story_title||"HN discussion", url:h.url||`https://news.ycombinator.com/item?id=${h.objectID}`, snippet:(h.story_text||h.comment_text||"").replace(/<[^>]*>/g,"").slice(0,300), source:"hackernews", relevanceScore: Math.min(1,0.5+(h.points||0)/200+(h.num_comments||0)/200), publishedAt:h.created_at })); }catch{ return []; }
+}
+async function fetchStackExchange(q: string, site: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.stackexchange(q,site)); if (!r.ok) return []; const j=await r.json(); return (j.items||[]).slice(0,5).map((it:any)=>({ title:it.title||"SE question", url:it.link, snippet:`${it.answer_count} answer(s) · score ${it.score} · [${site}]`, source:`se-${site}`, relevanceScore: Math.min(1,0.4+(it.score||0)/50+(it.is_answered?0.2:0)) })); }catch{ return []; }
+}
+async function fetchReddit(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.reddit(q),{headers:{"User-Agent":"veritas-scraper/1.0"}}); if (!r.ok) return []; const j=await r.json(); return (j.data?.children||[]).slice(0,10).map((c:any)=>({ title:c.data.title, url:`https://reddit.com${c.data.permalink}`, snippet:(c.data.selftext||"").slice(0,300), source:`reddit-r/${c.data.subreddit}`, relevanceScore: Math.min(1,0.4+(c.data.score||0)/500) })); }catch{ return []; }
+}
+async function fetchSemanticScholar(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.semanticscholar(q)); if (!r.ok) return []; const j=await r.json(); return (j.data||[]).slice(0,10).map((p:any)=>({ title:p.title, url:p.url, snippet:(p.abstract||"").slice(0,400), source:"semantic-scholar", relevanceScore: Math.min(1,0.6+(p.citationCount||0)/500), publishedAt:p.year?String(p.year):undefined })); }catch{ return []; }
+}
+async function fetchCrossref(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.crossref(q)); if (!r.ok) return []; const j=await r.json(); return (j.message?.items||[]).slice(0,10).map((it:any)=>({ title:(it.title||[""])[0], url:it.URL||(it.DOI?`https://doi.org/${it.DOI}`:""), snippet:(it.abstract||"").replace(/<[^>]*>/g,"").slice(0,400), source:"crossref", relevanceScore: Math.min(1,0.55+(it["is-referenced-by-count"]||0)/300), publishedAt:it.created?.["date-time"] })); }catch{ return []; }
+}
+async function fetchOpenAlex(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.openalex(q)); if (!r.ok) return []; const j=await r.json(); return (j.results||[]).slice(0,10).map((w:any)=>({ title:w.title, url:w.doi?`https://doi.org/${w.doi.replace("https://doi.org/","")}`:w.id, snippet:(w.abstract_inverted_index ? Object.keys(w.abstract_inverted_index).slice(0,60).join(" ") : "").slice(0,400), source:"openalex", relevanceScore: Math.min(1,0.55+(w.cited_by_count||0)/500), publishedAt:w.publication_year?String(w.publication_year):undefined })); }catch{ return []; }
+}
+async function fetchPubMed(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.pubmed(q)); if (!r.ok) return []; const j=await r.json(); const ids:string[]=j?.esearchresult?.idlist||[]; return ids.slice(0,10).map((id)=>({ title:`PubMed PMID ${id}`, url:`https://pubmed.ncbi.nlm.nih.gov/${id}/`, snippet:"PubMed record — resolve via PMID for abstract", source:"pubmed", relevanceScore:0.7 })); }catch{ return []; }
+}
+async function fetchGitHub(q: string): Promise<SearchResult[]> {
+  try{ const r=await fetch(NICHE_APIS.github(q)); if (!r.ok) return []; const j=await r.json(); return (j.items||[]).slice(0,10).map((it:any)=>({ title:`${it.full_name} — ${it.description||"GitHub repo"}`, url:it.html_url, snippet:(it.description||"").slice(0,300), source:"github", relevanceScore: Math.min(1,0.4+(it.stargazers_count||0)/5000) })); }catch{ return []; }
+}
+export async function deepScrape(url: string): Promise<ScraperResult> {
+  try{ const r=await fetch(url,{headers:{"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) veritas-scraper/2.0",Accept:"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5"}}); if (r.ok){ const html=await r.text(); return parseAndExtract(html,url); } }catch{}
+  for (const proxy of PROXY_FLEET){ try{ const r=await fetch(proxy.build(url),{headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",Accept:"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Accept-Language":"en-US,en;q=0.9"}}); if (!r.ok) continue; const html=await r.text(); return parseAndExtract(html,url); }catch{ continue; } }
+  throw new Error(`Failed to scrape ${url} — all proxies exhausted`);
+}
+function parseSearchResults(html: string, engine: string): SearchResult[] {
+  const results: SearchResult[]=[]; const titleRegex=/<h3[^>]*>([\s\S]*?)<\/h3>/gi; let m; while((m=titleRegex.exec(html))!==null){ results.push({ title:m[1].replace(/<[^>]*>/g,"").trim(), url:"", snippet:"", source:engine, relevanceScore:0.5 }); } return results;
+}
+function parseAndExtract(html: string, url: string): ScraperResult {
+  const titleMatch=html.match(/<title[^>]*>([\s\S]*?)<\/title>/i); const title=titleMatch?titleMatch[1].trim():"Untitled";
+  const bodyMatch=html.match(/<body[^>]*>([\s\S]*?)<\/body>/i); const body=bodyMatch?bodyMatch[1]:html;
+  const content=body.replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<[^>]+>/g," ").replace(/\s+/g," ").trim().slice(0,80000);
+  const extractedData={ dates: content.match(/\b(19|20)\d{2}\b/g)?.slice(0,100)||[], numbers: content.match(/\b\d+(?:\.\d+)?(?:%|K|M|B|T)?\b/g)?.slice(0,200)||[], entities: content.match(/\b[A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+)+\b/g)?.slice(0,100)||[], citations: content.match(/\[\d+\]/g)?.slice(0,100)||[], emails: content.match(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g)?.slice(0,50)||[], urls: content.match(/https?:\/\/[^\s<>"']+/g)?.slice(0,100)||[] };
+  let qualityScore=0.4; if (content.length>3000) qualityScore+=0.15; if (content.length>15000) qualityScore+=0.1; if (extractedData.dates.length>5) qualityScore+=0.1; if (extractedData.numbers.length>15) qualityScore+=0.1; if (extractedData.entities.length>8) qualityScore+=0.1; if (extractedData.citations.length>3) qualityScore+=0.05; qualityScore+=trustBoost(url);
+  return { url, title, content, extractedData, qualityScore: Math.min(1.0,qualityScore), contentLength: content.length };
+}
+export const STACK_EXCHANGE_ALL_SITES=STACK_EXCHANGE_SITES;
+export { PROXY_FLEET, TRUSTED_DOMAINS };
+export * from "./scraper-hardener";

package/src/lib/scraper-enhanced.ts

@@ -0,0 +1,10 @@
+// SIDECAR SEAM — see flatten-guide.md.
+// Alias-reachable retrieval lane: the packaged `v15-grounding.ts` imports this
+// module via `@/lib/scraper-enhanced`, so this seam IS on the live path.
+// Adds a provenance tap only; retrieval behaviour is byte-for-byte unchanged.
+export * from "./scraper-enhanced.orig";
+
+import { enhancedSearch as packageEnhancedSearch } from "./scraper-enhanced.orig";
+import { withLaneTap } from "@/lib/citation-lane-tap";
+
+export const enhancedSearch = withLaneTap("enhanced-scraper", packageEnhancedSearch);

package/src/lib/scraper-hardener.base.orig.ts

@@ -0,0 +1,155 @@
+/**
+ * Deterministic CORS Proxy & Scraper Hardening (May 2026 fleet).
+ *
+ * Public CORS relays churn constantly, so we keep a broad, prioritized list and
+ * rotate through several response shapes (raw passthrough vs. JSON-wrapped).
+ * The first relay to return usable bytes wins.
+ */
+
+export interface ProxyDef {
+  name: string;
+  build: (url: string) => string;
+  /** Some relays wrap the body in JSON ({contents:"..."}); extract if so. */
+  unwrap?: (body: string) => string;
+}
+
+// Ordered fastest/most-reliable first as of May 2026. Raw passthrough relays are
+// preferred because they don't double-encode HTML entities. Expanded fleet for
+// resilience — the OG browser scraper is the workhorse so it must rarely fail.
+export const PROXY_FLEET: ProxyDef[] = [
+  { name: "corsproxy.io", build: (u) => `https://corsproxy.io/?url=${encodeURIComponent(u)}` },
+  { name: "allorigins-raw", build: (u) => `https://api.allorigins.win/raw?url=${encodeURIComponent(u)}` },
+  { name: "codetabs", build: (u) => `https://api.codetabs.com/v1/proxy/?quest=${encodeURIComponent(u)}` },
+  { name: "corsproxy.org", build: (u) => `https://corsproxy.org/?${encodeURIComponent(u)}` },
+  { name: "thingproxy", build: (u) => `https://thingproxy.freeboard.io/fetch/${u}` },
+  { name: "cors.eu.org", build: (u) => `https://cors.eu.org/${u}` },
+  { name: "cors-anywhere-hf", build: (u) => `https://cors-anywhere.herokuapp.com/${u}` },
+  { name: "proxy.cors.sh", build: (u) => `https://proxy.cors.sh/${u}` },
+  { name: "yacdn", build: (u) => `https://yacdn.org/proxy/${u}` },
+  { name: "whateverorigin", build: (u) => `https://whateverorigin.org/get?url=${encodeURIComponent(u)}`, unwrap: jsonContents },
+  { name: "allorigins-get", build: (u) => `https://api.allorigins.win/get?url=${encodeURIComponent(u)}`, unwrap: jsonContents },
+  { name: "jsonp.afeld", build: (u) => `https://jsonp.afeld.me/?url=${encodeURIComponent(u)}` },
+];
+
+function jsonContents(body: string): string {
+  try {
+    const data = JSON.parse(body);
+    return typeof data?.contents === "string" ? data.contents : body;
+  } catch {
+    return body;
+  }
+}
+
+const BROWSER_HEADERS: Record<string, string> = {
+  "User-Agent":
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5",
+  "Accept-Language": "en-US,en;q=0.9",
+};
+
+async function timedFetch(url: string, signal: AbortSignal | undefined, headers: Record<string, string>): Promise<Response> {
+  // Compose caller signal with an internal timeout so one dead relay can't hang the chain.
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), 8_000);
+  const onAbort = () => ctrl.abort();
+  signal?.addEventListener("abort", onAbort, { once: true });
+  try {
+    return await fetch(url, { signal: ctrl.signal, headers, redirect: "follow" });
+  } finally {
+    clearTimeout(timer);
+    signal?.removeEventListener("abort", onAbort);
+  }
+}
+
+function looksUsable(text: string): boolean {
+  if (!text) return false;
+  if (text.length < 80) return false;
+  // Reject obvious relay error envelopes.
+  if (/^\s*(error|forbidden|too many requests|rate limit)/i.test(text) && text.length < 400) return false;
+  return true;
+}
+
+export interface RobustFetchResult {
+  text: string;
+  source: "direct" | "proxy";
+  proxy?: string;
+}
+
+/** Try a single proxy, resolving to a usable body or rejecting. */
+async function tryProxy(proxy: ProxyDef, url: string, signal?: AbortSignal): Promise<RobustFetchResult> {
+  const res = await timedFetch(proxy.build(url), signal, BROWSER_HEADERS);
+  if (!res.ok) throw new Error(`${proxy.name}:${res.status}`);
+  const raw = await res.text();
+  const body = proxy.unwrap ? proxy.unwrap(raw) : raw;
+  if (!looksUsable(body)) throw new Error(`${proxy.name}:thin`);
+  return { text: body, source: "proxy", proxy: proxy.name };
+}
+
+/** Resolve the first fulfilled promise; reject only if ALL reject. */
+function firstSuccess<T>(promises: Promise<T>[]): Promise<T> {
+  return new Promise((resolve, reject) => {
+    let remaining = promises.length;
+    const errors: any[] = [];
+    if (remaining === 0) reject(new Error("no candidates"));
+    promises.forEach(p => p.then(resolve, (e) => {
+      errors.push(e);
+      if (--remaining === 0) reject(new Error(errors.map(x => x?.message ?? x).join(", ")));
+    }));
+  });
+}
+
+/**
+ * Resilient fetch: optional pinned proxy → direct → RACE the proxy fleet.
+ * Racing collapses the old sequential per-proxy delay into ~1 wave latency.
+ * Returns the first usable body. Throws CORS_BLOCKED only when everything fails.
+ */
+export async function fetchRobust(url: string, signal?: AbortSignal, preferProxy?: string): Promise<RobustFetchResult> {
+  // 0. Pinned proxy first (e.g. codetabs for Bing — confirmed working combos).
+  if (preferProxy) {
+    const pinned = PROXY_FLEET.find(p => p.name === preferProxy);
+    if (pinned) {
+      try { return await tryProxy(pinned, url, signal); } catch { /* fall through */ }
+    }
+  }
+
+  // 1. Direct (works for CORS-enabled origins) — short timeout.
+  try {
+    const res = await timedFetch(url, signal, BROWSER_HEADERS);
+    if (res.ok) {
+      const text = await res.text();
+      if (looksUsable(text)) return { text, source: "direct" };
+    }
+  } catch { /* fall through */ }
+
+  // 2. Race the proxy fleet in small waves — first usable wins. Keep this at
+  // 2 because cluster search can launch several scraper reads in parallel.
+  const WAVE = 2;
+  const errs: string[] = [];
+  for (let i = 0; i < PROXY_FLEET.length; i += WAVE) {
+    if (signal?.aborted) break;
+    const wave = PROXY_FLEET.slice(i, i + WAVE);
+    try {
+      return await firstSuccess(wave.map(p => tryProxy(p, url, signal)));
+    } catch (e: any) {
+      errs.push(e?.message ?? "wave-failed");
+    }
+  }
+
+  throw new Error(`CORS_BLOCKED: all paths failed for ${url} [${errs.join("; ").slice(0, 200)}]`);
+}
+
+/**
+ * Extract substantive text from raw HTML (lightweight, no DOM).
+ */
+export function extractTextFromHtml(html: string): string {
+  let text = html.replace(/<(script|style|nav|footer|header|noscript|svg)[^>]*>[\s\S]*?<\/\1>/gi, "");
+  text = text.replace(/<[^>]+>/g, " ");
+  text = text
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#x27;|&#39;/g, "'");
+  return text.replace(/\s+/g, " ").trim().slice(0, 12_000);
+}