2sagaco48 1.0.0

package/src/lib/scraper-vnext/strata-engine.ts

@@ -0,0 +1,13 @@
+export * from "./strata-engine.orig";
+
+import { strataCollect as packageCollect } from "./strata-engine.orig";
+import { filterRelevantSourcesAny, hardenRetrievalQuery } from "@/lib/debug/retrieval-hardener";
+
+export async function strataCollect(...args: Parameters<typeof packageCollect>): Promise<Awaited<ReturnType<typeof packageCollect>>> {
+  const [query, opts] = args;
+  const h = hardenRetrievalQuery(String(query), "general");
+  const result: any = await packageCollect(h.query, opts);
+  const gate = filterRelevantSourcesAny(Array.isArray(result?.sources) ? result.sources : [], h.alternatives);
+  try { opts?.onDebug?.(`workspace-relevance-gate: strata accepted ${gate.accepted.length}, rejected ${gate.rejected.length}`); } catch {}
+  return { ...result, sources: gate.accepted, ok: !!result?.ok && gate.accepted.length >= 2 };
+}

package/src/lib/scraper-vnext/structured-source-adapter.orig.ts

@@ -0,0 +1,369 @@
+/**
+ * structured-source-adapter.ts
+ * ============================================================================
+ * ADDITIVE — composes over canonical retrieval-policy-augments.ts +
+ * retrieval-control-plane.ts without modifying any existing file.
+ */
+
+import type { CrawlPayload, ScheduleContext, RetrievalPolicy } from "./retrieval-control-plane";
+
+// ═══════════════════════════════════════════════════════════════════════════
+// TYPES
+// ═══════════════════════════════════════════════════════════════════════════
+
+export type StructuredSourceKind =
+  | "openalex"
+  | "crossref"
+  | "semantic-scholar"
+  | "pubmed"
+  | "arxiv"
+  | "wikipedia"
+  | "internet-archive"
+  | "rss-atom"
+  | "partial-spa"
+  | "none";
+
+export interface StructuredItem {
+  kind: StructuredSourceKind;
+  title: string;
+  abstract?: string;
+  authors?: string[];
+  publishedDate?: string;
+  doi?: string;
+  externalUrl?: string;
+  openAccessUrl?: string;
+  citationCount?: number;
+  pageContent: string;
+  markdown: string;
+  metadata: Record<string, string | number | boolean | undefined>;
+  confidence: number;
+}
+
+export interface StructuredQueryResult {
+  kind: StructuredSourceKind;
+  items: StructuredItem[];
+  apiUrl: string;
+  totalAvailable?: number;
+}
+
+export interface StructuredAdapterOptions {
+  signal?: AbortSignal;
+  limitPerSource?: number;
+  sources?: StructuredSourceKind[];
+  openAlexApiKey?: string;
+  deduplicate?: boolean;
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// DOMAIN ROUTING
+// ═══════════════════════════════════════════════════════════════════════════
+
+const STRUCTURED_DOMAINS: Record<string, StructuredSourceKind> = {
+  "arxiv.org":                  "arxiv",
+  "export.arxiv.org":           "arxiv",
+  "openalex.org":               "openalex",
+  "api.openalex.org":           "openalex",
+  "crossref.org":               "crossref",
+  "api.crossref.org":           "crossref",
+  "doi.org":                    "crossref",
+  "semanticscholar.org":        "semantic-scholar",
+  "api.semanticscholar.org":    "semantic-scholar",
+  "pubmed.ncbi.nlm.nih.gov":    "pubmed",
+  "ncbi.nlm.nih.gov":           "pubmed",
+  "eutils.ncbi.nlm.nih.gov":    "pubmed",
+  "en.wikipedia.org":           "wikipedia",
+  "wikipedia.org":              "wikipedia",
+  "archive.org":                "internet-archive",
+  "scholar.archive.org":        "internet-archive",
+};
+
+function hostOf(url: string): string {
+  try { return new URL(url).hostname.toLowerCase().replace(/^www\./, ""); }
+  catch { return ""; }
+}
+
+export function detectStructuredSource(url: string): StructuredSourceKind {
+  const host = hostOf(url);
+  if (STRUCTURED_DOMAINS[host]) return STRUCTURED_DOMAINS[host];
+  for (const [suffix, kind] of Object.entries(STRUCTURED_DOMAINS)) {
+    if (host.endsWith("." + suffix)) return kind;
+  }
+  return "none";
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// POLITE-FETCH
+// ═══════════════════════════════════════════════════════════════════════════
+
+const API_LAST_CALL = new Map<string, number>();
+const API_MIN_INTERVAL_MS = 1_000;
+
+async function politeFetch(url: string, signal?: AbortSignal, accept = "application/json"): Promise<Response | null> {
+  const apiHost = hostOf(url);
+  const last = API_LAST_CALL.get(apiHost) ?? 0;
+  const wait = Math.max(0, last + API_MIN_INTERVAL_MS - Date.now());
+  if (wait > 0) await new Promise<void>((r) => setTimeout(r, wait));
+  API_LAST_CALL.set(apiHost, Date.now());
+  try {
+    const res = await fetch(url, {
+      signal,
+      credentials: "omit",
+      referrerPolicy: "no-referrer",
+      headers: { Accept: accept },
+    });
+    return res.ok ? res : null;
+  } catch { return null; }
+}
+
+async function politeJson<T>(url: string, signal?: AbortSignal): Promise<T | null> {
+  const res = await politeFetch(url, signal, "application/json");
+  if (!res) return null;
+  try { return (await res.json()) as T; } catch { return null; }
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// TEXT BUILDERS
+// ═══════════════════════════════════════════════════════════════════════════
+
+function safeString(v: unknown, fallback = ""): string {
+  if (v == null) return fallback;
+  if (typeof v === "string") return v.trim();
+  if (typeof v === "number") return String(v);
+  if (Array.isArray(v)) return v.map((x) => safeString(x)).join(", ");
+  return fallback;
+}
+
+function buildPageContent(item: Omit<StructuredItem, "pageContent" | "markdown">): string {
+  const lines: string[] = [];
+  if (item.title) lines.push(`# ${item.title}`);
+  if (item.authors?.length) lines.push(`Authors: ${item.authors.slice(0, 6).join(", ")}`);
+  if (item.publishedDate) lines.push(`Published: ${item.publishedDate}`);
+  if (item.doi) lines.push(`DOI: https://doi.org/${item.doi}`);
+  if (item.externalUrl) lines.push(`URL: ${item.externalUrl}`);
+  if (item.openAccessUrl && item.openAccessUrl !== item.externalUrl) {
+    lines.push(`Open Access: ${item.openAccessUrl}`);
+  }
+  if (item.citationCount != null) lines.push(`Citations: ${item.citationCount}`);
+  if (item.abstract) lines.push(`\n## Abstract\n\n${item.abstract}`);
+  return lines.join("\n").trim();
+}
+
+function makeItem(
+  kind: StructuredSourceKind,
+  partial: Omit<StructuredItem, "pageContent" | "markdown" | "kind" | "confidence">,
+  confidence = 0.85,
+): StructuredItem {
+  const full = { ...partial, kind, confidence, pageContent: "", markdown: "" } as StructuredItem;
+  full.pageContent = buildPageContent(full);
+  full.markdown = full.pageContent;
+  return full;
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// ADAPTERS
+// ═══════════════════════════════════════════════════════════════════════════
+
+export async function queryCrossref(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 8, 20);
+  const url = `https://api.crossref.org/works?query=${encodeURIComponent(query)}&rows=${limit}&mailto=oss-rcp@example.com`;
+  const data = await politeJson<{ message?: { items?: any[]; "total-results"?: number } }>(url, opts?.signal);
+  if (!data?.message?.items) return { kind: "crossref", items: [], apiUrl: url };
+  const items = data.message.items.map((w) => makeItem("crossref", {
+    title: (w.title ?? ["Untitled"])[0],
+    abstract: w.abstract ? w.abstract.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() : undefined,
+    authors: (w.author ?? []).map((a: any) => [a.given, a.family].filter(Boolean).join(" ")),
+    publishedDate: (w["published-print"] ?? w["published-online"])?.["date-parts"]?.[0]?.filter(Boolean).join("-"),
+    doi: w.DOI,
+    externalUrl: w.URL ?? (w.DOI ? `https://doi.org/${w.DOI}` : undefined),
+    citationCount: w["is-referenced-by-count"],
+    metadata: { doi: w.DOI, citations: w["is-referenced-by-count"] },
+  }));
+  return { kind: "crossref", items, apiUrl: url, totalAvailable: data.message["total-results"] };
+}
+
+export async function querySemanticScholar(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 8, 20);
+  const fields = "paperId,title,abstract,year,authors,externalIds,openAccessPdf,citationCount,url";
+  const url = `https://api.semanticscholar.org/graph/v1/paper/search?query=${encodeURIComponent(query)}&limit=${limit}&fields=${fields}`;
+  const data = await politeJson<{ data?: any[]; total?: number }>(url, opts?.signal);
+  if (!data?.data) return { kind: "semantic-scholar", items: [], apiUrl: url };
+  const items = data.data.map((p) => makeItem("semantic-scholar", {
+    title: p.title ?? "Untitled",
+    abstract: p.abstract,
+    authors: (p.authors ?? []).map((a: any) => a.name ?? "").filter(Boolean),
+    publishedDate: p.year ? String(p.year) : undefined,
+    doi: p.externalIds?.DOI,
+    externalUrl: p.url ?? (p.externalIds?.DOI ? `https://doi.org/${p.externalIds.DOI}` : undefined),
+    openAccessUrl: p.openAccessPdf?.url,
+    citationCount: p.citationCount,
+    metadata: { paperId: p.paperId, arxivId: p.externalIds?.ArXiv },
+  }));
+  return { kind: "semantic-scholar", items, apiUrl: url, totalAvailable: data.total };
+}
+
+export function invertedIndexToText(inv: Record<string, number[]> | undefined): string {
+  if (!inv) return "";
+  const words: string[] = [];
+  for (const [word, positions] of Object.entries(inv)) {
+    for (const p of positions) if (p < 10_000) words[p] = word;
+  }
+  return words.filter(Boolean).join(" ");
+}
+
+export async function queryOpenAlex(query: string, opts?: { signal?: AbortSignal; limit?: number; apiKey?: string }): Promise<StructuredQueryResult> {
+  if (!opts?.apiKey) return { kind: "openalex", items: [], apiUrl: "" };
+  const limit = Math.min(opts.limit ?? 8, 25);
+  const fields = "id,title,doi,publication_date,authorships,abstract_inverted_index,best_oa_location,cited_by_count";
+  const url = `https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}&select=${fields}&api_key=${encodeURIComponent(opts.apiKey)}`;
+  const data = await politeJson<{ results?: any[]; meta?: { count?: number } }>(url, opts.signal);
+  if (!data?.results) return { kind: "openalex", items: [], apiUrl: url };
+  const items = data.results.map((w) => {
+    const oaUrl = w.best_oa_location?.pdf_url ?? w.best_oa_location?.landing_page_url;
+    return makeItem("openalex", {
+      title: w.title ?? "Untitled",
+      abstract: invertedIndexToText(w.abstract_inverted_index) || undefined,
+      authors: (w.authorships ?? []).map((a: any) => a.author?.display_name ?? "").filter(Boolean),
+      publishedDate: w.publication_date,
+      doi: w.doi?.replace(/^https?:\/\/doi\.org\//, ""),
+      externalUrl: w.doi ?? w.id,
+      openAccessUrl: oaUrl,
+      citationCount: w.cited_by_count,
+      metadata: { openalexId: w.id, hasOA: !!oaUrl },
+    }, 0.90);
+  });
+  return { kind: "openalex", items, apiUrl: url, totalAvailable: data.meta?.count };
+}
+
+export async function queryPubMed(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 8, 20);
+  const esearchUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=${encodeURIComponent(query)}&retmax=${limit}&retmode=json`;
+  const esearch = await politeJson<{ esearchresult?: { idlist?: string[]; count?: string } }>(esearchUrl, opts?.signal);
+  const ids = esearch?.esearchresult?.idlist ?? [];
+  if (!ids.length) return { kind: "pubmed", items: [], apiUrl: esearchUrl };
+  const esummaryUrl = `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${ids.join(",")}&retmode=json`;
+  const summary = await politeJson<{ result?: Record<string, any> }>(esummaryUrl, opts?.signal);
+  const result = summary?.result ?? {};
+  const items = ids.flatMap((id) => {
+    const s = result[id]; if (!s) return [];
+    return [makeItem("pubmed", {
+      title: s.title ?? "Untitled",
+      authors: (s.authors ?? []).map((a: any) => a.name ?? "").filter(Boolean),
+      publishedDate: (s.sortpubdate ?? s.epubdate)?.split(" ")[0],
+      doi: s.elocationid?.startsWith("doi: ") ? s.elocationid.slice(5) : undefined,
+      externalUrl: `https://pubmed.ncbi.nlm.nih.gov/${id}/`,
+      metadata: { pmid: id, journal: s.fulljournalname },
+    })];
+  });
+  return { kind: "pubmed", items, apiUrl: esearchUrl, totalAvailable: esearch?.esearchresult?.count ? Number(esearch.esearchresult.count) : undefined };
+}
+
+export async function queryArXiv(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 8, 20);
+  const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&max_results=${limit}`;
+  const res = await politeFetch(url, opts?.signal, "application/atom+xml,application/xml");
+  if (!res) return { kind: "arxiv", items: [], apiUrl: url };
+  const xml = await res.text(); if (typeof DOMParser === "undefined") return { kind: "arxiv", items: [], apiUrl: url };
+  let doc: Document; try { doc = new DOMParser().parseFromString(xml, "application/xml"); } catch { return { kind: "arxiv", items: [], apiUrl: url }; }
+  const ns = "http://www.w3.org/2005/Atom";
+  const entries = Array.from(doc.getElementsByTagNameNS(ns, "entry"));
+  const items = entries.map((e) => {
+    const get = (tag: string) => e.getElementsByTagNameNS(ns, tag)[0]?.textContent?.trim() ?? "";
+    const links = Array.from(e.getElementsByTagNameNS(ns, "link"));
+    const pdfLink = links.find((l) => l.getAttribute("title") === "pdf")?.getAttribute("href") ?? "";
+    const absLink = links.find((l) => l.getAttribute("rel") === "alternate")?.getAttribute("href") ?? "";
+    const authors = Array.from(e.getElementsByTagNameNS(ns, "author")).map((a) => a.getElementsByTagNameNS(ns, "name")[0]?.textContent?.trim() ?? "").filter(Boolean);
+    const arXivId = get("id").split("/abs/").pop() ?? "";
+    return makeItem("arxiv", { title: get("title").replace(/\s+/g, " "), abstract: get("summary").replace(/\s+/g, " "), authors: authors.length ? authors : undefined, publishedDate: get("published").slice(0, 10), externalUrl: absLink || (arXivId ? `https://arxiv.org/abs/${arXivId}` : undefined), openAccessUrl: pdfLink || (arXivId ? `https://arxiv.org/pdf/${arXivId}` : undefined), metadata: { arxivId: arXivId } });
+  });
+  return { kind: "arxiv", items, apiUrl: url };
+}
+
+export async function queryWikipedia(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 5, 10);
+  const searchUrl = `https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=${encodeURIComponent(query)}&srlimit=${limit}&srprop=snippet&format=json&origin=*`;
+  const data = await politeJson<{ query?: { search?: any[] } }>(searchUrl, opts?.signal);
+  if (!data?.query?.search) return { kind: "wikipedia", items: [], apiUrl: searchUrl };
+  const items = await Promise.all(data.query.search.map(async (s) => {
+    const extractUrl = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(s.title ?? "")}&prop=extracts&exintro&explaintext&exsentences=10&format=json&origin=*`;
+    const ext = await politeJson<{ query?: { pages?: Record<string, any> } }>(extractUrl, opts?.signal);
+    const page = Object.values(ext?.query?.pages ?? {})[0] as any;
+    return makeItem("wikipedia", { title: s.title ?? "Untitled", abstract: page?.extract || (s.snippet ?? "").replace(/<[^>]+>/g, ""), externalUrl: page?.canonicalurl ?? `https://en.wikipedia.org/wiki/${encodeURIComponent(s.title ?? "")}`, metadata: { pageid: s.pageid } }, 0.80);
+  }));
+  return { kind: "wikipedia", items, apiUrl: searchUrl };
+}
+
+export async function queryInternetArchive(query: string, opts?: { signal?: AbortSignal; limit?: number }): Promise<StructuredQueryResult> {
+  const limit = Math.min(opts?.limit ?? 8, 25);
+  const url = `https://scholar.archive.org/search?q=${encodeURIComponent(query)}&limit=${limit}&format=json`;
+  const data = await politeJson<{ results?: any[]; count_found?: number }>(url, opts?.signal);
+  if (!data?.results) return { kind: "internet-archive", items: [], apiUrl: url };
+  const items = data.results.map((r) => {
+    const b = r.biblio ?? {};
+    return makeItem("internet-archive", { title: b.title ?? "Untitled", abstract: r.abstracts?.[0]?.body, authors: b.contributor_raw?.filter(Boolean), publishedDate: b.release_year ? String(b.release_year) : undefined, doi: b.doi, externalUrl: b.release_url ?? (b.doi ? `https://doi.org/${b.doi}` : undefined), metadata: { key: r.key } }, 0.82);
+  });
+  return { kind: "internet-archive", items, apiUrl: url, totalAvailable: data.count_found };
+}
+
+export function extractSpaStructuredData(html: string, sourceUrl: string): StructuredItem | null {
+  if (typeof DOMParser === "undefined") return null;
+  let doc: Document; try { const safe = html.replace(/<script\b(?![^>]*type=["']application\/ld\+json["'])[^>]*>[\s\S]*?<\/script>/gi, " ").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, " "); doc = new DOMParser().parseFromString(safe, "text/html"); } catch { return null; }
+  const ldScripts = Array.from(doc.querySelectorAll('script[type="application/ld+json"]'));
+  for (const script of ldScripts) {
+    try {
+      const data = JSON.parse(script.textContent ?? ""); const items = Array.isArray(data) ? data : [data];
+      for (const d of items) {
+        const type = safeString(d["@type"]).toLowerCase(); if (!["article", "newsarticle", "blogposting", "scholarlyarticle", "webpage"].some((t) => type.includes(t))) continue;
+        const title = safeString(d.headline ?? d.name), abstract = safeString(d.description ?? d.abstract ?? d.articleBody).slice(0, 2000);
+        const authorRaw = d.author, authors: string[] = [];
+        if (Array.isArray(authorRaw)) for (const a of authorRaw) authors.push(safeString(typeof a === "object" ? a?.name ?? a : a));
+        else if (authorRaw) authors.push(safeString(typeof authorRaw === "object" ? authorRaw?.name ?? authorRaw : authorRaw));
+        if (!title && !abstract) continue;
+        return makeItem("partial-spa", { title: title || "Untitled", abstract: abstract || undefined, authors: authors.filter(Boolean), publishedDate: safeString(d.datePublished ?? d.dateCreated), externalUrl: safeString(d.url ?? d.mainEntityOfPage?.["@id"] ?? sourceUrl), doi: safeString(d.sameAs).includes("doi.org") ? safeString(d.sameAs).split("doi.org/")[1] : undefined, metadata: { schemaType: d["@type"], sourceUrl } }, 0.55);
+      }
+    } catch {}
+  }
+  const og: Record<string, string> = {}; doc.querySelectorAll("meta[property]").forEach(m => { const p = m.getAttribute("property")||""; if(p.startsWith("og:")) og[p.slice(3)] = m.getAttribute("content")||""; });
+  if (og.title || og.description) return makeItem("partial-spa", { title: og.title || doc.querySelector("title")?.textContent?.trim() || "Untitled", abstract: og.description, publishedDate: og["article:published_time"] || undefined, externalUrl: og.url || sourceUrl, metadata: { og_site_name: og.site_name, sourceUrl } }, 0.40);
+  return null;
+}
+
+export async function structuredSearch(query: string, opts?: StructuredAdapterOptions): Promise<{ items: StructuredItem[]; totalFound: number; queriedSources: StructuredSourceKind[] }> {
+  const sources = opts?.sources ?? ["crossref", "semantic-scholar", "arxiv", "pubmed", "wikipedia", "internet-archive"];
+  const limit = opts?.limitPerSource ?? 6; const runners: Array<Promise<StructuredQueryResult>> = [];
+  if (sources.includes("crossref")) runners.push(queryCrossref(query, { signal: opts?.signal, limit }));
+  if (sources.includes("semantic-scholar")) runners.push(querySemanticScholar(query, { signal: opts?.signal, limit }));
+  if (sources.includes("arxiv")) runners.push(queryArXiv(query, { signal: opts?.signal, limit }));
+  if (sources.includes("pubmed")) runners.push(queryPubMed(query, { signal: opts?.signal, limit }));
+  if (sources.includes("wikipedia")) runners.push(queryWikipedia(query, { signal: opts?.signal, limit }));
+  if (sources.includes("internet-archive")) runners.push(queryInternetArchive(query, { signal: opts?.signal, limit }));
+  if (sources.includes("openalex") && opts?.openAlexApiKey) runners.push(queryOpenAlex(query, { signal: opts?.signal, limit, apiKey: opts.openAlexApiKey }));
+  const settled = await Promise.allSettled(runners), seen = new Set<string>(), all: StructuredItem[] = [], queried: StructuredSourceKind[] = [];
+  for (const r of settled) { if (r.status !== "fulfilled") continue; queried.push(r.value.kind); for (const item of r.value.items) { const key = item.doi ?? item.externalUrl; if (key && !seen.has(key)) { seen.add(key); all.push(item); } } }
+  return { items: all, totalFound: all.length, queriedSources: queried };
+}
+
+export function withStructuredSourceInterception<T>(reader: (url: string, ctx: ScheduleContext, policy: RetrievalPolicy) => Promise<CrawlPayload<T>>, adapterOpts?: Pick<StructuredAdapterOptions, "signal" | "openAlexApiKey">): (url: string, ctx: ScheduleContext, policy: RetrievalPolicy) => Promise<CrawlPayload<T>> {
+  return async (url, ctx, policy) => {
+    const kind = detectStructuredSource(url);
+    if (kind !== "none") {
+      const item = await resolveStructuredUrl(url, kind, ctx.signal ?? adapterOpts?.signal, adapterOpts?.openAlexApiKey);
+      if (item && item.pageContent.length >= 80) return { value: item as unknown as T, text: item.pageContent, markdown: item.markdown, contentType: "text/markdown", canonicalUrl: item.externalUrl ?? url, bytesRead: item.pageContent.length };
+    }
+    return await reader(url, ctx, policy);
+  };
+}
+
+async function resolveStructuredUrl(url: string, kind: StructuredSourceKind, signal?: AbortSignal, openAlexApiKey?: string): Promise<StructuredItem | null> {
+  try {
+    switch (kind) {
+      case "arxiv": { const id = new URL(url).pathname.replace(/^\/(?:abs|pdf|html)\//, "").replace(/v\d+$/, ""); return id ? (await queryArXiv(`id:${id}`, { signal, limit: 1 })).items[0] || null : null; }
+      case "pubmed": { const pmid = url.match(/\/(\d+)\/?(?:\?|$)/)?.[1]; return pmid ? (await queryPubMed(pmid, { signal, limit: 1 })).items[0] || null : null; }
+      case "crossref": { const doi = url.match(/doi\.org\/(.+)$/)?.[1]; return doi ? (await queryCrossref(doi, { signal, limit: 1 })).items[0] || null : null; }
+      case "openalex": return openAlexApiKey ? (await queryOpenAlex(url, { signal, limit: 1, apiKey: openAlexApiKey })).items[0] || null : null;
+      case "semantic-scholar": return (await querySemanticScholar(url, { signal, limit: 1 })).items[0] || null;
+      case "wikipedia": { const title = decodeURIComponent(new URL(url).pathname.replace(/^\/wiki\//, "")); return (await queryWikipedia(title, { signal, limit: 1 })).items[0] || null; }
+      case "internet-archive": return (await queryInternetArchive(url, { signal, limit: 1 })).items[0] || null;
+      default: return null;
+    }
+  } catch { return null; }
+}

package/src/lib/scraper-vnext/structured-source-adapter.ts

@@ -0,0 +1,34 @@
+/** Alias-reachable T0 structured search wrapper. */
+export * from "./structured-source-adapter.orig";
+
+import { structuredSearch as packageStructuredSearch } from "./structured-source-adapter.orig";
+import { dedupeSources, filterRelevantSourcesAny, hardenRetrievalQuery } from "@/lib/debug/retrieval-hardener";
+
+type StructuredOpts = Parameters<typeof packageStructuredSearch>[1];
+type StructuredResult = Awaited<ReturnType<typeof packageStructuredSearch>>;
+
+export async function structuredSearch(query: string, opts?: StructuredOpts): Promise<StructuredResult> {
+  const hardened = hardenRetrievalQuery(query, "general");
+  const selected = hardened.alternatives.slice(0, 3);
+  const results = await Promise.all(
+    selected.map((q) => packageStructuredSearch(q.q, opts).catch(() => ({ items: [], totalFound: 0, queriedSources: [] } as StructuredResult)))
+  );
+  const allItems = dedupeSources(results.flatMap((r) => r.items));
+  // Accept if coherent with ANY dispatched facet query; reject only if all say drift.
+  const accepted = filterRelevantSourcesAny(allItems, selected, 0.2).accepted;
+  return {
+    items: accepted,
+    totalFound: results.reduce((n, r) => n + Number(r.totalFound ?? 0), 0),
+    queriedSources: [...new Set(results.flatMap((r) => r.queriedSources))],
+  } as StructuredResult;
+}
+
+
+// [unify.mjs] Explicit type re-exports for Rollup resolution
+export type { StructuredItem } from "./structured-source-adapter.orig";
+
+
+// [unify.mjs] Emergency mocks for symbols lost from package/wrapper
+export function buildAmpCacheUrl(url: string): string { return "https://cdn.ampproject.org/c/s/" + url.replace(/^https?:\/\//, ""); }
+export async function fetchViaAmpCache(url: string, init?: any): Promise<Response> { return fetch(buildAmpCacheUrl(url), init); }
+export async function wrappedStructuredChallengeReader(url: string): Promise<any> { return { url, content: "", title: "Fallback" }; }

package/src/lib/scraper-vnext/terminal-complete.ts

@@ -0,0 +1,224 @@
+/**
+ * terminal-complete.ts
+ * ============================================================================
+ * FINAL TERMINAL SYNTHESIS
+ *
+ * Chain:
+ *   canonical-portfolio-orchestrator.ts  — multi-engine portfolio
+ *   portfolio-consensus-adjudicator.ts   — family-aware corroboration
+ *   portfolio-consensus-memory.ts        — honest cross-session memory
+ *   portfolio-terminal-governor.ts       — corrected adaptive quarantine
+ *   this file                            — structured pre-fetch + real-time
+ *                                           independence + marginal advisory
+ */
+
+import {
+  ENGINE_FAMILY,
+  familyOf,
+  groundWithAdaptivePortfolioConsensus,
+  runPortfolioConsensusMemoryDiagnostics,
+  type AdaptivePortfolioOptions,
+  type PortfolioMemorySnapshot,
+} from "./portfolio-consensus-memory";
+import {
+  groundWithTerminalPortfolioGovernor,
+  runPortfolioTerminalGovernorDiagnostics,
+} from "./portfolio-terminal-governor";
+import {
+  runPortfolioConsensusDiagnostics,
+  type ConsensusConfidence,
+  type ConsensusSelection,
+  type PortfolioConsensusResult,
+} from "./portfolio-consensus-adjudicator";
+import {
+  runCanonicalPortfolioDiagnostics,
+  type CanonicalLaneId,
+  type LaneExecution,
+  type NormalizedLaneResult,
+  type StandardSource,
+} from "./canonical-portfolio-orchestrator";
+import {
+  runSpaRescueBridgeDiagnostics,
+} from "./spa-rescue-bridge";
+import {
+  structuredSearch,
+  type StructuredAdapterOptions,
+  type StructuredItem,
+} from "./structured-source-adapter";
+
+export interface TerminalCompleteOptions extends AdaptivePortfolioOptions {
+  structuredPreFetch?: boolean;
+  openAlexApiKey?: string;
+  terminalGovernor?: boolean;
+  memoryProfile?: string;
+  adaptiveQuarantine?: boolean;
+  quarantineMinSamples?: number;
+  quarantineCooldownMs?: number;
+  minimumEnabledLanes?: number;
+  minimumIndependentFamilies?: number;
+}
+
+export interface TerminalPortfolioIndependence {
+  distinctFamilies: number;
+  fulfilledFamilies: string[];
+  effectiveWitnessCount: number;
+}
+
+export interface TerminalMarginalGain {
+  fulfilledCount: number;
+  failedCount: number;
+  extraLanesChangedWinner: boolean;
+  attestedDelta: number;
+  assessment: "high-value" | "moderate-value" | "low-value" | "single-lane-only";
+}
+
+export interface TerminalCompleteResult extends PortfolioConsensusResult {
+  prunedLanes: CanonicalLaneId[];
+  memorySnapshot: PortfolioMemorySnapshot;
+  structuredItems: StructuredItem[];
+  portfolioIndependence: TerminalPortfolioIndependence;
+  marginalGain: TerminalMarginalGain;
+}
+
+function computeRealTimeIndependence(laneResults: LaneExecution[]): TerminalPortfolioIndependence {
+  const families = new Set<string>();
+  for (const lr of laneResults) {
+    if (lr.status === "fulfilled") families.add(familyOf(lr.id));
+  }
+  const sorted = Array.from(families).sort();
+  return { distinctFamilies: families.size, fulfilledFamilies: sorted, effectiveWitnessCount: families.size };
+}
+
+function computeMarginalGain(
+  laneResults: LaneExecution[],
+  canonicalWinnerId: CanonicalLaneId | undefined,
+  consensusWinnerId: CanonicalLaneId | undefined,
+): TerminalMarginalGain {
+  const fulfilled = laneResults.filter((lr) => lr.status === "fulfilled");
+  const failedCount = laneResults.length - fulfilled.length;
+  if (fulfilled.length <= 1) {
+    return { fulfilledCount: fulfilled.length, failedCount, extraLanesChangedWinner: false, attestedDelta: 0, assessment: "single-lane-only" };
+  }
+  const changedWinner = canonicalWinnerId !== undefined && consensusWinnerId !== undefined && canonicalWinnerId !== consensusWinnerId;
+  const effectiveWinnerId = consensusWinnerId ?? canonicalWinnerId;
+  const winnerLane = fulfilled.find((lr) => lr.id === effectiveWinnerId);
+  const winnerAttested = winnerLane?.normalized?.attestedCount ?? 0;
+  const bestOtherAttested = Math.max(0, ...fulfilled.filter((lr) => lr.id !== effectiveWinnerId).map((lr) => lr.normalized?.attestedCount ?? 0));
+  const attestedDelta = winnerAttested - bestOtherAttested;
+  const assessment = changedWinner || attestedDelta >= 2 ? "high-value" : attestedDelta >= 1 || fulfilled.length >= 3 ? "moderate-value" : "low-value";
+  return { fulfilledCount: fulfilled.length, failedCount, extraLanesChangedWinner: changedWinner, attestedDelta, assessment };
+}
+
+function buildEmpty(provider: string): TerminalCompleteResult {
+  return {
+    ok: false,
+    provider,
+    selection: "canonical" as ConsensusSelection,
+    count: 0,
+    sources: [],
+    evidenceBlock: "",
+    consensusConfidence: "none" as ConsensusConfidence,
+    consensusCaveat: "",
+    laneConsensus: [],
+    claimCorroboration: [],
+    decisionLedger: [],
+    agreement: [],
+    laneResults: [],
+    winnerRaw: undefined,
+    prunedLanes: [],
+    memorySnapshot: { laneReliability: [], familyPriors: [] },
+    structuredItems: [],
+    portfolioIndependence: { distinctFamilies: 0, fulfilledFamilies: [], effectiveWitnessCount: 0 },
+    marginalGain: { fulfilledCount: 0, failedCount: 0, extraLanesChangedWinner: false, attestedDelta: 0, assessment: "single-lane-only" },
+  };
+}
+
+export async function groundTerminalComplete(question: string, opts?: TerminalCompleteOptions): Promise<TerminalCompleteResult> {
+  const dbg = opts?.onDebug ?? (() => {});
+  const normalized = (question ?? "").replace(/\s+/g, " ").trim();
+  if (!normalized) return buildEmpty("terminal-complete(empty)");
+  const doStructured = opts?.structuredPreFetch !== false;
+
+  const structuredPromise = doStructured
+    ? structuredSearch(normalized, { signal: opts?.signal, limitPerSource: 6, openAlexApiKey: opts?.openAlexApiKey } satisfies StructuredAdapterOptions).catch(() => ({ items: [] as StructuredItem[], queriedSources: [] as string[], totalFound: 0 }))
+    : Promise.resolve({ items: [] as StructuredItem[], queriedSources: [] as string[], totalFound: 0 });
+
+  dbg("terminal-complete: portfolio memory governor");
+  const portfolioResult = opts?.terminalGovernor === false
+    ? await groundWithAdaptivePortfolioConsensus(normalized, { ...opts, onDebug: (m) => dbg(`[adaptive] ${m}`) })
+    : await groundWithTerminalPortfolioGovernor(normalized, { ...opts, onDebug: (m) => dbg(`[governor] ${m}`) });
+
+  const structured = await structuredPromise;
+  dbg(`terminal-complete: structured → ${structured.items.length} items from [${structured.queriedSources?.join(",") ?? ""}]`);
+
+  const portfolioIndependence = computeRealTimeIndependence(portfolioResult.laneResults);
+  const marginalGain = computeMarginalGain(portfolioResult.laneResults, portfolioResult.canonicalWinnerLane, portfolioResult.consensusWinnerLane);
+
+  const seenUrls = new Set<string>(portfolioResult.sources.map((s) => s.url));
+  const structuredAsSources: StandardSource[] = [];
+  for (const item of structured.items) {
+    const url = item.externalUrl ?? (item.doi ? `https://doi.org/${item.doi}` : undefined) ?? `structured:${item.kind}:${item.title.slice(0, 40)}`;
+    if (!url || seenUrls.has(url)) continue;
+    seenUrls.add(url);
+    structuredAsSources.push({ title: item.title || "Structured Item", url, content: (item.pageContent || item.abstract || "").slice(0, 1800) });
+  }
+  const mergedSources = [...portfolioResult.sources, ...structuredAsSources];
+
+  let evidenceBlock = portfolioResult.evidenceBlock;
+  if (structured.items.length > 0 && evidenceBlock.length > 0) {
+    const lines = structured.items.slice(0, 6).map((it, i) => `[STRUCTURED:${it.kind.toUpperCase()}:${i + 1}] ${it.title}\n${(it.pageContent || "").slice(0, 360)}`).join("\n\n");
+    const insertAt = evidenceBlock.lastIndexOf("END RETRIEVED CONTENT");
+    if (insertAt >= 0) {
+      evidenceBlock = evidenceBlock.slice(0, insertAt) + "BEGIN STRUCTURED API ITEMS\n" + lines + "\nEND STRUCTURED API ITEMS\n\n" + evidenceBlock.slice(insertAt);
+    }
+  }
+
+  const provider = `terminal-complete(${portfolioResult.provider}+structured:${structured.items.length}+pruned:${portfolioResult.prunedLanes.length}+families:${portfolioIndependence.distinctFamilies})`;
+  return { ...portfolioResult, sources: mergedSources, count: mergedSources.length, evidenceBlock, provider, structuredItems: structured.items, portfolioIndependence, marginalGain };
+}
+
+export async function runTerminalCompleteDiagnostics(): Promise<{
+  ok: boolean;
+  suites: Array<{ name: string; ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }>;
+}> {
+  const suites: Array<{ name: string; ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }> = [];
+  const run = async (name: string, fn: (() => Promise<{ ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }>) | (() => { ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> })) => {
+    try {
+      const result = await fn();
+      suites.push({ name, ok: result.ok, checks: result.checks });
+    } catch (e) {
+      suites.push({ name, ok: false, checks: [{ id: "suite-error", passed: false, detail: e instanceof Error ? e.message : String(e) }] });
+    }
+  };
+  await run("canonical-portfolio-orchestrator", () => runCanonicalPortfolioDiagnostics());
+  await run("portfolio-consensus-adjudicator", () => runPortfolioConsensusDiagnostics());
+  await run("portfolio-consensus-memory", () => runPortfolioConsensusMemoryDiagnostics());
+  await run("portfolio-terminal-governor", () => runPortfolioTerminalGovernorDiagnostics());
+  await run("spa-rescue-bridge", () => runSpaRescueBridgeDiagnostics());
+
+  const synChecks: Array<{ id: string; passed: boolean; detail: string }> = [];
+  const add = (id: string, passed: boolean, detail: string) => synChecks.push({ id, passed, detail });
+  const mockFulfilled: LaneExecution[] = [
+    { id: "terminal-wire" as CanonicalLaneId, priority: 100, status: "fulfilled", elapsedMs: 300, normalized: { ok: true, acceptable: true, eligibleForWinner: true, provider: "tw", evidenceBlock: "x", sources: [], sourceCount: 1, claimCount: 1, structuredItemCount: 0, attestedCount: 2, supportedCount: 0, conflictedCount: 0, proof: "verified", qualityVector: [1, 4, 2, 0, 1, 1, 100] } as NormalizedLaneResult },
+    { id: "omni-nexus" as CanonicalLaneId, priority: 60, status: "fulfilled", elapsedMs: 400, normalized: { ok: true, acceptable: true, eligibleForWinner: true, provider: "on", evidenceBlock: "y", sources: [], sourceCount: 1, claimCount: 1, structuredItemCount: 0, attestedCount: 0, supportedCount: 1, conflictedCount: 0, proof: "bound", qualityVector: [1, 3, 0, 1, 1, 1, 100] } as NormalizedLaneResult },
+    { id: "sentinel-orchestrator" as CanonicalLaneId, priority: 95, status: "rejected", elapsedMs: 50, error: "timeout" },
+  ];
+  const ind = computeRealTimeIndependence(mockFulfilled);
+  add("independence-two-fulfilled-families", ind.distinctFamilies === 2 && ind.fulfilledFamilies.includes("conclave-core") && ind.fulfilledFamilies.includes("omni-standalone"), `families=${ind.fulfilledFamilies.join(",")}`);
+  add("independence-rejected-excluded", ind.effectiveWitnessCount === ind.distinctFamilies, `ewc=${ind.effectiveWitnessCount}`);
+  const mg1 = computeMarginalGain([mockFulfilled[0]], "terminal-wire", "terminal-wire");
+  add("marginal-single-lane", mg1.assessment === "single-lane-only", `assessment=${mg1.assessment}`);
+  const mg2 = computeMarginalGain([mockFulfilled[0], mockFulfilled[1]], "terminal-wire", "omni-nexus");
+  add("marginal-changed-winner-high-value", mg2.extraLanesChangedWinner && mg2.assessment === "high-value" && mg2.fulfilledCount === 2, `changed=${mg2.extraLanesChangedWinner}`);
+  add("engine-family-conclave-core", ENGINE_FAMILY["terminal-wire"] === "conclave-core" && ENGINE_FAMILY["sentinel-orchestrator"] === "conclave-core" && ENGINE_FAMILY["vanguard-packer"] === "conclave-core", `tw=${ENGINE_FAMILY["terminal-wire"]}`);
+  add("engine-family-standalone", ENGINE_FAMILY["sentinel-omega"] === "omega-standalone" && ENGINE_FAMILY["omni-nexus"] === "omni-standalone", `so=${ENGINE_FAMILY["sentinel-omega"]}`);
+  const empty = buildEmpty("test-empty");
+  add("empty-zero-state", !empty.ok && empty.count === 0 && empty.sources.length === 0 && empty.prunedLanes.length === 0 && empty.portfolioIndependence.distinctFamilies === 0, `ok=${empty.ok}`);
+  suites.push({ name: "terminal-complete-synthesis", ok: synChecks.every((c) => c.passed), checks: synChecks });
+  return { ok: suites.every((s) => s.ok), suites };
+}
+
+export { groundWithAdaptivePortfolioConsensus, getFamilyCorroborationPrior, getLaneReliability, getPortfolioMemorySnapshot, runPortfolioConsensusMemoryDiagnostics, familyOf, ENGINE_FAMILY, type AdaptivePortfolioOptions, type FamilyCorroborationPrior, type LaneReliability, type PortfolioMemorySnapshot } from "./portfolio-consensus-memory";
+export { groundWithPortfolioConsensus, runPortfolioConsensusDiagnostics, type ClaimCorroboration, type ConsensusConfidence, type ConsensusSelection, type DecisionLedgerEntry, type LaneConsensus, type PortfolioConsensusOptions, type PortfolioConsensusResult } from "./portfolio-consensus-adjudicator";
+export { groundWithCanonicalPortfolio, runCanonicalPortfolioDiagnostics, type AgreementCell, type CanonicalLaneId, type LaneExecution, type NormalizedLaneResult, type PortfolioGroundingResult, type PortfolioOptions, type StandardSource } from "./canonical-portfolio-orchestrator";
+export { runSpaRescueBridgeDiagnostics, withSpaRescueCause, withSpaRescueFromRawResponse } from "./spa-rescue-bridge";