2sagaco48 1.0.0

package/src/lib/scraper-vnext/terminal-final.ts

@@ -0,0 +1,277 @@
+/**
+ * terminal-final.ts
+ * ============================================================================
+ * TRUE TERMINAL — chains the corrected governor with structured pre-fetch,
+ * real-time independence, marginal gain, and a unified diagnostic hub.
+ *
+ * ADDITIVE ONLY. No canonical file modified. Browser/static-build compatible.
+ * ============================================================================ */
+
+import {
+  ENGINE_FAMILY,
+  familyOf,
+  groundWithTerminalPortfolioGovernor,
+  runPortfolioTerminalGovernorDiagnostics,
+  type TerminalGovernorOptions,
+  type TerminalGovernorResult,
+} from "./portfolio-terminal-governor";
+import {
+  runPortfolioConsensusDiagnostics,
+} from "./portfolio-consensus-adjudicator";
+import {
+  runCanonicalPortfolioDiagnostics,
+  type CanonicalLaneId,
+  type LaneExecution,
+  type StandardSource,
+} from "./canonical-portfolio-orchestrator";
+import { runSpaRescueBridgeDiagnostics } from "./spa-rescue-bridge";
+import {
+  structuredSearch,
+  type StructuredAdapterOptions,
+  type StructuredItem,
+} from "./structured-source-adapter";
+import { cachedValue } from "./retrieval-accelerator";
+
+export interface TerminalFinalOptions extends TerminalGovernorOptions {
+  structuredPreFetch?: boolean;
+  openAlexApiKey?: string;
+}
+
+export interface TerminalIndependence {
+  distinctFamilies: number;
+  fulfilledFamilies: string[];
+  effectiveWitnessCount: number;
+}
+
+export interface TerminalMarginalGain {
+  fulfilledCount: number;
+  failedCount: number;
+  extraLanesChangedWinner: boolean;
+  attestedDelta: number;
+  assessment: "high-value" | "moderate-value" | "low-value" | "single-lane-only";
+}
+
+export interface TerminalFinalResult extends TerminalGovernorResult {
+  structuredItems: StructuredItem[];
+  portfolioIndependence: TerminalIndependence;
+  marginalGain: TerminalMarginalGain;
+}
+
+function computeIndependence(lanes: LaneExecution[]): TerminalIndependence {
+  const families = new Set<string>();
+  for (const lane of lanes) {
+    if (lane.status === "fulfilled") families.add(familyOf(lane.id));
+  }
+  const fulfilledFamilies = Array.from(families).sort();
+  return {
+    distinctFamilies: fulfilledFamilies.length,
+    fulfilledFamilies,
+    effectiveWitnessCount: fulfilledFamilies.length,
+  };
+}
+
+function computeMarginalGain(
+  lanes: LaneExecution[],
+  canonicalWinner: CanonicalLaneId | undefined,
+  consensusWinner: CanonicalLaneId | undefined,
+): TerminalMarginalGain {
+  const fulfilled = lanes.filter((lane) => lane.status === "fulfilled");
+  const failedCount = lanes.length - fulfilled.length;
+  if (fulfilled.length <= 1) {
+    return { fulfilledCount: fulfilled.length, failedCount, extraLanesChangedWinner: false, attestedDelta: 0, assessment: "single-lane-only" };
+  }
+  const extraLanesChangedWinner = canonicalWinner !== undefined && consensusWinner !== undefined && canonicalWinner !== consensusWinner;
+  const winnerId = consensusWinner ?? canonicalWinner;
+  const winnerAttested = fulfilled.find((lane) => lane.id === winnerId)?.normalized?.attestedCount ?? 0;
+  const bestOther = Math.max(0, ...fulfilled.filter((lane) => lane.id !== winnerId).map((lane) => lane.normalized?.attestedCount ?? 0));
+  const attestedDelta = winnerAttested - bestOther;
+  const assessment = extraLanesChangedWinner || attestedDelta >= 2
+    ? "high-value"
+    : attestedDelta >= 1 || fulfilled.length >= 3
+      ? "moderate-value"
+      : "low-value";
+  return { fulfilledCount: fulfilled.length, failedCount, extraLanesChangedWinner, attestedDelta, assessment };
+}
+
+// RFC 9309 strict helper: 4xx robots responses allow crawling, while network
+// errors and 5xx responses deny. It is available to crawl callers without
+// changing the package policy module.
+const ROBOTS_TTL_MS = 24 * 60 * 60_000;
+
+function parseRobotsStarRules(text: string): Array<{ allow: boolean; path: string }> {
+  const rules: Array<{ allow: boolean; path: string }> = [];
+  let inStarGroup = false;
+  let sawUserAgent = false;
+  for (const rawLine of text.split(/\r?\n/)) {
+    const line = rawLine.replace(/#.*$/, "").trim();
+    const separator = line.indexOf(":");
+    if (!line || separator < 0) continue;
+    const key = line.slice(0, separator).trim().toLowerCase();
+    const value = line.slice(separator + 1).trim();
+    if (key === "user-agent") {
+      sawUserAgent = true;
+      inStarGroup = value === "*";
+    } else if (sawUserAgent && inStarGroup && value && (key === "allow" || key === "disallow")) {
+      rules.push({ allow: key === "allow", path: value });
+    }
+  }
+  return rules;
+}
+
+function robotsPathMatches(path: string, rulePath: string): boolean {
+  if (!rulePath) return false;
+  if (rulePath.endsWith("$")) return path === rulePath.slice(0, -1);
+  return path.startsWith(rulePath);
+}
+
+export function makeRfc9309RobotsDecision(): (url: string, signal?: AbortSignal) => Promise<"allow" | "deny" | "unknown"> {
+  return async (url, signal) => {
+    let origin = "";
+    let path = "";
+    try {
+      const parsed = new URL(url);
+      origin = parsed.origin;
+      path = `${parsed.pathname}${parsed.search}`;
+    } catch {
+      return "unknown";
+    }
+    const cached = await cachedValue(`terminal-rfc9309\u0000${origin}`, ROBOTS_TTL_MS, async () => {
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 4_000);
+        signal?.addEventListener("abort", () => controller.abort(), { once: true });
+        const response = await fetch(`${origin}/robots.txt`, {
+          signal: controller.signal,
+          credentials: "omit",
+          referrerPolicy: "no-referrer",
+        });
+        clearTimeout(timeout);
+        if (response.status >= 500) return { kind: "unreachable" as const, rules: [] };
+        if (!response.ok) return { kind: "unavailable" as const, rules: [] };
+        return { kind: "ok" as const, rules: parseRobotsStarRules((await response.text()).slice(0, 512 * 1024)) };
+      } catch {
+        return { kind: "unreachable" as const, rules: [] };
+      }
+    });
+    if (cached.value.kind === "unreachable") return "deny";
+    if (cached.value.kind === "unavailable") return "allow";
+    let best: { allow: boolean; path: string } | undefined;
+    for (const rule of cached.value.rules) {
+      if (robotsPathMatches(path, rule.path) && (!best || rule.path.length > best.path.length)) best = rule;
+    }
+    return best?.allow === false ? "deny" : "allow";
+  };
+}
+
+export async function groundTerminalFinal(
+  question: string,
+  opts?: TerminalFinalOptions,
+): Promise<TerminalFinalResult> {
+  const dbg = opts?.onDebug ?? (() => {});
+  const normalized = (question ?? "").replace(/\s+/g, " ").trim();
+  const doStructured = opts?.structuredPreFetch !== false;
+
+  const structuredPromise = doStructured && normalized
+    ? structuredSearch(normalized, {
+        signal: opts?.signal,
+        limitPerSource: 6,
+        openAlexApiKey: opts?.openAlexApiKey,
+      } satisfies StructuredAdapterOptions).catch(() => ({
+        items: [] as StructuredItem[],
+        queriedSources: [] as string[],
+        totalFound: 0,
+      }))
+    : Promise.resolve({ items: [] as StructuredItem[], queriedSources: [] as string[], totalFound: 0 });
+
+  dbg("terminal-final: groundWithTerminalPortfolioGovernor");
+  const governorResult = await groundWithTerminalPortfolioGovernor(normalized, {
+    ...opts,
+    onDebug: (message) => dbg(`[governor] ${message}`),
+  });
+  const structured = await structuredPromise;
+  const portfolioIndependence = computeIndependence(governorResult.laneResults);
+  const marginalGain = computeMarginalGain(
+    governorResult.laneResults,
+    governorResult.canonicalWinnerLane,
+    governorResult.consensusWinnerLane,
+  );
+
+  const seenUrls = new Set(governorResult.sources.map((source) => source.url));
+  const extraSources: StandardSource[] = [];
+  for (const item of structured.items) {
+    const url = item.externalUrl ?? (item.doi ? `https://doi.org/${item.doi}` : undefined) ?? `structured:${item.kind}:${item.title.slice(0, 40)}`;
+    if (!url || seenUrls.has(url)) continue;
+    seenUrls.add(url);
+    extraSources.push({
+      title: item.title || "Structured Item",
+      url,
+      content: (item.pageContent || item.abstract || "").slice(0, 1800),
+    });
+  }
+
+  let evidenceBlock = governorResult.evidenceBlock;
+  if (structured.items.length > 0 && evidenceBlock.length > 0) {
+    const items = structured.items.slice(0, 6).map((item, index) =>
+      `[STRUCTURED:${item.kind.toUpperCase()}:${index + 1}] ${item.title}\n${(item.pageContent || "").slice(0, 360)}`,
+    ).join("\n\n");
+    const insertion = evidenceBlock.lastIndexOf("END RETRIEVED CONTENT");
+    if (insertion >= 0) {
+      evidenceBlock = `${evidenceBlock.slice(0, insertion)}BEGIN STRUCTURED API ITEMS\n${items}\nEND STRUCTURED API ITEMS\n\n${evidenceBlock.slice(insertion)}`;
+    }
+  }
+
+  return {
+    ...governorResult,
+    provider: `terminal-final(${governorResult.provider}+structured:${structured.items.length}+families:${portfolioIndependence.distinctFamilies})`,
+    count: governorResult.sources.length + extraSources.length,
+    sources: [...governorResult.sources, ...extraSources],
+    evidenceBlock,
+    structuredItems: structured.items,
+    portfolioIndependence,
+    marginalGain,
+  };
+}
+
+export async function runTerminalFinalDiagnostics(): Promise<{
+  ok: boolean;
+  suites: Array<{ name: string; ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }>;
+}> {
+  const suites: Array<{ name: string; ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }> = [];
+  const run = async (name: string, fn: () => Promise<{ ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }> | { ok: boolean; checks: Array<{ id: string; passed: boolean; detail: string }> }) => {
+    try {
+      const result = await fn();
+      suites.push({ name, ...result });
+    } catch (error) {
+      suites.push({ name, ok: false, checks: [{ id: "suite-error", passed: false, detail: error instanceof Error ? error.message : String(error) }] });
+    }
+  };
+
+  await run("canonical-portfolio-orchestrator", () => runCanonicalPortfolioDiagnostics());
+  await run("portfolio-consensus-adjudicator", () => runPortfolioConsensusDiagnostics());
+  await run("portfolio-terminal-governor", () => runPortfolioTerminalGovernorDiagnostics());
+  await run("spa-rescue-bridge", () => runSpaRescueBridgeDiagnostics());
+
+  const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
+  const add = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });
+  const mock: LaneExecution[] = [
+    { id: "terminal-wire", priority: 100, status: "fulfilled", elapsedMs: 300, normalized: { ok: true, acceptable: true, eligibleForWinner: true, provider: "tw", evidenceBlock: "x", sources: [], sourceCount: 1, claimCount: 1, structuredItemCount: 0, attestedCount: 2, supportedCount: 0, conflictedCount: 0, proof: "verified", qualityVector: [1, 4, 2, 0, 1, 1, 100] } },
+    { id: "omni-nexus", priority: 60, status: "fulfilled", elapsedMs: 400, normalized: { ok: true, acceptable: true, eligibleForWinner: true, provider: "on", evidenceBlock: "y", sources: [], sourceCount: 1, claimCount: 1, structuredItemCount: 0, attestedCount: 0, supportedCount: 1, conflictedCount: 0, proof: "bound", qualityVector: [1, 3, 0, 1, 1, 1, 100] } },
+    { id: "sentinel-orchestrator", priority: 95, status: "rejected", elapsedMs: 50, error: "timeout" },
+  ];
+  const independence = computeIndependence(mock);
+  add("independence-2-families", independence.distinctFamilies === 2 && independence.fulfilledFamilies.includes("conclave-core") && independence.fulfilledFamilies.includes("omni-standalone"), `families=${independence.fulfilledFamilies.join(",")}`);
+  const marginal = computeMarginalGain(mock, "terminal-wire", "omni-nexus");
+  add("marginal-changed-winner", marginal.extraLanesChangedWinner && marginal.assessment === "high-value", marginal.assessment);
+  add("engine-family-import", ENGINE_FAMILY["terminal-wire"] === "conclave-core" && ENGINE_FAMILY["omni-nexus"] === "omni-standalone", `tw=${ENGINE_FAMILY["terminal-wire"]}`);
+  add("robots-factory-exported", typeof makeRfc9309RobotsDecision() === "function", "factory ok");
+  suites.push({ name: "terminal-final-synthesis", ok: checks.every((check) => check.passed), checks });
+  return { ok: suites.every((suite) => suite.ok), suites };
+}
+
+export {
+  groundWithTerminalPortfolioGovernor,
+  familyOf,
+  ENGINE_FAMILY,
+  type TerminalGovernorOptions,
+  type TerminalGovernorResult,
+} from "./portfolio-terminal-governor";

package/src/lib/scraper-vnext/terminal-saturation.ts

@@ -0,0 +1,345 @@
+/**
+ * terminal-saturation.ts
+ * ============================================================================
+ * Optional supplementary structured metadata layer over terminal-final.ts.
+ * It preserves canonical output when mode is "off" (the default), surfaces
+ * DataCite and Europe PMC metadata as advisory data, and only merges marked
+ * supplementary records when callers explicitly opt in.
+ */
+
+import {
+  groundTerminalFinal,
+  type TerminalFinalOptions,
+  type TerminalFinalResult,
+} from "./terminal-final";
+
+export type SupplementaryMode = "off" | "advisory" | "merge";
+export type SupplementarySourceKind = "datacite" | "europe-pmc";
+
+export interface SupplementaryStructuredItem {
+  id: string;
+  kind: SupplementarySourceKind;
+  title: string;
+  abstract?: string;
+  authors?: string[];
+  publishedDate?: string;
+  doi?: string;
+  externalUrl?: string;
+  openAccessUrl?: string;
+  citationCount?: number;
+  pageContent: string;
+  sourceWeight: number;
+  metadata: Record<string, string | number | boolean | undefined>;
+}
+
+export interface SupplementaryManifest {
+  proof: "locally-bound" | "unavailable";
+  root: string;
+  itemCount: number;
+  hashAlgorithm: "SHA-256" | "unavailable";
+}
+
+export interface TerminalSaturationOptions extends TerminalFinalOptions {
+  supplementaryMode?: SupplementaryMode;
+  supplementarySources?: SupplementarySourceKind[];
+  supplementaryLimitPerSource?: number;
+  supplementaryTimeoutMs?: number;
+  cooperativeYield?: boolean;
+}
+
+export interface TerminalSaturationResult extends TerminalFinalResult {
+  supplementaryMode: SupplementaryMode;
+  supplementaryItems: SupplementaryStructuredItem[];
+  supplementaryQueriedSources: SupplementarySourceKind[];
+  supplementaryErrors: Array<{ source: SupplementarySourceKind; error: string }>;
+  supplementaryMergedItems: SupplementaryStructuredItem[];
+  supplementaryManifest: SupplementaryManifest;
+  runtimeYield: "scheduler" | "timer" | "disabled";
+}
+
+function clamp(value: number | undefined, fallback: number, minimum: number, maximum: number): number {
+  return value === undefined || !Number.isFinite(value)
+    ? fallback
+    : Math.max(minimum, Math.min(maximum, Math.floor(value)));
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function text(value: unknown): string {
+  return typeof value === "string" ? value.trim() : typeof value === "number" || typeof value === "boolean" ? String(value) : "";
+}
+
+function count(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function normalizeText(value: string): string {
+  return (value || "")
+    .replace(/<(script|style|noscript|iframe|object|embed)[^>]*>[\s\S]*?<\/\1>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/[\u202A-\u202E\u2066-\u2069\u200B-\u200F\uFEFF]/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function cleanDoi(value: string | undefined): string {
+  return (value || "").trim().replace(/^https?:\/\/doi\.org\//i, "").replace(/^doi:\s*/i, "").toLowerCase();
+}
+
+function canonicalUrl(value: string | undefined): string {
+  if (!value) return "";
+  try {
+    const url = new URL(value);
+    if (url.protocol !== "http:" && url.protocol !== "https:") return "";
+    if (url.username || url.password) return "";
+    url.hash = "";
+    url.hostname = url.hostname.toLowerCase();
+    for (const key of Array.from(url.searchParams.keys())) {
+      if (key.toLowerCase().startsWith("utm_") || ["fbclid", "gclid", "dclid", "msclkid"].includes(key.toLowerCase())) url.searchParams.delete(key);
+    }
+    url.searchParams.sort();
+    return url.toString();
+  } catch {
+    return "";
+  }
+}
+
+function sourceText(item: Omit<SupplementaryStructuredItem, "pageContent">): string {
+  const lines = [
+    `[SUPPLEMENTARY ${item.kind.toUpperCase()} - UNATTESTED METADATA]`,
+    `Title: ${item.title || "Untitled"}`,
+    item.authors?.length ? `Authors: ${item.authors.slice(0, 12).join(", ")}` : "",
+    item.publishedDate ? `Published: ${item.publishedDate}` : "",
+    item.doi ? `DOI: ${item.doi}` : "",
+    item.externalUrl ? `URL: ${item.externalUrl}` : "",
+    item.abstract ? `Abstract: ${item.abstract}` : "",
+  ].filter(Boolean);
+  return normalizeText(lines.join("\n"));
+}
+
+function makeItem(item: Omit<SupplementaryStructuredItem, "pageContent">): SupplementaryStructuredItem {
+  return { ...item, title: item.title || "Untitled", pageContent: sourceText(item) };
+}
+
+async function cooperativeYield(): Promise<"scheduler" | "timer"> {
+  const scheduler = (globalThis as { scheduler?: { yield?: () => Promise<void> } }).scheduler;
+  if (typeof scheduler?.yield === "function") {
+    await scheduler.yield();
+    return "scheduler";
+  }
+  await new Promise<void>((resolve) => setTimeout(resolve, 0));
+  return "timer";
+}
+
+function linkedSignal(parent: AbortSignal | undefined, timeoutMs: number): { signal: AbortSignal; cleanup: () => void } {
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  if (parent?.aborted) abort();
+  else parent?.addEventListener("abort", abort, { once: true });
+  const timeout = setTimeout(abort, timeoutMs);
+  return {
+    signal: controller.signal,
+    cleanup: () => {
+      clearTimeout(timeout);
+      parent?.removeEventListener("abort", abort);
+    },
+  };
+}
+
+async function fetchJson(url: string, signal: AbortSignal | undefined, timeoutMs: number): Promise<unknown> {
+  const linked = linkedSignal(signal, timeoutMs);
+  try {
+    const response = await fetch(url, {
+      signal: linked.signal,
+      credentials: "omit",
+      referrerPolicy: "no-referrer",
+      headers: { Accept: "application/json,application/vnd.api+json" },
+    });
+    if (!response.ok) throw new Error(`HTTP_${response.status}`);
+    const body = await response.text();
+    return JSON.parse(body.slice(0, 2_000_000));
+  } finally {
+    linked.cleanup();
+  }
+}
+
+export function parseDataCiteResponse(value: unknown): SupplementaryStructuredItem[] {
+  if (!isRecord(value) || !Array.isArray(value.data)) return [];
+  const output: SupplementaryStructuredItem[] = [];
+  for (const record of value.data) {
+    if (!isRecord(record)) continue;
+    const attributes = isRecord(record.attributes) ? record.attributes : {};
+    const titles = Array.isArray(attributes.titles) ? attributes.titles : [];
+    const titleRecord = titles.find(isRecord);
+    const title = titleRecord ? text(titleRecord.title) : text(attributes.title);
+    const authors = (Array.isArray(attributes.creators) ? attributes.creators : []).filter(isRecord).map((creator) => text(creator.name) || [text(creator.givenName), text(creator.familyName)].filter(Boolean).join(" ")).filter(Boolean);
+    const descriptions = (Array.isArray(attributes.descriptions) ? attributes.descriptions : []).filter(isRecord);
+    const abstractRecord = descriptions.find((description) => text(description.descriptionType).toLowerCase().includes("abstract")) ?? descriptions[0];
+    const abstract = abstractRecord ? normalizeText(text(abstractRecord.description)) : "";
+    const doi = cleanDoi(text(attributes.doi) || text(record.id));
+    const externalUrl = canonicalUrl(text(attributes.url)) || (doi ? `https://doi.org/${doi}` : "");
+    if (!title && !abstract && !doi) continue;
+    output.push(makeItem({
+      id: `datacite:${doi || text(record.id) || output.length}`,
+      kind: "datacite",
+      title: title || "Untitled",
+      abstract: abstract || undefined,
+      authors: authors.length ? authors : undefined,
+      publishedDate: text(attributes.publicationYear) || undefined,
+      doi: doi || undefined,
+      externalUrl: externalUrl || undefined,
+      sourceWeight: 0.6,
+      metadata: { publisher: text(attributes.publisher), structuredUnattested: true },
+    }));
+  }
+  return output;
+}
+
+export function parseEuropePmcResponse(value: unknown): SupplementaryStructuredItem[] {
+  if (!isRecord(value) || !isRecord(value.resultList) || !Array.isArray(value.resultList.result)) return [];
+  const output: SupplementaryStructuredItem[] = [];
+  for (const record of value.resultList.result) {
+    if (!isRecord(record)) continue;
+    const source = text(record.source);
+    const id = text(record.id) || text(record.pmid) || text(record.pmcid);
+    const doi = cleanDoi(text(record.doi));
+    const title = normalizeText(text(record.title));
+    const abstract = normalizeText(text(record.abstractText));
+    const authors = text(record.authorString).split(/\s*,\s*/).filter(Boolean);
+    const externalUrl = source && id ? `https://europepmc.org/article/${encodeURIComponent(source)}/${encodeURIComponent(id)}` : doi ? `https://doi.org/${doi}` : "";
+    if (!title && !abstract && !id && !doi) continue;
+    output.push(makeItem({
+      id: `europe-pmc:${source || "unknown"}:${id || doi || output.length}`,
+      kind: "europe-pmc",
+      title: title || "Untitled",
+      abstract: abstract || undefined,
+      authors: authors.length ? authors : undefined,
+      publishedDate: text(record.firstPublicationDate) || text(record.dateOfPublication) || text(record.pubYear) || undefined,
+      doi: doi || undefined,
+      externalUrl: canonicalUrl(externalUrl) || externalUrl || undefined,
+      openAccessUrl: text(record.pmcid) ? `https://europepmc.org/articles/${text(record.pmcid)}` : undefined,
+      citationCount: count(record.citedByCount),
+      sourceWeight: 0.65,
+      metadata: { source, recordId: id, pmid: text(record.pmid), pmcid: text(record.pmcid), structuredUnattested: true },
+    }));
+  }
+  return output;
+}
+
+async function queryDataCite(query: string, signal: AbortSignal | undefined, limit: number, timeoutMs: number): Promise<{ source: SupplementarySourceKind; items: SupplementaryStructuredItem[]; error?: string }> {
+  try {
+    const value = await fetchJson(`https://api.datacite.org/dois?query=${encodeURIComponent(query)}&page[size]=${limit}`, signal, timeoutMs);
+    return { source: "datacite", items: parseDataCiteResponse(value) };
+  } catch (error) {
+    return { source: "datacite", items: [], error: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+async function queryEuropePmc(query: string, signal: AbortSignal | undefined, limit: number, timeoutMs: number): Promise<{ source: SupplementarySourceKind; items: SupplementaryStructuredItem[]; error?: string }> {
+  try {
+    const value = await fetchJson(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${encodeURIComponent(query)}&pageSize=${limit}&resultType=core&format=json`, signal, timeoutMs);
+    return { source: "europe-pmc", items: parseEuropePmcResponse(value) };
+  } catch (error) {
+    return { source: "europe-pmc", items: [], error: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+function itemKey(item: Pick<SupplementaryStructuredItem, "doi" | "externalUrl" | "title">): string {
+  const doi = cleanDoi(item.doi);
+  if (doi) return `doi:${doi}`;
+  const url = canonicalUrl(item.externalUrl);
+  return url ? `url:${url}` : `title:${normalizeText(item.title).toLowerCase()}`;
+}
+
+function deduplicate(items: SupplementaryStructuredItem[], existing: Array<{ doi?: string; externalUrl?: string; title: string }>): SupplementaryStructuredItem[] {
+  const seen = new Set(existing.map(itemKey));
+  return items.filter((item) => {
+    const key = itemKey(item);
+    if (!key || key === "title:" || seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+async function manifestFor(items: SupplementaryStructuredItem[]): Promise<SupplementaryManifest> {
+  if (!items.length || typeof crypto === "undefined" || !crypto.subtle) return { proof: "unavailable", root: "", itemCount: items.length, hashAlgorithm: "unavailable" };
+  try {
+    const stable = items.map((item) => JSON.stringify({ id: item.id, kind: item.kind, title: item.title, doi: item.doi, externalUrl: item.externalUrl, pageContent: item.pageContent })).sort().join("\n");
+    const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(stable));
+    const root = Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
+    return { proof: "locally-bound", root, itemCount: items.length, hashAlgorithm: "SHA-256" };
+  } catch {
+    return { proof: "unavailable", root: "", itemCount: items.length, hashAlgorithm: "unavailable" };
+  }
+}
+
+export async function groundTerminalSaturated(question: string, options?: TerminalSaturationOptions): Promise<TerminalSaturationResult> {
+  const supplementaryMode = options?.supplementaryMode ?? "off";
+  const sources = options?.supplementarySources ?? ["datacite", "europe-pmc"];
+  const limit = clamp(options?.supplementaryLimitPerSource, 6, 1, 25);
+  const timeout = clamp(options?.supplementaryTimeoutMs, 8_000, 1_000, 30_000);
+  const supplementaryPromise = supplementaryMode === "off"
+    ? Promise.resolve([] as Array<{ source: SupplementarySourceKind; items: SupplementaryStructuredItem[]; error?: string }>)
+    : Promise.all(sources.map((source) => source === "datacite" ? queryDataCite(question, options?.signal, limit, timeout) : queryEuropePmc(question, options?.signal, limit, timeout)));
+  const canonicalResult = await groundTerminalFinal(question, options);
+  const settled = await supplementaryPromise;
+  let runtimeYield: "scheduler" | "timer" | "disabled" = "disabled";
+  if (supplementaryMode !== "off" && options?.cooperativeYield !== false) runtimeYield = await cooperativeYield();
+  const allItems = deduplicate(settled.flatMap((result) => result.items), canonicalResult.structuredItems);
+  const supplementaryErrors = settled.filter((result) => result.error).map((result) => ({ source: result.source, error: result.error! }));
+  const supplementaryManifest = await manifestFor(allItems);
+  const mergedItems = supplementaryMode === "merge" ? allItems : [];
+  const extraSources = mergedItems.map((item) => ({ title: `[UNATTESTED ${item.kind}] ${item.title}`, url: item.externalUrl || (item.doi ? `https://doi.org/${item.doi}` : `supplementary:${item.id}`), content: item.pageContent }));
+  let evidenceBlock = canonicalResult.evidenceBlock;
+  if (mergedItems.length && evidenceBlock.includes("END RETRIEVED CONTENT")) {
+    const insertion = evidenceBlock.lastIndexOf("END RETRIEVED CONTENT");
+    const section = [
+      "BEGIN SUPPLEMENTARY STRUCTURED ITEMS",
+      "SECURITY: This section contains UNATTESTED external metadata.",
+      "The canonical manifest does not cover this section.",
+      supplementaryManifest.proof === "locally-bound" ? `SUPPLEMENTARY MANIFEST ROOT: ${supplementaryManifest.root}` : "SUPPLEMENTARY MANIFEST: unavailable",
+      ...mergedItems.map((item, index) => `BEGIN SUPPLEMENTARY ITEM ${index + 1}\n${item.pageContent}\nEND SUPPLEMENTARY ITEM ${index + 1}`),
+      "END SUPPLEMENTARY STRUCTURED ITEMS",
+      "",
+    ].join("\n");
+    evidenceBlock = `${evidenceBlock.slice(0, insertion)}${section}${evidenceBlock.slice(insertion)}`;
+  }
+  return {
+    ...canonicalResult,
+    provider: supplementaryMode === "off" ? canonicalResult.provider : `terminal-saturation(${canonicalResult.provider}+supplementary:${allItems.length}+merged:${mergedItems.length})`,
+    sources: mergedItems.length ? [...canonicalResult.sources, ...extraSources] : canonicalResult.sources,
+    count: mergedItems.length ? canonicalResult.count + extraSources.length : canonicalResult.count,
+    evidenceBlock,
+    supplementaryMode,
+    supplementaryItems: allItems,
+    supplementaryQueriedSources: settled.map((result) => result.source),
+    supplementaryErrors,
+    supplementaryMergedItems: mergedItems,
+    supplementaryManifest,
+    runtimeYield,
+  };
+}
+
+export async function runTerminalSaturationDiagnostics(): Promise<{
+  ok: boolean;
+  checks: Array<{ id: string; passed: boolean; detail: string }>;
+}> {
+  const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
+  const add = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail });
+  const dataCite = parseDataCiteResponse({ data: [{ id: "10.1234/example", attributes: { doi: "10.1234/example", titles: [{ title: "Dataset title" }], creators: [{ name: "Ada Example" }], descriptions: [{ descriptionType: "Abstract", description: "<p>Dataset abstract.</p>" }] } }] });
+  add("datacite-parser", dataCite.length === 1 && dataCite[0].doi === "10.1234/example" && dataCite[0].abstract === "Dataset abstract.", `items=${dataCite.length}`);
+  const europePmc = parseEuropePmcResponse({ resultList: { result: [{ source: "MED", id: "12345", doi: "10.1000/example", title: "Biomedical article", abstractText: "<p>Biomedical abstract.</p>", citedByCount: 7 }] } });
+  add("europe-pmc-parser", europePmc.length === 1 && europePmc[0].doi === "10.1000/example" && europePmc[0].citationCount === 7, `items=${europePmc.length}`);
+  add("supplementary-doi-dedupe", deduplicate([dataCite[0], { ...europePmc[0], doi: "10.1234/example" }], []).length === 1, "doi dedupe");
+  const manifest = await manifestFor(dataCite);
+  add("supplementary-manifest", typeof crypto !== "undefined" && crypto.subtle ? manifest.proof === "locally-bound" && manifest.root.length === 64 : manifest.proof === "unavailable", `proof=${manifest.proof}`);
+  const yielded = await cooperativeYield();
+  add("cooperative-yield", yielded === "scheduler" || yielded === "timer", `mode=${yielded}`);
+  return { ok: checks.every((check) => check.passed), checks };
+}