@tangle-network/agent-knowledge 1.9.0 → 1.10.0

package/dist/index.js

@@ -77,6 +77,495 @@ import {
   researcherProfile
 } from "./chunk-MU5CEOGE.js";
 
+// src/web-research-worker.ts
+var DEFAULT_MODEL = "glm-5.2";
+var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
+var MIN_MAX_TOKENS = 1200;
+var RouterError = class extends Error {
+  constructor(status, message) {
+    super(`router ${status}: ${message}`);
+    this.status = status;
+    this.name = "RouterError";
+  }
+  status;
+};
+function createTangleRouterClient(options = {}) {
+  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
+  const apiKey = options.apiKey ?? process.env.TANGLE_API_KEY;
+  if (!apiKey) {
+    throw new RouterError(401, "no TANGLE_API_KEY (pass apiKey or set the env var)");
+  }
+  const model = options.model ?? DEFAULT_MODEL;
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${apiKey}`
+  };
+  const price = { prompt: 0.95 / 1e6, completion: 3 / 1e6 };
+  const acc = {
+    chatCalls: 0,
+    searchCalls: 0,
+    promptTokens: 0,
+    completionTokens: 0,
+    usd: 0,
+    wallMs: 0
+  };
+  return {
+    async search(query, opts) {
+      const t0 = Date.now();
+      const res = await fetch(`${baseUrl}/search`, {
+        method: "POST",
+        headers,
+        signal: options.signal,
+        body: JSON.stringify({
+          query,
+          ...options.searchProvider ? { provider: options.searchProvider } : {},
+          ...opts?.maxResults != null ? { maxResults: opts.maxResults } : {}
+        })
+      });
+      acc.searchCalls += 1;
+      acc.wallMs += Date.now() - t0;
+      if (!res.ok) {
+        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
+      }
+      const body = await res.json();
+      return (body.data ?? []).filter((hit) => typeof hit?.url === "string" && hit.url.length > 0).map((hit) => ({ title: hit.title ?? hit.url, url: hit.url, snippet: hit.snippet }));
+    },
+    async chat(messages, maxTokens) {
+      const max_tokens = Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS);
+      const t0 = Date.now();
+      const res = await fetch(`${baseUrl}/chat/completions`, {
+        method: "POST",
+        headers,
+        signal: options.signal,
+        body: JSON.stringify({ model, messages, max_tokens, temperature: 0.2, stream: false })
+      });
+      if (!res.ok) {
+        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
+      }
+      const body = await res.json();
+      const promptTokens = body.usage?.prompt_tokens ?? 0;
+      const completionTokens = body.usage?.completion_tokens ?? 0;
+      acc.chatCalls += 1;
+      acc.promptTokens += promptTokens;
+      acc.completionTokens += completionTokens;
+      acc.usd += promptTokens * price.prompt + completionTokens * price.completion;
+      acc.wallMs += Date.now() - t0;
+      return body.choices?.[0]?.message?.content ?? "";
+    },
+    usage() {
+      return { ...acc };
+    }
+  };
+}
+function resolveRouter(opts) {
+  return opts.router ?? createTangleRouterClient(opts.router_options);
+}
+function createWebResearchWorker(options = {}) {
+  const queriesPerGap = Math.max(1, options.queriesPerGap ?? 2);
+  const resultsPerQuery = Math.max(1, options.resultsPerQuery ?? 3);
+  const maxSourcesPerRound = Math.max(1, options.maxSourcesPerRound ?? 6);
+  const minTextChars = Math.max(1, options.minTextChars ?? 200);
+  const maxTextChars = Math.max(minTextChars, options.maxTextChars ?? 4e3);
+  return async (ctx) => {
+    const router = resolveRouter(options);
+    const targetGaps = ctx.gaps.filter((gap) => gap.blocking);
+    const gaps = targetGaps.length > 0 ? targetGaps : ctx.gaps;
+    if (gaps.length === 0) {
+      return { sources: [], notes: "no open gaps to research" };
+    }
+    const queries = await formSearchQueries(router, ctx, gaps, queriesPerGap);
+    const proposals = [];
+    const seenUris = /* @__PURE__ */ new Set();
+    for (const query of queries) {
+      if (proposals.length >= maxSourcesPerRound) break;
+      if (ctx.signal?.aborted) break;
+      let hits;
+      try {
+        hits = await router.search(query, { maxResults: resultsPerQuery });
+      } catch (error) {
+        if (error.name === "AbortError") break;
+        continue;
+      }
+      for (const hit of hits) {
+        if (proposals.length >= maxSourcesPerRound) break;
+        if (seenUris.has(hit.url)) continue;
+        const fetched = await politeFetch(hit.url, {
+          signal: ctx.signal,
+          cacheDir: options.cacheDir
+        });
+        if (!fetched.verifiable) continue;
+        const text = htmlToText(fetched.body).slice(0, maxTextChars);
+        if (text.length < minTextChars) continue;
+        seenUris.add(hit.url);
+        proposals.push({
+          uri: hit.url,
+          title: hit.title || hit.url,
+          text,
+          // We just fetched + verified this page, so stamp `lastVerifiedAt` with
+          // fetch time. Do NOT set `validUntil`: a live page has no inherent
+          // future expiry, and the readiness freshness check treats any
+          // `validUntil <= now` as EXPIRED (score 0). The page's `Last-Modified`
+          // is a PAST date, so writing it here would mark every real source stale
+          // and zero out coverage. Record it as provenance metadata instead.
+          lastVerifiedAt: fetched.fetchedAt,
+          metadata: {
+            discoveredVia: query,
+            snippet: hit.snippet ?? "",
+            goal: ctx.goal,
+            sourceUpdatedAt: fetched.sourceUpdatedAt
+          }
+        });
+      }
+    }
+    return {
+      sources: proposals,
+      buildPages: buildCitingPages(proposals),
+      notes: `web-research worker: ${queries.length} queries \u2192 ${proposals.length} fetched sources`
+    };
+  };
+}
+async function formSearchQueries(router, ctx, gaps, queriesPerGap) {
+  const gapLines = gaps.map((gap, i) => `${i + 1}. ${gap.description} (readiness query: "${gap.query}")`).join("\n");
+  const want = gaps.length * queriesPerGap;
+  const system = "You are a research librarian. Turn knowledge gaps into precise web search queries that will surface authoritative primary sources (papers, docs, standards, official pages). Return ONLY a JSON array of query strings, no prose.";
+  const user = [
+    `Research goal: ${ctx.goal}`,
+    ctx.steer ? `Steer from the coordinator:
+${ctx.steer}` : "",
+    `Open knowledge gaps:
+${gapLines}`,
+    `Return up to ${want} search query strings as a JSON array (e.g. ["query one","query two"]).`
+  ].filter(Boolean).join("\n\n");
+  let raw = "";
+  try {
+    raw = await router.chat(
+      [
+        { role: "system", content: system },
+        { role: "user", content: user }
+      ],
+      MIN_MAX_TOKENS
+    );
+  } catch {
+    raw = "";
+  }
+  const parsed = parseQueryList(raw);
+  const fromLlm = parsed.slice(0, want);
+  if (fromLlm.length > 0) return dedupeStrings(fromLlm);
+  return dedupeStrings(gaps.map((gap) => gap.query || gap.description));
+}
+function parseQueryList(raw) {
+  const text = raw.trim();
+  if (!text) return [];
+  const candidates = [];
+  const arrayMatch = text.match(/\[[\s\S]*\]/);
+  if (arrayMatch) {
+    try {
+      const arr = JSON.parse(arrayMatch[0]);
+      for (const item of arr)
+        if (typeof item === "string" && item.trim()) candidates.push(item.trim());
+    } catch {
+    }
+  }
+  if (candidates.length === 0) {
+    for (const line of text.split("\n")) {
+      const cleaned = line.replace(/^\s*(?:[-*]|\d+[.)])\s*/, "").replace(/^["']|["']$/g, "").trim();
+      if (cleaned && cleaned.length > 2 && !cleaned.startsWith("{")) candidates.push(cleaned);
+    }
+  }
+  return candidates;
+}
+function dedupeStrings(values) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const value of values) {
+    const key = value.toLowerCase().trim();
+    if (key && !seen.has(key)) {
+      seen.add(key);
+      out.push(value.trim());
+    }
+  }
+  return out;
+}
+function buildCitingPages(proposals) {
+  return (acceptedSources) => {
+    if (acceptedSources.length === 0) return void 0;
+    const blocks = acceptedSources.map((record) => {
+      const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri);
+      const uri = proposal?.uri ?? record.id;
+      const slug = uri.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 120) || record.id;
+      const title = proposal?.title ?? record.id;
+      const body = proposal?.text ?? "";
+      return [
+        `---FILE: knowledge/${slug}.md---`,
+        "---",
+        `title: ${escapeYaml(title)}`,
+        `sources: ["${record.id}"]`,
+        `source_url: ${uri}`,
+        "---",
+        `# ${title}`,
+        "",
+        body,
+        "",
+        `Source: ${uri}`,
+        "---END FILE---"
+      ].join("\n");
+    });
+    return blocks.join("\n");
+  };
+}
+function escapeYaml(value) {
+  return value.replace(/[\r\n]+/g, " ").replace(/"/g, "'").trim();
+}
+function createVerifyingResearchDriver(options = {}) {
+  const acceptOnParseFailure = options.acceptOnParseFailure ?? false;
+  return {
+    async verifySource(source, ctx) {
+      const router = resolveRouter(options);
+      const gapLines = ctx.gaps.map((gap) => `- ${gap.description} (query: "${gap.query}")`).join("\n");
+      const acceptedTitles = ctx.acceptedThisRound.map((accepted) => `- ${accepted.title ?? accepted.uri}`).join("\n");
+      const excerpt = source.text.slice(0, 1500);
+      const system = 'You verify whether a fetched web source belongs in a curated knowledge base. Accept a source ONLY if it is genuinely on-topic for the research goal and helps close one of the open gaps, AND it is not a near-duplicate of an already-accepted source. Reject spam, listicles, off-topic pages, marketing, and near-duplicates. Respond with ONLY a JSON object: {"accept": true|false, "reason": "<short reason>"}.';
+      const user = [
+        `Research goal: ${ctx.goal}`,
+        `Open gaps:
+${gapLines || "(none specified)"}`,
+        acceptedTitles ? `Already accepted this round:
+${acceptedTitles}` : "Nothing accepted yet this round.",
+        `Candidate source:
+URL: ${source.uri}
+Title: ${source.title ?? "(none)"}
+Excerpt:
+${excerpt}`,
+        'Verdict as JSON {"accept": boolean, "reason": string}:'
+      ].join("\n\n");
+      let raw = "";
+      try {
+        raw = await router.chat(
+          [
+            { role: "system", content: system },
+            { role: "user", content: user }
+          ],
+          MIN_MAX_TOKENS
+        );
+      } catch (error) {
+        if (error.name === "AbortError") throw error;
+        return acceptOnParseFailure ? { accept: true } : { accept: false, reason: `verifier unavailable: ${error.message}` };
+      }
+      const verdict = parseVerdict(raw);
+      if (verdict) return verdict;
+      return acceptOnParseFailure ? { accept: true } : { accept: false, reason: "verifier returned an unparseable verdict" };
+    }
+  };
+}
+function parseVerdict(raw) {
+  const text = raw.trim();
+  if (!text) return null;
+  const objMatch = text.match(/\{[\s\S]*\}/);
+  if (!objMatch) return null;
+  try {
+    const parsed = JSON.parse(objMatch[0]);
+    if (typeof parsed.accept !== "boolean") return null;
+    if (parsed.accept) return { accept: true };
+    return {
+      accept: false,
+      reason: typeof parsed.reason === "string" && parsed.reason.trim() ? parsed.reason.trim() : "rejected by verifier"
+    };
+  } catch {
+    return null;
+  }
+}
+
+// src/adaptive-driver.ts
+function canonicalizeUrl(uri) {
+  const trimmed = uri.trim();
+  try {
+    const url = new URL(trimmed);
+    const host = url.hostname.toLowerCase().replace(/^www\./, "");
+    const kept = [];
+    for (const [key, value] of url.searchParams) {
+      const lower = key.toLowerCase();
+      if (lower.startsWith("utm_")) continue;
+      if (trackingParams.has(lower)) continue;
+      kept.push([key, value]);
+    }
+    kept.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+    const query = kept.map(([k, v]) => `${k}=${v}`).join("&");
+    const path = url.pathname.replace(/\/+$/, "") || "/";
+    return `${host}${path}${query ? `?${query}` : ""}`;
+  } catch {
+    return trimmed.toLowerCase();
+  }
+}
+var trackingParams = /* @__PURE__ */ new Set([
+  "ref",
+  "ref_src",
+  "source",
+  "fbclid",
+  "gclid",
+  "mc_cid",
+  "mc_eid",
+  "igshid",
+  "spm",
+  "_hsenc",
+  "_hsmi"
+]);
+function contentKey(text) {
+  const normalized = text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();
+  return sha256(normalized);
+}
+var defaultAuthoritativeHosts = [
+  "arxiv.org",
+  "aclanthology.org",
+  "openreview.net",
+  "dl.acm.org",
+  "ieeexplore.ieee.org",
+  "nature.com",
+  "science.org",
+  "pubmed.ncbi.nlm.nih.gov",
+  "ncbi.nlm.nih.gov",
+  ".edu",
+  ".gov",
+  "docs.python.org",
+  "pytorch.org",
+  "tensorflow.org",
+  "huggingface.co",
+  "github.com",
+  "developer.mozilla.org",
+  "wikipedia.org",
+  "w3.org",
+  "ietf.org",
+  "rfc-editor.org"
+];
+var defaultSpamPatterns = [
+  /\bbuy\b.*\b(cheap|now|deal|sale|discount)\b/i,
+  /\b\d+\s+(things|ways|reasons|tips|tricks|secrets|hacks)\b.*\b(you|that|will)\b/i,
+  /\bshock(ing|ed)?\b/i,
+  /\bclickbait\b/i,
+  /!!!|\$\$\$/,
+  /\b(coupon|promo code|affiliate|sponsored)\b/i,
+  /\bbest .* (of \d{4}|in \d{4})\b/i
+];
+function triageSource(source, options) {
+  const titleAndSnippet = `${source.title ?? ""} ${typeof source.metadata?.snippet === "string" ? source.metadata.snippet : ""}`.trim();
+  const bodyLen = source.text.trim().length;
+  for (const pattern of options.spamPatterns) {
+    if (pattern.test(titleAndSnippet)) {
+      return { triage: "drop", reason: `spam/listicle title (${pattern.source})` };
+    }
+  }
+  if (bodyLen < options.minBodyChars) {
+    return { triage: "drop", reason: `thin body (${bodyLen} < ${options.minBodyChars} chars)` };
+  }
+  const host = hostOf(source.uri);
+  const authoritative = host.length > 0 && options.authoritativeHosts.some((suffix) => hostMatches(host, suffix));
+  if (authoritative && bodyLen >= options.substantialBodyChars) {
+    return { triage: "keep", reason: `authoritative host ${host} + substantial body (${bodyLen})` };
+  }
+  return { triage: "ambiguous", reason: `unknown host ${host || "(none)"}, body ${bodyLen} chars` };
+}
+function hostOf(uri) {
+  try {
+    return new URL(uri.trim()).hostname.toLowerCase().replace(/^www\./, "");
+  } catch {
+    return "";
+  }
+}
+function hostMatches(host, suffix) {
+  if (suffix.startsWith(".")) return host.endsWith(suffix) || host === suffix.slice(1);
+  return host === suffix || host.endsWith(`.${suffix}`);
+}
+function createAdaptiveResearchDriver(options = {}) {
+  const authoritativeHosts = options.authoritativeHosts ?? defaultAuthoritativeHosts;
+  const spamPatterns = options.spamPatterns ?? defaultSpamPatterns;
+  const minBodyChars = Math.max(1, options.minBodyChars ?? 400);
+  const substantialBodyChars = Math.max(minBodyChars, options.substantialBodyChars ?? 600);
+  const relevance = createVerifyingResearchDriver({
+    router: options.router,
+    router_options: options.router_options,
+    acceptOnParseFailure: options.verifying?.acceptOnParseFailure
+  });
+  const acceptedUrlKeys = /* @__PURE__ */ new Set();
+  const acceptedContentKeys = /* @__PURE__ */ new Set();
+  const stats = {
+    total: 0,
+    dedupRejected: 0,
+    heuristicKept: 0,
+    heuristicDropped: 0,
+    llmCalls: 0,
+    llmAccepted: 0,
+    decisions: []
+  };
+  function record(decision) {
+    stats.decisions.push(decision);
+    options.onDecision?.(decision);
+  }
+  return {
+    async verifySource(source, ctx) {
+      stats.total += 1;
+      const urlKey = canonicalizeUrl(source.uri);
+      const cKey = contentKey(source.text);
+      const roundUrlKeys = new Set(
+        ctx.acceptedThisRound.map((accepted) => canonicalizeUrl(accepted.uri))
+      );
+      const roundContentKeys = new Set(
+        ctx.acceptedThisRound.map((accepted) => contentKey(accepted.text))
+      );
+      const indexUrlKeys = new Set(
+        ctx.index.sources.flatMap(
+          (indexed) => typeof indexed.metadata?.originalUri === "string" ? [canonicalizeUrl(indexed.metadata.originalUri)] : []
+        )
+      );
+      const dupUrl = acceptedUrlKeys.has(urlKey) || roundUrlKeys.has(urlKey) || indexUrlKeys.has(urlKey);
+      const dupContent = acceptedContentKeys.has(cKey) || roundContentKeys.has(cKey);
+      if (dupUrl || dupContent) {
+        stats.dedupRejected += 1;
+        const reason2 = dupUrl ? "duplicate-url" : "duplicate-content";
+        record({ uri: source.uri, stage: "dedup", accepted: false, reason: reason2 });
+        return { accept: false, reason: `dedup: ${reason2}` };
+      }
+      const { triage, reason } = triageSource(source, {
+        authoritativeHosts,
+        spamPatterns,
+        minBodyChars,
+        substantialBodyChars
+      });
+      if (triage === "keep") {
+        stats.heuristicKept += 1;
+        acceptedUrlKeys.add(urlKey);
+        acceptedContentKeys.add(cKey);
+        record({ uri: source.uri, stage: "heuristic", accepted: true, triage, reason });
+        return { accept: true };
+      }
+      if (triage === "drop") {
+        stats.heuristicDropped += 1;
+        record({ uri: source.uri, stage: "heuristic", accepted: false, triage, reason });
+        return { accept: false, reason: `heuristic drop: ${reason}` };
+      }
+      stats.llmCalls += 1;
+      const verdict = await relevance.verifySource(source, ctx);
+      if (verdict.accept) {
+        stats.llmAccepted += 1;
+        acceptedUrlKeys.add(urlKey);
+        acceptedContentKeys.add(cKey);
+      }
+      record({
+        uri: source.uri,
+        stage: "llm",
+        accepted: verdict.accept,
+        triage,
+        reason: verdict.accept ? "llm accepted" : verdict.reason
+      });
+      return verdict;
+    },
+    stats() {
+      return {
+        ...stats,
+        decisions: [...stats.decisions]
+      };
+    }
+  };
+}
+
 // src/changes.ts
 function detectChanges(prev, next, options = {}) {
   const skipUnverifiable = options.skipUnverifiable ?? true;
@@ -286,8 +775,158 @@ ${chunk.text}`;
     } else {
       out.push({ ...chunk });
     }
-  }
-  return out;
+  }
+  return out;
+}
+
+// src/claim-grounding.ts
+var citedClaimKey = "citedClaim";
+function citedClaimOf(source) {
+  const claim = source.metadata?.[citedClaimKey];
+  return typeof claim === "string" && claim.trim() ? claim.trim() : void 0;
+}
+function withCitedClaim(source, claim) {
+  return { ...source, metadata: { ...source.metadata, [citedClaimKey]: claim } };
+}
+var stopwords = /* @__PURE__ */ new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "but",
+  "of",
+  "to",
+  "in",
+  "on",
+  "for",
+  "with",
+  "as",
+  "by",
+  "at",
+  "from",
+  "that",
+  "this",
+  "these",
+  "those",
+  "it",
+  "its",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "has",
+  "have",
+  "had",
+  "can",
+  "will",
+  "would",
+  "should",
+  "may",
+  "might",
+  "not",
+  "no",
+  "than",
+  "then",
+  "over",
+  "under",
+  "about",
+  "into",
+  "their",
+  "they",
+  "them"
+]);
+function normalize(text) {
+  return text.toLowerCase().replace(/[^\p{L}\p{N}\s]+/gu, " ").replace(/\s+/g, " ").trim();
+}
+function contentWords(claim, minWordLength) {
+  const words = normalize(claim).split(" ").filter((word) => word.length >= minWordLength && !stopwords.has(word));
+  return [...new Set(words)];
+}
+function groundClaimInText(claim, pageText, options = {}) {
+  const minOverlap = options.minOverlap ?? 0.7;
+  const minWordLength = Math.max(1, options.minWordLength ?? 3);
+  const claimTrimmed = claim.trim();
+  if (!claimTrimmed) return { grounded: false, mode: "empty-claim", overlap: 0, missingWords: [] };
+  if (!pageText.trim()) return { grounded: false, mode: "empty-text", overlap: 0, missingWords: [] };
+  const haystackLower = pageText.toLowerCase();
+  if (haystackLower.includes(claimTrimmed.toLowerCase())) {
+    return { grounded: true, mode: "verbatim", overlap: 1, missingWords: [] };
+  }
+  const haystackNorm = normalize(pageText);
+  const claimNorm = normalize(claimTrimmed);
+  if (claimNorm && haystackNorm.includes(claimNorm)) {
+    return { grounded: true, mode: "normalized", overlap: 1, missingWords: [] };
+  }
+  const words = contentWords(claimTrimmed, minWordLength);
+  if (words.length === 0) {
+    return { grounded: false, mode: "absent", overlap: 0, missingWords: [] };
+  }
+  const present = words.filter(
+    (word) => new RegExp(`\\b${word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`).test(haystackNorm)
+  );
+  const missingWords = words.filter((word) => !present.includes(word));
+  const overlap = present.length / words.length;
+  const grounded = overlap >= minOverlap;
+  return { grounded, mode: grounded ? "overlap" : "absent", overlap, missingWords };
+}
+function createClaimGroundingVerifier(options = {}) {
+  const onMissingClaim = options.onMissingClaim ?? "reject";
+  return async function verifySource(source, ctx) {
+    const claim = citedClaimOf(source);
+    if (!claim) {
+      if (onMissingClaim === "reject") {
+        return {
+          accept: false,
+          reason: "no cited claim: claim-grounding mode requires every source to declare its claim"
+        };
+      }
+      return options.relevanceVerifier ? options.relevanceVerifier(source, ctx) : { accept: true };
+    }
+    const grounding = groundClaimInText(claim, source.text, options);
+    if (!grounding.grounded) {
+      const detail = grounding.mode === "empty-text" ? "fetched page has no text" : `claim not found in the fetched page (overlap ${(grounding.overlap * 100).toFixed(0)}%${grounding.missingWords.length ? `, missing: ${grounding.missingWords.slice(0, 6).join(", ")}` : ""})`;
+      return {
+        accept: false,
+        reason: `misattributed citation: ${detail}`
+      };
+    }
+    if (options.relevanceVerifier) return options.relevanceVerifier(source, ctx);
+    return { accept: true };
+  };
+}
+function createClaimDecorator(options = {}) {
+  const maxTokens = options.maxTokens ?? 1200;
+  return async function decorate(source, goal) {
+    const router = options.router ?? createTangleRouterClient(options.router_options);
+    const excerpt = source.text.slice(0, 1500);
+    const system = "You extract the single most important factual claim a researcher would cite a page for. State ONE concrete, checkable sentence using the page's own key terms and numbers. Do NOT invent facts not in the excerpt. Respond with ONLY the claim sentence, no prose.";
+    const user = [
+      `Research goal: ${goal}`,
+      `Page title: ${source.title ?? "(none)"}`,
+      `Page excerpt:
+${excerpt}`,
+      "The single claim this page should be cited for:"
+    ].join("\n\n");
+    let raw = "";
+    try {
+      raw = await router.chat(
+        [
+          { role: "system", content: system },
+          { role: "user", content: user }
+        ],
+        maxTokens
+      );
+    } catch {
+      return source;
+    }
+    const claim = raw.trim().split("\n")[0]?.trim();
+    if (!claim) return source;
+    return withCitedClaim(source, claim);
+  };
 }
 
 // src/discovery.ts
@@ -1145,408 +1784,133 @@ async function runTwoAgentResearchLoop(options) {
         signal: options.signal
       });
       driverNotes = driverContribution.notes;
-      driverSources = await registerSources(options, driverContribution.sources ?? []);
-      writtenPages.push(...await applyPages(options.root, driverContribution, driverSources));
-      index = await buildKnowledgeIndex(options.root);
-      readiness = readinessFor(options, index);
-    }
-    ready = isReady(readiness?.report);
-    const remainingGaps = gapsFromReadiness(readiness);
-    steer = ready ? void 0 : foldGaps(options.driver, remainingGaps);
-    const step = {
-      round: round2,
-      gaps,
-      acceptedWorkerSources,
-      rejectedWorkerSources,
-      driverSources,
-      writtenPages,
-      readiness,
-      ready,
-      event: createKnowledgeEvent({
-        type: "research.iteration",
-        actor: options.actor,
-        target: options.root,
-        metadata: {
-          goal: options.goal,
-          round: round2,
-          ready,
-          acceptedWorkerSourceCount: acceptedWorkerSources.length,
-          rejectedWorkerSourceCount: rejectedWorkerSources.length,
-          driverSourceCount: driverSources.length,
-          writtenPageCount: writtenPages.length,
-          remainingGapCount: remainingGaps.length
-        }
-      }),
-      notes: { worker: workerContribution.notes, driver: driverNotes }
-    };
-    steps.push(step);
-    await options.onRound?.(step);
-  }
-  return {
-    root: options.root,
-    goal: options.goal,
-    rounds: steps.length,
-    ready,
-    index,
-    readiness,
-    steps
-  };
-}
-function isReady(report) {
-  if (!report) return false;
-  return report.blockingMissingRequirements.length === 0;
-}
-function gapsFromReadiness(readiness) {
-  if (!readiness) return [];
-  const blocking = readiness.report.blockingMissingRequirements.map(
-    (requirement) => gapFor(requirement, readiness, true)
-  );
-  const nonBlocking = readiness.report.nonBlockingGaps.map(
-    (requirement) => gapFor(requirement, readiness, false)
-  );
-  return [...blocking, ...nonBlocking];
-}
-function gapFor(requirement, readiness, blocking) {
-  const spec = readiness.requirements.find((entry) => entry.id === requirement.id);
-  const query = typeof spec?.metadata?.query === "string" ? spec.metadata.query : requirement.description;
-  return { id: requirement.id, description: requirement.description, query, blocking };
-}
-function foldGaps(driver, gaps) {
-  if (gaps.length === 0) return void 0;
-  if (driver.foldGaps) return driver.foldGaps(gaps);
-  return [
-    "The knowledge base is still missing the following. Prioritise these next round:",
-    ...gaps.map(
-      (gap) => `- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`
-    )
-  ].join("\n");
-}
-function isDuplicate(source, existingUris, accepted) {
-  return existingUris.has(source.uri) || accepted.some((candidate) => candidate.uri === source.uri);
-}
-async function registerSources(options, sources) {
-  const records = [];
-  for (const source of sources) {
-    records.push(await addSourceText(options.root, source, options.sourceOptions));
-  }
-  return records;
-}
-async function applyPages(root, contribution, acceptedSources) {
-  if (acceptedSources.length === 0) return [];
-  const parts = [];
-  if (contribution.proposalText) parts.push(contribution.proposalText);
-  const built = contribution.buildPages?.(acceptedSources);
-  if (built) parts.push(built);
-  if (parts.length === 0) return [];
-  const applied = await applyKnowledgeWriteBlocks(root, parts.join("\n"));
-  return applied.written;
-}
-function requireReadiness(readiness, options) {
-  if (readiness) return readiness;
-  return buildEvalKnowledgeBundle({
-    ...options.readiness ?? {},
-    taskId: options.readinessTaskId ?? options.goal,
-    index: emptyIndex(options.root),
-    specs: []
-  });
-}
-function emptyIndex(root) {
-  return {
-    root,
-    generatedAt: (/* @__PURE__ */ new Date(0)).toISOString(),
-    sources: [],
-    pages: [],
-    graph: { nodes: [], edges: [] }
-  };
-}
-function sourceMatchesGaps(source, index, gaps) {
-  const haystack = `${source.title ?? ""}
-${source.text}`.toLowerCase();
-  const hits = [];
-  for (const gap of gaps) {
-    for (const token of gap.query.toLowerCase().split(/\s+/).filter(Boolean)) {
-      if (haystack.includes(token)) {
-        hits.push(...searchKnowledge(index, gap.query, 1));
-        break;
-      }
-    }
-  }
-  return hits;
-}
-
-// src/web-research-worker.ts
-var DEFAULT_MODEL = "glm-5.2";
-var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
-var MIN_MAX_TOKENS = 1200;
-var RouterError = class extends Error {
-  constructor(status, message) {
-    super(`router ${status}: ${message}`);
-    this.status = status;
-    this.name = "RouterError";
-  }
-  status;
-};
-function createTangleRouterClient(options = {}) {
-  const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
-  const apiKey = options.apiKey ?? process.env.TANGLE_API_KEY;
-  if (!apiKey) {
-    throw new RouterError(401, "no TANGLE_API_KEY (pass apiKey or set the env var)");
-  }
-  const model = options.model ?? DEFAULT_MODEL;
-  const headers = {
-    "Content-Type": "application/json",
-    Authorization: `Bearer ${apiKey}`
-  };
-  return {
-    async search(query, opts) {
-      const res = await fetch(`${baseUrl}/search`, {
-        method: "POST",
-        headers,
-        signal: options.signal,
-        body: JSON.stringify({
-          query,
-          ...options.searchProvider ? { provider: options.searchProvider } : {},
-          ...opts?.maxResults != null ? { maxResults: opts.maxResults } : {}
-        })
-      });
-      if (!res.ok) {
-        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
-      }
-      const body = await res.json();
-      return (body.data ?? []).filter((hit) => typeof hit?.url === "string" && hit.url.length > 0).map((hit) => ({ title: hit.title ?? hit.url, url: hit.url, snippet: hit.snippet }));
-    },
-    async chat(messages, maxTokens) {
-      const max_tokens = Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS);
-      const res = await fetch(`${baseUrl}/chat/completions`, {
-        method: "POST",
-        headers,
-        signal: options.signal,
-        body: JSON.stringify({ model, messages, max_tokens, temperature: 0.2, stream: false })
-      });
-      if (!res.ok) {
-        throw new RouterError(res.status, await res.text().catch(() => res.statusText));
-      }
-      const body = await res.json();
-      return body.choices?.[0]?.message?.content ?? "";
-    }
-  };
-}
-function resolveRouter(opts) {
-  return opts.router ?? createTangleRouterClient(opts.router_options);
-}
-function createWebResearchWorker(options = {}) {
-  const queriesPerGap = Math.max(1, options.queriesPerGap ?? 2);
-  const resultsPerQuery = Math.max(1, options.resultsPerQuery ?? 3);
-  const maxSourcesPerRound = Math.max(1, options.maxSourcesPerRound ?? 6);
-  const minTextChars = Math.max(1, options.minTextChars ?? 200);
-  const maxTextChars = Math.max(minTextChars, options.maxTextChars ?? 4e3);
-  return async (ctx) => {
-    const router = resolveRouter(options);
-    const targetGaps = ctx.gaps.filter((gap) => gap.blocking);
-    const gaps = targetGaps.length > 0 ? targetGaps : ctx.gaps;
-    if (gaps.length === 0) {
-      return { sources: [], notes: "no open gaps to research" };
-    }
-    const queries = await formSearchQueries(router, ctx, gaps, queriesPerGap);
-    const proposals = [];
-    const seenUris = /* @__PURE__ */ new Set();
-    for (const query of queries) {
-      if (proposals.length >= maxSourcesPerRound) break;
-      if (ctx.signal?.aborted) break;
-      let hits;
-      try {
-        hits = await router.search(query, { maxResults: resultsPerQuery });
-      } catch (error) {
-        if (error.name === "AbortError") break;
-        continue;
-      }
-      for (const hit of hits) {
-        if (proposals.length >= maxSourcesPerRound) break;
-        if (seenUris.has(hit.url)) continue;
-        const fetched = await politeFetch(hit.url, {
-          signal: ctx.signal,
-          cacheDir: options.cacheDir
-        });
-        if (!fetched.verifiable) continue;
-        const text = htmlToText(fetched.body).slice(0, maxTextChars);
-        if (text.length < minTextChars) continue;
-        seenUris.add(hit.url);
-        proposals.push({
-          uri: hit.url,
-          title: hit.title || hit.url,
-          text,
-          // We just fetched + verified this page, so stamp `lastVerifiedAt` with
-          // fetch time. Do NOT set `validUntil`: a live page has no inherent
-          // future expiry, and the readiness freshness check treats any
-          // `validUntil <= now` as EXPIRED (score 0). The page's `Last-Modified`
-          // is a PAST date, so writing it here would mark every real source stale
-          // and zero out coverage. Record it as provenance metadata instead.
-          lastVerifiedAt: fetched.fetchedAt,
-          metadata: {
-            discoveredVia: query,
-            snippet: hit.snippet ?? "",
-            goal: ctx.goal,
-            sourceUpdatedAt: fetched.sourceUpdatedAt
-          }
-        });
-      }
+      driverSources = await registerSources(options, driverContribution.sources ?? []);
+      writtenPages.push(...await applyPages(options.root, driverContribution, driverSources));
+      index = await buildKnowledgeIndex(options.root);
+      readiness = readinessFor(options, index);
     }
-    return {
-      sources: proposals,
-      buildPages: buildCitingPages(proposals),
-      notes: `web-research worker: ${queries.length} queries \u2192 ${proposals.length} fetched sources`
+    ready = isReady(readiness?.report);
+    const remainingGaps = gapsFromReadiness(readiness);
+    steer = ready ? void 0 : foldGaps(options.driver, remainingGaps);
+    const step = {
+      round: round2,
+      gaps,
+      acceptedWorkerSources,
+      rejectedWorkerSources,
+      driverSources,
+      writtenPages,
+      readiness,
+      ready,
+      event: createKnowledgeEvent({
+        type: "research.iteration",
+        actor: options.actor,
+        target: options.root,
+        metadata: {
+          goal: options.goal,
+          round: round2,
+          ready,
+          acceptedWorkerSourceCount: acceptedWorkerSources.length,
+          rejectedWorkerSourceCount: rejectedWorkerSources.length,
+          driverSourceCount: driverSources.length,
+          writtenPageCount: writtenPages.length,
+          remainingGapCount: remainingGaps.length
+        }
+      }),
+      notes: { worker: workerContribution.notes, driver: driverNotes }
     };
+    steps.push(step);
+    await options.onRound?.(step);
+  }
+  return {
+    root: options.root,
+    goal: options.goal,
+    rounds: steps.length,
+    ready,
+    index,
+    readiness,
+    steps
   };
 }
-async function formSearchQueries(router, ctx, gaps, queriesPerGap) {
-  const gapLines = gaps.map((gap, i) => `${i + 1}. ${gap.description} (readiness query: "${gap.query}")`).join("\n");
-  const want = gaps.length * queriesPerGap;
-  const system = "You are a research librarian. Turn knowledge gaps into precise web search queries that will surface authoritative primary sources (papers, docs, standards, official pages). Return ONLY a JSON array of query strings, no prose.";
-  const user = [
-    `Research goal: ${ctx.goal}`,
-    ctx.steer ? `Steer from the coordinator:
-${ctx.steer}` : "",
-    `Open knowledge gaps:
-${gapLines}`,
-    `Return up to ${want} search query strings as a JSON array (e.g. ["query one","query two"]).`
-  ].filter(Boolean).join("\n\n");
-  let raw = "";
-  try {
-    raw = await router.chat(
-      [
-        { role: "system", content: system },
-        { role: "user", content: user }
-      ],
-      MIN_MAX_TOKENS
-    );
-  } catch {
-    raw = "";
-  }
-  const parsed = parseQueryList(raw);
-  const fromLlm = parsed.slice(0, want);
-  if (fromLlm.length > 0) return dedupeStrings(fromLlm);
-  return dedupeStrings(gaps.map((gap) => gap.query || gap.description));
+function isReady(report) {
+  if (!report) return false;
+  return report.blockingMissingRequirements.length === 0;
 }
-function parseQueryList(raw) {
-  const text = raw.trim();
-  if (!text) return [];
-  const candidates = [];
-  const arrayMatch = text.match(/\[[\s\S]*\]/);
-  if (arrayMatch) {
-    try {
-      const arr = JSON.parse(arrayMatch[0]);
-      for (const item of arr)
-        if (typeof item === "string" && item.trim()) candidates.push(item.trim());
-    } catch {
-    }
-  }
-  if (candidates.length === 0) {
-    for (const line of text.split("\n")) {
-      const cleaned = line.replace(/^\s*(?:[-*]|\d+[.)])\s*/, "").replace(/^["']|["']$/g, "").trim();
-      if (cleaned && cleaned.length > 2 && !cleaned.startsWith("{")) candidates.push(cleaned);
-    }
-  }
-  return candidates;
+function gapsFromReadiness(readiness) {
+  if (!readiness) return [];
+  const blocking = readiness.report.blockingMissingRequirements.map(
+    (requirement) => gapFor(requirement, readiness, true)
+  );
+  const nonBlocking = readiness.report.nonBlockingGaps.map(
+    (requirement) => gapFor(requirement, readiness, false)
+  );
+  return [...blocking, ...nonBlocking];
 }
-function dedupeStrings(values) {
-  const seen = /* @__PURE__ */ new Set();
-  const out = [];
-  for (const value of values) {
-    const key = value.toLowerCase().trim();
-    if (key && !seen.has(key)) {
-      seen.add(key);
-      out.push(value.trim());
-    }
+function gapFor(requirement, readiness, blocking) {
+  const spec = readiness.requirements.find((entry) => entry.id === requirement.id);
+  const query = typeof spec?.metadata?.query === "string" ? spec.metadata.query : requirement.description;
+  return { id: requirement.id, description: requirement.description, query, blocking };
+}
+function foldGaps(driver, gaps) {
+  if (gaps.length === 0) return void 0;
+  if (driver.foldGaps) return driver.foldGaps(gaps);
+  return [
+    "The knowledge base is still missing the following. Prioritise these next round:",
+    ...gaps.map(
+      (gap) => `- (${gap.blocking ? "blocking" : "soft"}) ${gap.description} [${gap.id}]`
+    )
+  ].join("\n");
+}
+function isDuplicate(source, existingUris, accepted) {
+  return existingUris.has(source.uri) || accepted.some((candidate) => candidate.uri === source.uri);
+}
+async function registerSources(options, sources) {
+  const records = [];
+  for (const source of sources) {
+    records.push(await addSourceText(options.root, source, options.sourceOptions));
   }
-  return out;
+  return records;
 }
-function buildCitingPages(proposals) {
-  return (acceptedSources) => {
-    if (acceptedSources.length === 0) return void 0;
-    const blocks = acceptedSources.map((record) => {
-      const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri);
-      const uri = proposal?.uri ?? record.id;
-      const slug = uri.replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 120) || record.id;
-      const title = proposal?.title ?? record.id;
-      const body = proposal?.text ?? "";
-      return [
-        `---FILE: knowledge/${slug}.md---`,
-        "---",
-        `title: ${escapeYaml(title)}`,
-        `sources: ["${record.id}"]`,
-        `source_url: ${uri}`,
-        "---",
-        `# ${title}`,
-        "",
-        body,
-        "",
-        `Source: ${uri}`,
-        "---END FILE---"
-      ].join("\n");
-    });
-    return blocks.join("\n");
-  };
+async function applyPages(root, contribution, acceptedSources) {
+  if (acceptedSources.length === 0) return [];
+  const parts = [];
+  if (contribution.proposalText) parts.push(contribution.proposalText);
+  const built = contribution.buildPages?.(acceptedSources);
+  if (built) parts.push(built);
+  if (parts.length === 0) return [];
+  const applied = await applyKnowledgeWriteBlocks(root, parts.join("\n"));
+  return applied.written;
 }
-function escapeYaml(value) {
-  return value.replace(/[\r\n]+/g, " ").replace(/"/g, "'").trim();
+function requireReadiness(readiness, options) {
+  if (readiness) return readiness;
+  return buildEvalKnowledgeBundle({
+    ...options.readiness ?? {},
+    taskId: options.readinessTaskId ?? options.goal,
+    index: emptyIndex(options.root),
+    specs: []
+  });
 }
-function createVerifyingResearchDriver(options = {}) {
-  const acceptOnParseFailure = options.acceptOnParseFailure ?? false;
+function emptyIndex(root) {
   return {
-    async verifySource(source, ctx) {
-      const router = resolveRouter(options);
-      const gapLines = ctx.gaps.map((gap) => `- ${gap.description} (query: "${gap.query}")`).join("\n");
-      const acceptedTitles = ctx.acceptedThisRound.map((accepted) => `- ${accepted.title ?? accepted.uri}`).join("\n");
-      const excerpt = source.text.slice(0, 1500);
-      const system = 'You verify whether a fetched web source belongs in a curated knowledge base. Accept a source ONLY if it is genuinely on-topic for the research goal and helps close one of the open gaps, AND it is not a near-duplicate of an already-accepted source. Reject spam, listicles, off-topic pages, marketing, and near-duplicates. Respond with ONLY a JSON object: {"accept": true|false, "reason": "<short reason>"}.';
-      const user = [
-        `Research goal: ${ctx.goal}`,
-        `Open gaps:
-${gapLines || "(none specified)"}`,
-        acceptedTitles ? `Already accepted this round:
-${acceptedTitles}` : "Nothing accepted yet this round.",
-        `Candidate source:
-URL: ${source.uri}
-Title: ${source.title ?? "(none)"}
-Excerpt:
-${excerpt}`,
-        'Verdict as JSON {"accept": boolean, "reason": string}:'
-      ].join("\n\n");
-      let raw = "";
-      try {
-        raw = await router.chat(
-          [
-            { role: "system", content: system },
-            { role: "user", content: user }
-          ],
-          MIN_MAX_TOKENS
-        );
-      } catch (error) {
-        if (error.name === "AbortError") throw error;
-        return acceptOnParseFailure ? { accept: true } : { accept: false, reason: `verifier unavailable: ${error.message}` };
-      }
-      const verdict = parseVerdict(raw);
-      if (verdict) return verdict;
-      return acceptOnParseFailure ? { accept: true } : { accept: false, reason: "verifier returned an unparseable verdict" };
-    }
+    root,
+    generatedAt: (/* @__PURE__ */ new Date(0)).toISOString(),
+    sources: [],
+    pages: [],
+    graph: { nodes: [], edges: [] }
   };
 }
-function parseVerdict(raw) {
-  const text = raw.trim();
-  if (!text) return null;
-  const objMatch = text.match(/\{[\s\S]*\}/);
-  if (!objMatch) return null;
-  try {
-    const parsed = JSON.parse(objMatch[0]);
-    if (typeof parsed.accept !== "boolean") return null;
-    if (parsed.accept) return { accept: true };
-    return {
-      accept: false,
-      reason: typeof parsed.reason === "string" && parsed.reason.trim() ? parsed.reason.trim() : "rejected by verifier"
-    };
-  } catch {
-    return null;
+function sourceMatchesGaps(source, index, gaps) {
+  const haystack = `${source.title ?? ""}
+${source.text}`.toLowerCase();
+  const hits = [];
+  for (const gap of gaps) {
+    for (const token of gap.query.toLowerCase().split(/\s+/).filter(Boolean)) {
+      if (haystack.includes(token)) {
+        hits.push(...searchKnowledge(index, gap.query, 1));
+        break;
+      }
+    }
   }
+  return hits;
 }
 export {
   AgentMemoryHitSchema,
@@ -1581,7 +1945,14 @@ export {
   buildEvalKnowledgeBundle,
   buildKnowledgeGraph,
   buildKnowledgeIndex,
+  canonicalizeUrl,
   chunkMarkdown,
+  citedClaimKey,
+  citedClaimOf,
+  contentKey,
+  createAdaptiveResearchDriver,
+  createClaimDecorator,
+  createClaimGroundingVerifier,
   createCornellLiiSource,
   createD1FreshnessStoreStub,
   createFileSystemFreshnessStore,
@@ -1602,6 +1973,7 @@ export {
   extractWikilinks,
   firstMatch,
   formatFrontmatter,
+  groundClaimInText,
   htmlToText,
   initKnowledgeBase,
   innerHtmlById,
@@ -1638,7 +2010,9 @@ export {
   stripFrontmatter,
   textSourceAdapter,
   tokenizeQuery,
+  triageSource,
   validateKnowledgeIndex,
+  withCitedClaim,
   writeJson,
   writeKnowledgeIndex,
   writeSourceRegistry