extract-webpage 1.2.5

package/src/search/tavily.ts

@@ -0,0 +1,106 @@
+/**
+ * @module research/search/tavily
+ * @description Research library module.
+ */
+import axios from "axios";
+import configManager from "../config";
+
+interface TavilySearchOptions {
+  searchDepth?: "basic" | "advanced";
+  maxResults?: number;
+  includeDomains?: string[];
+  excludeDomains?: string[];
+}
+
+interface TavilySearchResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+  raw_content?: string;
+}
+
+interface TavilyResponse {
+  results: TavilySearchResult[];
+  query: string;
+}
+
+export const searchTavily = async (
+  query: string,
+  opts?: TavilySearchOptions,
+): Promise<{ results: TavilySearchResult[]; suggestions: string[] }> => {
+  const tavilyApiKey =
+    configManager.getConfig("search.tavilyApiKey", "") ||
+    (typeof process !== "undefined" ? process.env.TAVILY_API_KEY : "") ||
+    "";
+
+  if (!tavilyApiKey) {
+    throw new Error(
+      "Tavily API key not configured. Please add your API key in Settings > Search.",
+    );
+  }
+
+  // Sanitize query - Tavily has a maximum query length limit (~400 characters)
+  // If the query is too long, truncate it to avoid 400 Bad Request errors
+  let sanitizedQuery = query.trim();
+  if (sanitizedQuery.length > 400) {
+    console.warn(`[Tavily] Query too long (${sanitizedQuery.length} chars), truncating to 400 chars`);
+    sanitizedQuery = sanitizedQuery.slice(0, 400);
+  }
+
+  try {
+    const response = await axios.post<TavilyResponse>(
+      "https://api.tavily.com/search",
+      {
+        api_key: tavilyApiKey,
+        query: sanitizedQuery,
+        search_depth: opts?.searchDepth || "basic",
+        max_results: opts?.maxResults || 10,
+        include_domains: opts?.includeDomains || [],
+        exclude_domains: opts?.excludeDomains || [],
+        include_answer: false,
+        include_raw_content: false,
+      },
+      {
+        headers: {
+          "Content-Type": "application/json",
+        },
+      },
+    );
+
+    const results = response.data.results || [];
+
+    return {
+      results: results.map((r) => ({
+        title: r.title,
+        url: r.url,
+        content: r.content,
+        score: r.score,
+        raw_content: r.raw_content,
+      })),
+      suggestions: [],
+    };
+  } catch (error: any) {
+    console.error("Tavily search error:", error);
+
+    // Provide more context for debugging
+    if (error.response?.status === 400) {
+      console.error("Tavily 400 error - Query length:", sanitizedQuery.length);
+      console.error("Tavily 400 error - Query preview:", sanitizedQuery.slice(0, 200));
+    }
+
+    throw new Error(
+      `Tavily search failed: ${error.response?.data?.error || error.message}`,
+    );
+  }
+};
+
+export const getTavilyApiKey = () =>
+  configManager.getConfig("search.tavilyApiKey", "") ||
+  (typeof process !== "undefined" ? process.env.TAVILY_API_KEY : "") ||
+  "";
+
+export const isTavilyConfigured = () => {
+  const apiKey = getTavilyApiKey();
+  return apiKey && apiKey.length > 0;
+};

package/src/search/url-to-html.ts

@@ -0,0 +1,278 @@
+import grab from "../utils/grab";
+
+/**
+ * ### Tardigrade the Web Crawler
+ *
+ * 1. **Use Fetch API, check for bot detection.
+ * Scrape  any domain's URL to get its HTML, JSON, or arraybuffer.<br />
+ * Scraping internet pages is a [free speech right
+ * ](https://blog.apify.com/is-web-scraping-legal/).
+ * 2. Features: timeout, redirects, default UA, referer as google, and bot
+ * detection checking. <br />
+ * 3. If fetch method does not get needed HTML, use Docker proxy as backup.
+ *
+ * 4. [Setup Docker](https://github.com/vtempest/ai-research-agent/tree/master/src/crawler)
+ *  container with NodeJS server API renders with puppeteer DOM to get all HTML loaded by
+ *  secondary in-page API requests after the initial page request, including user login and cookie storage.
+ * 5. Bypass Cloudflare bot check: A webpage proxy that request
+ * through Chromium (puppeteer) - can be used to bypass Cloudflare
+ * anti bot using cookie id javascript method.
+ * 6. Send your request to the server with the port 3000 and add your URL to the "url"
+ *  query string like this: `http://localhost:3000/?url=https://example.org`
+ *
+ * 7. Optional: Setup residential IP proxy to access sites that IP-block datacenters
+ *  and manage rotation with [Scrapoxy](https://scrapoxy.io). Recommended:
+ * [Hypeproxy](https://hypeproxy.io/products/static-residential-proxies)
+ * [NinjasProxy](https://ninjasproxy.com/residential-proxies/)
+ * [Proxy-Cheap](https://app.proxy-cheap.com/order)
+ * [LiveProxies](https://liveproxies.io/rotating-residential-proxies-pricing)
+ *
+ * @param {string} url - any domain's URL
+ * @param {Object} [options]
+ * @param {number} options.timeout default=5 -  abort request if not retrived, in seconds
+ * @param {number} options.maxRedirects default=3 - max redirects to follow
+ * @param {number} options.checkBotDetection default=true - check for bot detection messages
+ * @param {number} options.changeReferer default=true - set referer as google
+ * @param {number} options.userAgentIndex default=0 - index of [google bot, default chrome]
+ * @param {string} options.proxy default=false - use proxy url
+ * @param {boolean} options.checkRobotsAllowed default=false - check robots.txt rules
+ * @returns {Promise<string>} -  HTML, JSON, arraybuffer, or error object
+ * @category Extract
+ * @example await scrapeURL("https://hckrnews.com", {timeout: 5, userAgentIndex: 1})
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @license MIT
+ */
+export async function scrapeURL(url, options = {} as any) {
+  // try {
+  let {
+    timeout = 5,
+    checkBotDetection = true,
+    maxRedirects = 3,
+    changeReferer = 0,
+    userAgentIndex = 0,
+    proxy = null,
+    useProxyAsBackup = true,
+    checkRobotsAllowed = false,
+  } = options;
+
+  if (checkRobotsAllowed) {
+    const rules = await fetchScrapingRules(url);
+    if (!isAllowedToScrape(rules, url)) {
+      return { error: "Robots.txt forbids to scrape there" };
+    }
+  }
+
+  if (proxy) url = proxy + url;
+
+  var userAgentStrings = [
+    "Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible ; Googlebot/2.1 ; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)",
+  ];
+
+  var headers = {
+    ...options,
+    "User-Agent": userAgentStrings[userAgentIndex],
+    signal: AbortSignal.timeout(timeout * 1000),
+    accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "en-US,en;q=0.9",
+  };
+
+  if (changeReferer) headers["Referer"] = "https://www.google.com/";
+
+  let response;
+  try {
+    response = await grab(url, {
+      ...headers,
+      responseType: "raw",
+    });
+  } catch (e) {
+    return { error: "Error in fetch", msg: e.message };
+  }
+
+  if (response.redirected) {
+    if (maxRedirects <= 0) return { error: "Max redirects exceeded" };
+    maxRedirects--;
+    options = { ...options, maxRedirects };
+
+    return scrapeURL(response.url, options);
+  }
+
+  //return based on content type
+  const contentType = response.headers.get("Content-Type");
+
+  // if (contentType.includes("application/json")) {
+  //   return await response.json();
+  // } else
+  // if (contentType.includes("text")) {
+  var html = await response.text();
+
+  if (checkBotDetection && checkHTMLForBotDetection(html)) {
+    html = await scrapeJINA(url, timeout);
+
+    //if all methods fail -- return jina
+    if (checkBotDetection && checkHTMLForBotDetection(html))
+      return { error: "Bot detected" }; //, html: response.html };
+  }
+
+  return html;
+}
+
+/**
+ * As backup, scrape with JINA to get html
+ * @param {string} url
+ * @returns {Promise<string>}
+ */
+export async function scrapeJINA(url, timeout = 15) {
+  let articleExtract = "";
+  try {
+    articleExtract = await grab("https://r.jina.ai/" + url, {
+      timeout: timeout * 1000,
+      responseType: "text",
+    });
+  } catch {
+    return "";
+  }
+
+  //convert Title: to <title>
+  var title = articleExtract.match(/Title: (.*)/)?.[1];
+
+  if (articleExtract.includes("===============\n"))
+    articleExtract = articleExtract
+      .split("===============\n")
+      .slice(1)
+      .join(" ");
+
+  var match = articleExtract.match(/Markdown Content:([\s\S]*)/);
+  articleExtract = match ? match[1] : articleExtract;
+
+  // articleExtract = convertMarkdownToHTML(articleExtract);
+
+  if (title) articleExtract = "<title>" + title + "</title>" + articleExtract;
+
+  return articleExtract;
+}
+
+/**
+ * Check html for bot block messages
+ * @param {string} html
+ * @returns {Boolean} true if bot detection message found
+ */
+function checkHTMLForBotDetection(html) {
+  var commonBlocks = [
+    "Error 403 - Unavailable",
+    "The security system for this website has been triggered",
+    "You do not have permission to view this page.",
+    "Our systems have detected unusual traffic from your computer network.",
+    "Your request has been blocked due to a network policy.",
+    "Cloudflare Ray ID found ",
+    "Please verify you are a human",
+    "Our systems have detected unusual traffic activity from your network. Please complete this reCAPTCHA",
+    "Sorry, we just need to make sure you're not a robot",
+    "Access to this page has been denied",
+    "<p>Please enable JS and disable any ad blocker",
+    "Please make sure your browser supports JavaScript",
+    "Please complete the security check to access",
+    "https://errors.edgesuite.net",
+    "Please enable JS and disable any ad blocker",
+    "The resource you are looking for might have been removed, had its name changed, or is temporarily unavailable.",
+    "We\u2019re currently checking your connection. This shouldn\u2019t take long.",
+    "Generated by cloudfront (CloudFront)",
+    "You don't have permission to access",
+    "The request could not be satisfied.",
+    "Enable JavaScript and cookies to continue",
+    "Something went wrong. Wait a moment and try again.",
+    "You\u2019re using a web browser that isn\u2019t supported",
+    "403 Forbidden",
+    "504 Gateway Timeout",
+    "You\u2019re Temporarily Blocked",
+    "Our systems have detected unusual activity",
+    "Agree & Join LinkedIn",
+    "Verifying you are human. This may take a few seconds",
+    "500 Internal Server Error",
+    "By clicking Continue to join or sign in, you agree to LinkedIn",
+    "Enable JS in your browser",
+    "Verifying you are human",
+    "Your request has been blocked",
+    "You've been blocked by network security",
+    "You've hit the rate limit.",
+  ];
+
+  return (
+    html &&
+    typeof html?.indexOf !== "undefined" &&
+    commonBlocks.filter((m) => html?.indexOf(m) > -1).length > 0
+  );
+}
+
+/**
+ * Fetches and parses the robots.txt file for a given URL.
+ * @param {string} url - The base URL to fetch the robots.txt from.
+ * @returns {Promise<Object>} A JSON object representing the parsed robots.txt.
+ */
+export async function fetchScrapingRules(url) {
+  const robotsUrl = `https://${url.split("//")[1].split("/")[0]}/robots.txt`;
+  let content;
+  try {
+    content = await grab(robotsUrl, { responseType: "text" });
+  } catch (e) {
+    return { error: "No robots.txt found" };
+  }
+
+  const rules = {
+    directives: {},
+    crawlDelay: {},
+    sitemaps: [],
+    preferredHost: null,
+  };
+  let currentUserAgents = [];
+
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const [directive, value] = line.split(":").map((s) => s.trim());
+    switch (directive.toLowerCase()) {
+      case "user-agent":
+        currentUserAgents = [value.toLowerCase()];
+        break;
+      case "disallow":
+      case "allow":
+        for (const ua of currentUserAgents) {
+          rules.directives[ua] = rules.directives[ua] || [];
+          rules.directives[ua].push({
+            path: value,
+            allow: directive.toLowerCase() === "allow",
+          });
+        }
+        break;
+      case "crawl-delay":
+        for (const ua of currentUserAgents) {
+          rules.crawlDelay[ua] = parseFloat(value);
+        }
+        break;
+      case "sitemap":
+        rules.sitemaps.push(value);
+        break;
+      case "host":
+        rules.preferredHost = value.toLowerCase();
+        break;
+    }
+  }
+  return rules;
+}
+
+/**
+ * Checks if a given path is allowed for a specific user agent.
+ * //TODO cache rules per domain
+ * @param {Object} rules - The parsed rules from robots.txt.
+ * @param {string} path - The path to check.
+ * @param {string} [userAgent='*'] - The user agent to check for.
+ * @returns {boolean} True if the path is allowed, false otherwise.
+ */
+function isAllowedToScrape(rules, path, userAgent = "*") {
+  const relevantRules =
+    rules.directives[userAgent.toLowerCase()] || rules.directives["*"] || [];
+  for (const rule of relevantRules)
+    if (path.startsWith(rule.path)) return rule.allow;
+
+  return true; // If no rules match, it's allowed by default
+}

package/src/seektopic/fold-keyphrases.ts

@@ -0,0 +1,87 @@
+/**
+ * @fileoverview Utility for grouping and merging similar keyphrases.
+ * Handles normalization and score aggregation for related terms.
+ */
+import type { KeyphraseEntry } from "./types";
+
+/**
+ * Folds smaller keyphrases that are near-subsets of larger ones, merging their
+ * weights and sentence lists upward.
+ *
+ * Two phrases are considered overlapping when they share all-but-one word
+ * (i.e., the number of words in the smaller phrase that do **not** appear in
+ * the larger phrase is fewer than 2). When a merge occurs:
+ * - The larger phrase absorbs a fraction of the smaller one's weight
+ *   (`smallWeight / largerWordCount`).
+ * - Sentence indices are union-merged.
+ * - If the smaller phrase actually outweighs the larger at that point, the
+ *   larger entry adopts the smaller phrase's text (best representative wins).
+ *
+ * Phrases are processed largest-first so that supersets are always evaluated
+ * before their constituent sub-phrases.
+ *
+ * @param keyphrases - Raw scored keyphrases, any order.
+ * @returns Deduplicated, folded array \u2014 larger representative phrases only.
+ *
+ * @example
+ * const folded = foldSubphrases([
+ *   { keyphrase: "machine learning", words: 2, weight: 10, sentences: [0, 1] },
+ *   { keyphrase: "machine",          words: 1, weight:  4, sentences: [0, 2] },
+ * ]);
+ * // "machine" is absorbed into "machine learning"
+ */
+export function foldSubphrases(keyphrases: KeyphraseEntry[]): KeyphraseEntry[] {
+  // Largest n-grams first so supersets are already in `folded` when we check
+  const sorted = keyphrases.slice().sort((a, b) => b.words - a.words);
+  const folded: KeyphraseEntry[] = [];
+
+  outer: for (const curr of sorted) {
+    const currWords = curr.keyphrase.split(" ");
+
+    for (const existing of folded) {
+      const existingWords = existing.keyphrase.split(" ");
+
+      // Count how many words in curr are absent from existing
+      let diff = 0;
+      for (const w of currWords) {
+        if (!existingWords.includes(w) && ++diff >= 2) break;
+      }
+
+      if (diff < 2) {
+        // Absorb curr's weight (proportionally discounted by target word count)
+        existing.weight += curr.weight / existingWords.length;
+
+        // Union of sentence indices
+        const existingSentences = Array.isArray(existing.sentences)
+          ? existing.sentences
+          : typeof existing.sentences === "string"
+            ? existing.sentences.split(",").map(Number).filter((n) => !isNaN(n))
+            : [];
+        const currSentences = Array.isArray(curr.sentences)
+          ? curr.sentences
+          : typeof curr.sentences === "string"
+            ? curr.sentences.split(",").map(Number).filter((n) => !isNaN(n))
+            : [];
+
+        const sentSet = new Set<number>(existingSentences);
+        for (const s of currSentences) sentSet.add(s);
+        existing.sentences = Array.from(sentSet);
+
+        // Promote curr as the representative if it now outweighs existing
+        if (existing.weight < curr.weight) {
+          existing.keyphrase = curr.keyphrase;
+          existing.words = curr.words;
+        }
+
+        continue outer; // merged \u2014 skip push
+      }
+    }
+
+    // No compatible existing phrase found \u2014 add as new entry
+    if (curr.sentences.length >= 1) {
+      folded.push({ ...curr });
+    }
+  }
+
+  return folded;
+}

package/src/seektopic/ngrams.ts

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview Utility for extracting noun-anchored n-grams from tokenized text.
+ * Implements logic to find high-quality candidate phrases.
+ */
+import type { NgramMap, TopicToken } from "./types";
+import { isWordCommonIgnored } from "../tokenize/word-is-ignored";
+
+/** POS tag values that qualify as topic-worthy terms (noun=1, wiki-entity=5) */
+const TOPIC_TAGS = new Set([1, 5]);
+
+/**
+ * Extracts noun-anchored edge-grams from a token array and accumulates them into `nGrams`.
+ *
+ * An edge-gram is a contiguous slice of `nGramSize` tokens where:
+ * - The **first** and **last** tokens are topic entities (noun or wiki title).
+ * - Every token is either a topic entity or a common stop word (e.g. "of", "the").
+ * - Every token meets the `minWordLength` character threshold.
+ *
+ * This allows natural multi-word keyphrases like "state of the art" or
+ * "machine learning" while ignoring pure function-word sequences.
+ *
+ * @param nGramSize - Number of tokens in the slice to evaluate.
+ * @param terms - Full token array for the current sentence.
+ * @param index - Start position for this slice within `terms`.
+ * @param nGrams - Accumulator map mutated in-place: `nGrams[size][phrase] = [sentenceIdx, ...]`.
+ * @param minWordLength - Minimum character length for any word to be included.
+ * @param sentenceIndex - Index of the originating sentence, appended to the phrase's entry.
+ * @returns The same `nGrams` reference (mutated).
+ *
+ * @example
+ * const terms: TopicToken[] = [["machine", 1, 4, ""], ["learning", 1, 5, ""]];
+ * const nGrams: NgramMap = {};
+ * extractNounEdgeGrams(2, terms, 0, nGrams, 3, 0);
+ * // nGrams[2]["machine learning"] === [0]
+ */
+export function extractNounEdgeGrams(
+  nGramSize: number,
+  terms: TopicToken[],
+  index: number,
+  nGrams: NgramMap,
+  minWordLength: number,
+  sentenceIndex: number,
+): NgramMap {
+  // Bail early if the slice would extend past the end of the array
+  if (index + nGramSize - 1 >= terms.length) return nGrams;
+
+  const slice = terms.slice(index, index + nGramSize);
+
+  // Edge tokens must be topic entities
+  if (!TOPIC_TAGS.has(slice[0][1]) || !TOPIC_TAGS.has(slice[nGramSize - 1][1])) return nGrams;
+
+  // Every token must meet length + category requirements
+  for (let i = 0; i < slice.length; i++) {
+    const [word, tag] = slice[i];
+    if (word.length < minWordLength) return nGrams;
+    if (!TOPIC_TAGS.has(tag) && !isWordCommonIgnored(word)) return nGrams;
+  }
+
+  const bucket = nGrams[nGramSize] ?? (nGrams[nGramSize] = {});
+  const phrase = slice.map(t => t[0]).join(" ");
+  (bucket[phrase] ?? (bucket[phrase] = [])).push(sentenceIndex);
+
+  return nGrams;
+}

package/src/seektopic/rank-sentences-keyphrases.ts

@@ -0,0 +1,132 @@
+/**
+ * @fileoverview Implementation of TextRank algorithm for ranking sentences based on keyphrase centrality.
+ * Uses random walk simulations to identify the most relevant sentences in a document.
+ */
+import type { SentenceEntry, RankOptions } from "./types";
+
+/**
+ * Selects the next vertex in a random walk using weighted sampling.
+ *
+ * Instead of building a distribution array (O(totalWeight) space per step),
+ * this scans the adjacency map once with a single random draw \u2014 O(degree) time
+ * and O(1) additional space.
+ *
+ * @param adjacent - Map of neighbour key \u2192 edge weight.
+ * @returns The selected neighbour key.
+ */
+function weightedRandom(adjacent: Map<string, number>): string {
+  let total = 0;
+  for (const w of adjacent.values()) total += w;
+
+  let r = Math.random() * total;
+  for (const [key, w] of adjacent) {
+    r -= w;
+    if (r <= 0) return key;
+  }
+
+  // Floating-point rounding safety: return the last entry
+  return [...adjacent.keys()][adjacent.size - 1];
+}
+
+/**
+ * ### TextRank: Rank Sentences by Centrality to Shared Keyphrases
+ *
+ * Builds a weighted undirected graph where each node is a sentence and edges
+ * connect sentences that share keyphrases. Sentence importance is estimated by
+ * a random-walk simulation (analogous to PageRank): nodes visited more often
+ * during the walk are considered more central to the document's key concepts.
+ *
+ * **Key optimisations vs. na\u00efve implementation:**
+ * - Weighted sampling via a single O(degree) scan instead of an O(weight) array.
+ * - `Set`-based keyphrase intersection (O(1) lookup) instead of `Array.includes`.
+ * - `Map<text, index>` for O(1) weight increment instead of O(n) linear scan.
+ * - Flat `Map<string, Map<string, number>>` graph \u2014 no object-method overhead.
+ * - Periodic forced reset prevents the walk from getting stuck in dense clusters.
+ *
+ * **References:**
+ * 1. Zhao & Xie (2021) \u2014 "An Improved TextRank Multi-feature Fusion Algorithm"
+ *    https://iopscience.iop.org/article/10.1088/1742-6596/2078/1/012021/pdf
+ * 2. Pan et al. (2019) \u2014 "An improved TextRank keywords extraction algorithm"
+ *    https://dl.acm.org/doi/10.1145/3321408.3326659
+ *
+ * @param sentencesWithKeyphrases - Sentences with pre-attached keyphrase lists.
+ * @param options - Walk parameters.
+ * @returns The same array with `weight` set on each sentence, or `undefined`
+ *          if no edges exist (no shared keyphrases between any pair).
+ */
+export function rankSentencesCentralToKeyphrase(
+  sentencesWithKeyphrases: SentenceEntry[],
+  options: RankOptions = {},
+): SentenceEntry[] | undefined {
+  const { iterations = 1000, resetInterval = 100 } = options;
+
+  // \u2500\u2500 Build graph \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  // graph: sentence text \u2192 { neighbour text \u2192 edge weight }
+  const graph = new Map<string, Map<string, number>>();
+
+  // text \u2192 array index for O(1) weight increments during the walk
+  const textToIndex = new Map<string, number>();
+
+  const sentences: SentenceEntry[] = sentencesWithKeyphrases.map((s, i) => {
+    textToIndex.set(s.text, i);
+    return { ...s, weight: 0 };
+  });
+
+  for (let i = 0; i < sentences.length; i++) {
+    const kp1 = sentences[i].keyphrases;
+
+    for (let j = i + 1; j < sentences.length; j++) {
+      const kp2 = sentences[j].keyphrases;
+
+      // Compare shorter list against a Set built from the longer list
+      let longerList = kp1;
+      let shorterList = kp2;
+      if (kp2.length > kp1.length) { longerList = kp2; shorterList = kp1; }
+
+      const shorterSet = new Set(shorterList.map(k => k.keyphrase));
+
+      let edgeWeight = 0;
+      for (const k of longerList) {
+        if (shorterSet.has(k.keyphrase)) edgeWeight += k.weight / 100;
+      }
+
+      if (edgeWeight > 0) {
+        const ti = sentences[i].text;
+        const tj = sentences[j].text;
+
+        if (!graph.has(ti)) graph.set(ti, new Map());
+        if (!graph.has(tj)) graph.set(tj, new Map());
+        graph.get(ti)!.set(tj, edgeWeight);
+        graph.get(tj)!.set(ti, edgeWeight);
+      }
+    }
+  }
+
+  const allVertices = [...graph.keys()];
+  if (allVertices.length === 0) return undefined;
+
+  // \u2500\u2500 Random walk \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  let currentKey = allVertices[Math.floor(Math.random() * allVertices.length)];
+
+  for (let i = 0; i < iterations; i++) {
+    // Periodic reset \u2014 escape dense clusters and improve global coverage
+    if (i % resetInterval === 0) {
+      currentKey = allVertices[Math.floor(Math.random() * allVertices.length)];
+    }
+
+    const adjacent = graph.get(currentKey);
+    if (!adjacent || adjacent.size === 0) {
+      currentKey = allVertices[Math.floor(Math.random() * allVertices.length)];
+      continue;
+    }
+
+    const nextKey = weightedRandom(adjacent);
+
+    const idx = textToIndex.get(nextKey);
+    if (idx !== undefined) sentences[idx].weight++;
+
+    currentKey = nextKey;
+  }
+
+  return sentences;
+}