extract-webpage 1.2.34 → 1.2.36

package/src/seektopic/seektopic-keyphrases.ts

@@ -1,279 +1,279 @@
-/**
- * @fileoverview Main orchestrator for the SEEKTOPIC keyphrase extraction pipeline.
- * Coordinates cleaning, segmentation, LLM extraction, and fallback n-gram ranking.
- */
-import type {
-  NgramMap,
-  KeyphraseEntry,
-  SentenceEntry,
-  SEEKTOPICOptions,
-  SEEKTOPICResult,
-} from "./types";
-import { splitTextToSentences } from "../tokenize/text-to-sentences";
-import { convertTextToTokens } from "../tokenize/text-to-topic-tokens";
-import { extractNounEdgeGrams } from "./ngrams";
-import { foldSubphrases } from "./fold-keyphrases";
-import { weightKeyphrasesBySpecificity } from "./weight-keyphrases";
-import { rankSentencesCentralToKeyphrase } from "./rank-sentences-keyphrases";
-import { weighRelevanceConceptVectorMultiple } from "./vector-search";
-
-/**
- * ### SEEKTOPIC \u2014 Keyphrase & Sentence Extraction
- *
- * Pulls the most important phrases and sentences out of any document.
- * Given raw text, it returns a ranked list of key concepts (e.g. "neural
- * network", "climate change") and, optionally, the sentences that best
- * summarise the document around those concepts.
- *
- * <img src="https://i.imgur.com/gZ4kI1V.png" width="360px" />
- *
- * **How it works \u2014 8-step pipeline:**
- *
- * 1. **Clean** \u2014 strip HTML tags and entities.
- * 2. **Split** \u2014 break text into sentences, respecting abbreviations and URLs.
- * 3. **Topic Extraction** *(LLM Path)* \u2014 sends the first 5000 words to an LLM
- *    to extract the most descriptive key topic phrases labeling the document.
- * 4. **Vector Search** *(LLM Path)* \u2014 calculates cosine similarity embeddings
- *    for all sentences vs topic phrases, finding the top relevant sentences per topic.
- * 5. **Tokenise & Extract** *(Fallback)* \u2014 if LLM fails, identify noun-anchored
- *    n-grams (1-N words).
- * 6. **Score & Fold** *(Fallback)* \u2014 weight phrases and merge subsumed phrases.
- * 7. **Re-weight** *(Fallback)* \u2014 boost Wikipedia entities and rare domain terms.
- * 8. **TextRank** *(Fallback)* \u2014 build sentence similarity graph and run random walk.
- *
- * <video src="https://github.com/user-attachments/assets/73348d63-7671-4e20-8df9-29a13d5b0768"
- *  width="550px" controls />
- *
- * @param docText - Plain text or HTML document to analyse.
- * @param options - Optional tuning parameters (word limits, thresholds, query bias).
- * @returns
- *   - `optionSkipRanking: true` (default) \u2192 `KeyphraseEntry[]` sorted by weight.
- *   - `optionSkipRanking: false` \u2192 `SEEKTOPICResult` with `topSentences`,
- *     `keyphrases`, and the full `sentences` array.
- *
- * @example
- * // Fast: just extract keyphrases
- * const keyphrases = extractSEEKTOPIC(articleText, { phrasesModel });
- * // \u2192 [{ keyphrase: "machine learning", weight: 84 }, ...]
- *
- * @example
- * // Full: keyphrases + summary sentences, biased toward a search query
- * const { topSentences, keyphrases } = extractSEEKTOPIC(articleText, {
- *   phrasesModel,
- *   optionSkipRanking: false,
- *   heavyWeightQuery: "transformer attention",
- *   limitTopSentences: 5,
- * }) as SEEKTOPICResult;
- *
- * @author [ai-research-agent (2024)](https://airesearch.js.org)
- * @category Topics
- */
-export async function extractSEEKTOPIC(
-  docText: string,
-  options: SEEKTOPICOptions = {},
-): Promise<KeyphraseEntry[] | SEEKTOPICResult> {
-  if (typeof docText !== "string") throw new Error("docText must be a string");
-
-  const {
-    phrasesModel,
-    maxWords = 2,
-    minWords = 1,
-    minWordLength = 3,
-    topKeyphrasesPercent = 0.5,
-    limitTopSentences = 5,
-    limitTopKeyphrases = 10,
-    minKeyPhraseLength = 5,
-    heavyWeightQuery = "",
-    removeHTML = true,
-    optionSkipRanking = true,
-    getEnv = () => "",
-  } = options;
-
-  // \u2500\u2500 1. Normalize HTML \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  let text = docText
-    .replace(/</g, " <")
-    .replace(/>/g, "> ")
-    .replace(/&.{2,5};/g, ""); // strip &quot; &amp; &lt; &gt; &nbsp;
-  if (removeHTML) text = text.replace(/<[^>]*>/g, "");
-
-  // \u2500\u2500 2. Sentence segmentation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  const sentencesArray = splitTextToSentences(text);
-
-  // \u2500\u2500 3. LLM Topic Extraction (Starting Step) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  const first5kWords = text.split(/\s+/).slice(0, 5000).join(" ");
-  const llmPrompt = `Extract the 5-10 most important key topic phrases that best 
-  label this document. Return exclusively a comma-separated list of the key topic 
-  phrases. Document text: ${first5kWords}`;
-
-  let llmTopics: string[] = [];
-  try {
-    // `generate-language` moved into the chat-agent-toolkit workspace package.
-    // Load it lazily (and tolerate absence) since the LLM path is an optional
-    // enhancement — the n-gram fallback below covers extraction without it.
-    const { writeLanguageResponse } = await import("chat-agent-toolkit");
-    const aiResponse = await writeLanguageResponse({
-      query: llmPrompt,
-      provider: getEnv("LLM_PROVIDER") || "openai",
-      apiKey: getEnv("LLM_API_KEY") || getEnv("OPENAI_API_KEY") || "dummy",
-    });
-    if (aiResponse.content && !aiResponse.error) {
-      llmTopics = aiResponse.content
-        .split(",")
-        .map((s: string) => s.trim().replace(/^['"]|['"]$/g, ""))
-        .filter(Boolean);
-    }
-  } catch (e) {
-    console.error("LLM topic extraction failed", e);
-  }
-
-  // \u2500\u2500 4. Use vector-search.ts for top topic sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  if (llmTopics.length > 0) {
-    if (optionSkipRanking) {
-      return llmTopics.map((topic: string) => ({
-        keyphrase: topic,
-        sentences: [],
-        words: topic.split(/\s+/).length,
-        weight: 100,
-      }));
-    }
-
-    const { limitTopSentences = 3 } = options;
-    const allRelevant = await weighRelevanceConceptVectorMultiple(
-      sentencesArray,
-      llmTopics,
-      options,
-    );
-
-    const keyphraseObjects: any[] = [];
-    const topSentences: any[] = [];
-
-    for (const topic of llmTopics) {
-      // @ts-ignore
-      const topicRelevant = allRelevant[topic] || [];
-      const topX = topicRelevant.slice(0, limitTopSentences);
-
-      const topicSentences = topX.map((r: any) => ({
-        text: r.content,
-        similarity: r.similarity,
-      }));
-
-      keyphraseObjects.push({
-        keyphrase: topic,
-        topSentences: topicSentences,
-        words: topic.split(/\s+/).length,
-        weight: 100,
-      });
-
-      for (const r of topX) {
-        const sIdx = sentencesArray.indexOf(r.content);
-        topSentences.push({
-          text: r.content,
-          index: sIdx !== -1 ? sIdx : 0,
-          keyphrases: [{ keyphrase: topic, weight: r.similarity }],
-          weight: r.similarity,
-        });
-      }
-    }
-
-    // Deduplicate topSentences
-    const uniqueSentences = Array.from(
-      new Map(topSentences.map((s: any) => [s.text, s])).values(),
-    );
-
-    return {
-      topSentences: uniqueSentences.sort(
-        (a: any, b: any) => b.weight - a.weight,
-      ),
-      keyphrases: keyphraseObjects,
-      sentences: sentencesArray,
-    } as unknown as SEEKTOPICResult;
-  }
-
-  // \u2500\u2500 FALLBACK: Tokenise + extract n-grams \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  const nGrams: NgramMap = {};
-  const sentenceKeysMap: SentenceEntry[] = [];
-
-  for (let idx = 0; idx < sentencesArray.length; idx++) {
-    const sentence = sentencesArray[idx];
-    const tokens = convertTextToTokens(sentence, { phrasesModel });
-
-    sentenceKeysMap.push({
-      text: sentence,
-      index: idx,
-      keyphrases: [],
-      weight: 0,
-    });
-
-    for (let i = 0; i < tokens.length; i++) {
-      for (let n = minWords; n <= maxWords; n++) {
-        extractNounEdgeGrams(n, tokens, i, nGrams, minWordLength, idx);
-      }
-    }
-  }
-
-  // \u2500\u2500 5. Score raw keyphrases \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  let keyphraseObjects: KeyphraseEntry[] = [];
-  for (let n = minWords; n <= maxWords; n++) {
-    if (!nGrams[n]) continue;
-    for (const [keyphrase, sentencesArr] of Object.entries(nGrams[n])) {
-      keyphraseObjects.push({
-        keyphrase,
-        sentences: sentencesArr,
-        words: n,
-        weight: sentencesArr.length * n,
-      });
-    }
-  }
-
-  // \u2500\u2500 6. Fold subphrases + deduplicate \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  const folded = foldSubphrases(keyphraseObjects);
-  const seen = new Set<string>();
-  keyphraseObjects = folded
-    .sort((a, b) => b.weight - a.weight)
-    .filter((k) => !seen.has(k.keyphrase) && seen.add(k.keyphrase))
-    .map((k) => ({
-      ...k,
-      sentences: Array.isArray(k.sentences)
-        ? [...new Set(k.sentences as number[])]
-        : k.sentences,
-    }));
-
-  // \u2500\u2500 7. IDF + wiki-entity + heavy-query weighting \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  keyphraseObjects = weightKeyphrasesBySpecificity(
-    keyphraseObjects,
-    phrasesModel,
-    heavyWeightQuery,
-  )
-    .filter((k) => k.keyphrase.length > minKeyPhraseLength)
-    .sort((a, b) => b.weight - a.weight);
-
-  // Fast path: return keyphrases without TextRank
-  if (optionSkipRanking) return keyphraseObjects;
-
-  // \u2500\u2500 8. Attach keyphrases to sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  for (const { keyphrase, sentences, weight } of keyphraseObjects) {
-    if (Array.isArray(sentences)) {
-      for (const si of sentences) {
-        sentenceKeysMap[si as number]?.keyphrases.push({ keyphrase, weight });
-      }
-    }
-  }
-
-  // \u2500\u2500 9. TextRank \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
-  const ranked = rankSentencesCentralToKeyphrase(sentenceKeysMap);
-
-  const topSentences = (ranked ?? [])
-    .sort((a, b) => b.weight - a.weight)
-    .slice(0, limitTopSentences)
-    .map((s) => ({
-      ...s,
-      keyphrases: [...new Set(s.keyphrases.map((k) => k.keyphrase))],
-    }));
-
-  const keyphrases = keyphraseObjects.slice(0, limitTopKeyphrases).map((k) => ({
-    ...k,
-    sentences: Array.isArray(k.sentences) ? k.sentences.join(",") : k.sentences,
-  }));
-
-  return { topSentences, keyphrases, sentences: sentencesArray };
-}
+/**
+ * @fileoverview Main orchestrator for the SEEKTOPIC keyphrase extraction pipeline.
+ * Coordinates cleaning, segmentation, LLM extraction, and fallback n-gram ranking.
+ */
+import type {
+  NgramMap,
+  KeyphraseEntry,
+  SentenceEntry,
+  SEEKTOPICOptions,
+  SEEKTOPICResult,
+} from "./types";
+import { splitTextToSentences } from "../tokenize/text-to-sentences";
+import { convertTextToTokens } from "../tokenize/text-to-topic-tokens";
+import { extractNounEdgeGrams } from "./ngrams";
+import { foldSubphrases } from "./fold-keyphrases";
+import { weightKeyphrasesBySpecificity } from "./weight-keyphrases";
+import { rankSentencesCentralToKeyphrase } from "./rank-sentences-keyphrases";
+import { weighRelevanceConceptVectorMultiple } from "./vector-search";
+
+/**
+ * ### SEEKTOPIC \u2014 Keyphrase & Sentence Extraction
+ *
+ * Pulls the most important phrases and sentences out of any document.
+ * Given raw text, it returns a ranked list of key concepts (e.g. "neural
+ * network", "climate change") and, optionally, the sentences that best
+ * summarise the document around those concepts.
+ *
+ * <img src="https://i.imgur.com/gZ4kI1V.png" width="360px" />
+ *
+ * **How it works \u2014 8-step pipeline:**
+ *
+ * 1. **Clean** \u2014 strip HTML tags and entities.
+ * 2. **Split** \u2014 break text into sentences, respecting abbreviations and URLs.
+ * 3. **Topic Extraction** *(LLM Path)* \u2014 sends the first 5000 words to an LLM
+ *    to extract the most descriptive key topic phrases labeling the document.
+ * 4. **Vector Search** *(LLM Path)* \u2014 calculates cosine similarity embeddings
+ *    for all sentences vs topic phrases, finding the top relevant sentences per topic.
+ * 5. **Tokenise & Extract** *(Fallback)* \u2014 if LLM fails, identify noun-anchored
+ *    n-grams (1-N words).
+ * 6. **Score & Fold** *(Fallback)* \u2014 weight phrases and merge subsumed phrases.
+ * 7. **Re-weight** *(Fallback)* \u2014 boost Wikipedia entities and rare domain terms.
+ * 8. **TextRank** *(Fallback)* \u2014 build sentence similarity graph and run random walk.
+ *
+ * <video src="https://github.com/user-attachments/assets/73348d63-7671-4e20-8df9-29a13d5b0768"
+ *  width="550px" controls />
+ *
+ * @param docText - Plain text or HTML document to analyse.
+ * @param options - Optional tuning parameters (word limits, thresholds, query bias).
+ * @returns
+ *   - `optionSkipRanking: true` (default) \u2192 `KeyphraseEntry[]` sorted by weight.
+ *   - `optionSkipRanking: false` \u2192 `SEEKTOPICResult` with `topSentences`,
+ *     `keyphrases`, and the full `sentences` array.
+ *
+ * @example
+ * // Fast: just extract keyphrases
+ * const keyphrases = extractSEEKTOPIC(articleText, { phrasesModel });
+ * // \u2192 [{ keyphrase: "machine learning", weight: 84 }, ...]
+ *
+ * @example
+ * // Full: keyphrases + summary sentences, biased toward a search query
+ * const { topSentences, keyphrases } = extractSEEKTOPIC(articleText, {
+ *   phrasesModel,
+ *   optionSkipRanking: false,
+ *   heavyWeightQuery: "transformer attention",
+ *   limitTopSentences: 5,
+ * }) as SEEKTOPICResult;
+ *
+ * @author [ai-research-agent (2024)](https://airesearch.js.org)
+ * @category Topics
+ */
+export async function extractSEEKTOPIC(
+  docText: string,
+  options: SEEKTOPICOptions = {},
+): Promise<KeyphraseEntry[] | SEEKTOPICResult> {
+  if (typeof docText !== "string") throw new Error("docText must be a string");
+
+  const {
+    phrasesModel,
+    maxWords = 2,
+    minWords = 1,
+    minWordLength = 3,
+    topKeyphrasesPercent = 0.5,
+    limitTopSentences = 5,
+    limitTopKeyphrases = 10,
+    minKeyPhraseLength = 5,
+    heavyWeightQuery = "",
+    removeHTML = true,
+    optionSkipRanking = true,
+    getEnv = () => "",
+  } = options;
+
+  // \u2500\u2500 1. Normalize HTML \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  let text = docText
+    .replace(/</g, " <")
+    .replace(/>/g, "> ")
+    .replace(/&.{2,5};/g, ""); // strip &quot; &amp; &lt; &gt; &nbsp;
+  if (removeHTML) text = text.replace(/<[^>]*>/g, "");
+
+  // \u2500\u2500 2. Sentence segmentation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const sentencesArray = splitTextToSentences(text);
+
+  // \u2500\u2500 3. LLM Topic Extraction (Starting Step) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const first5kWords = text.split(/\s+/).slice(0, 5000).join(" ");
+  const llmPrompt = `Extract the 5-10 most important key topic phrases that best 
+  label this document. Return exclusively a comma-separated list of the key topic 
+  phrases. Document text: ${first5kWords}`;
+
+  let llmTopics: string[] = [];
+  try {
+    // `generate-language` moved into the chat-agent-toolkit workspace package.
+    // Load it lazily (and tolerate absence) since the LLM path is an optional
+    // enhancement — the n-gram fallback below covers extraction without it.
+    const { writeLanguageResponse } = await import("chat-agent-toolkit");
+    const aiResponse = await writeLanguageResponse({
+      query: llmPrompt,
+      provider: getEnv("LLM_PROVIDER") || "openai",
+      apiKey: getEnv("LLM_API_KEY") || getEnv("OPENAI_API_KEY") || "dummy",
+    });
+    if (aiResponse.content && !aiResponse.error) {
+      llmTopics = aiResponse.content
+        .split(",")
+        .map((s: string) => s.trim().replace(/^['"]|['"]$/g, ""))
+        .filter(Boolean);
+    }
+  } catch (e) {
+    console.error("LLM topic extraction failed", e);
+  }
+
+  // \u2500\u2500 4. Use vector-search.ts for top topic sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  if (llmTopics.length > 0) {
+    if (optionSkipRanking) {
+      return llmTopics.map((topic: string) => ({
+        keyphrase: topic,
+        sentences: [],
+        words: topic.split(/\s+/).length,
+        weight: 100,
+      }));
+    }
+
+    const { limitTopSentences = 3 } = options;
+    const allRelevant = await weighRelevanceConceptVectorMultiple(
+      sentencesArray,
+      llmTopics,
+      options,
+    );
+
+    const keyphraseObjects: any[] = [];
+    const topSentences: any[] = [];
+
+    for (const topic of llmTopics) {
+      // @ts-ignore
+      const topicRelevant = allRelevant[topic] || [];
+      const topX = topicRelevant.slice(0, limitTopSentences);
+
+      const topicSentences = topX.map((r: any) => ({
+        text: r.content,
+        similarity: r.similarity,
+      }));
+
+      keyphraseObjects.push({
+        keyphrase: topic,
+        topSentences: topicSentences,
+        words: topic.split(/\s+/).length,
+        weight: 100,
+      });
+
+      for (const r of topX) {
+        const sIdx = sentencesArray.indexOf(r.content);
+        topSentences.push({
+          text: r.content,
+          index: sIdx !== -1 ? sIdx : 0,
+          keyphrases: [{ keyphrase: topic, weight: r.similarity }],
+          weight: r.similarity,
+        });
+      }
+    }
+
+    // Deduplicate topSentences
+    const uniqueSentences = Array.from(
+      new Map(topSentences.map((s: any) => [s.text, s])).values(),
+    );
+
+    return {
+      topSentences: uniqueSentences.sort(
+        (a: any, b: any) => b.weight - a.weight,
+      ),
+      keyphrases: keyphraseObjects,
+      sentences: sentencesArray,
+    } as unknown as SEEKTOPICResult;
+  }
+
+  // \u2500\u2500 FALLBACK: Tokenise + extract n-grams \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const nGrams: NgramMap = {};
+  const sentenceKeysMap: SentenceEntry[] = [];
+
+  for (let idx = 0; idx < sentencesArray.length; idx++) {
+    const sentence = sentencesArray[idx];
+    const tokens = convertTextToTokens(sentence, { phrasesModel });
+
+    sentenceKeysMap.push({
+      text: sentence,
+      index: idx,
+      keyphrases: [],
+      weight: 0,
+    });
+
+    for (let i = 0; i < tokens.length; i++) {
+      for (let n = minWords; n <= maxWords; n++) {
+        extractNounEdgeGrams(n, tokens, i, nGrams, minWordLength, idx);
+      }
+    }
+  }
+
+  // \u2500\u2500 5. Score raw keyphrases \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  let keyphraseObjects: KeyphraseEntry[] = [];
+  for (let n = minWords; n <= maxWords; n++) {
+    if (!nGrams[n]) continue;
+    for (const [keyphrase, sentencesArr] of Object.entries(nGrams[n])) {
+      keyphraseObjects.push({
+        keyphrase,
+        sentences: sentencesArr,
+        words: n,
+        weight: sentencesArr.length * n,
+      });
+    }
+  }
+
+  // \u2500\u2500 6. Fold subphrases + deduplicate \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const folded = foldSubphrases(keyphraseObjects);
+  const seen = new Set<string>();
+  keyphraseObjects = folded
+    .sort((a, b) => b.weight - a.weight)
+    .filter((k) => !seen.has(k.keyphrase) && seen.add(k.keyphrase))
+    .map((k) => ({
+      ...k,
+      sentences: Array.isArray(k.sentences)
+        ? [...new Set(k.sentences as number[])]
+        : k.sentences,
+    }));
+
+  // \u2500\u2500 7. IDF + wiki-entity + heavy-query weighting \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  keyphraseObjects = weightKeyphrasesBySpecificity(
+    keyphraseObjects,
+    phrasesModel,
+    heavyWeightQuery,
+  )
+    .filter((k) => k.keyphrase.length > minKeyPhraseLength)
+    .sort((a, b) => b.weight - a.weight);
+
+  // Fast path: return keyphrases without TextRank
+  if (optionSkipRanking) return keyphraseObjects;
+
+  // \u2500\u2500 8. Attach keyphrases to sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  for (const { keyphrase, sentences, weight } of keyphraseObjects) {
+    if (Array.isArray(sentences)) {
+      for (const si of sentences) {
+        sentenceKeysMap[si as number]?.keyphrases.push({ keyphrase, weight });
+      }
+    }
+  }
+
+  // \u2500\u2500 9. TextRank \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const ranked = rankSentencesCentralToKeyphrase(sentenceKeysMap);
+
+  const topSentences = (ranked ?? [])
+    .sort((a, b) => b.weight - a.weight)
+    .slice(0, limitTopSentences)
+    .map((s) => ({
+      ...s,
+      keyphrases: [...new Set(s.keyphrases.map((k) => k.keyphrase))],
+    }));
+
+  const keyphrases = keyphraseObjects.slice(0, limitTopKeyphrases).map((k) => ({
+    ...k,
+    sentences: Array.isArray(k.sentences) ? k.sentences.join(",") : k.sentences,
+  }));
+
+  return { topSentences, keyphrases, sentences: sentencesArray };
+}