extract-webpage 1.2.5

package/src/seektopic/seektopic-keyphrases.ts

@@ -0,0 +1,279 @@
+/**
+ * @fileoverview Main orchestrator for the SEEKTOPIC keyphrase extraction pipeline.
+ * Coordinates cleaning, segmentation, LLM extraction, and fallback n-gram ranking.
+ */
+import type {
+  NgramMap,
+  KeyphraseEntry,
+  SentenceEntry,
+  SEEKTOPICOptions,
+  SEEKTOPICResult,
+} from "./types";
+import { splitTextToSentences } from "../tokenize/text-to-sentences";
+import { convertTextToTokens } from "../tokenize/text-to-topic-tokens";
+import { extractNounEdgeGrams } from "./ngrams";
+import { foldSubphrases } from "./fold-keyphrases";
+import { weightKeyphrasesBySpecificity } from "./weight-keyphrases";
+import { rankSentencesCentralToKeyphrase } from "./rank-sentences-keyphrases";
+import { weighRelevanceConceptVectorMultiple } from "./vector-search";
+
+/**
+ * ### SEEKTOPIC \u2014 Keyphrase & Sentence Extraction
+ *
+ * Pulls the most important phrases and sentences out of any document.
+ * Given raw text, it returns a ranked list of key concepts (e.g. "neural
+ * network", "climate change") and, optionally, the sentences that best
+ * summarise the document around those concepts.
+ *
+ * <img src="https://i.imgur.com/gZ4kI1V.png" width="360px" />
+ *
+ * **How it works \u2014 8-step pipeline:**
+ *
+ * 1. **Clean** \u2014 strip HTML tags and entities.
+ * 2. **Split** \u2014 break text into sentences, respecting abbreviations and URLs.
+ * 3. **Topic Extraction** *(LLM Path)* \u2014 sends the first 5000 words to an LLM
+ *    to extract the most descriptive key topic phrases labeling the document.
+ * 4. **Vector Search** *(LLM Path)* \u2014 calculates cosine similarity embeddings
+ *    for all sentences vs topic phrases, finding the top relevant sentences per topic.
+ * 5. **Tokenise & Extract** *(Fallback)* \u2014 if LLM fails, identify noun-anchored
+ *    n-grams (1-N words).
+ * 6. **Score & Fold** *(Fallback)* \u2014 weight phrases and merge subsumed phrases.
+ * 7. **Re-weight** *(Fallback)* \u2014 boost Wikipedia entities and rare domain terms.
+ * 8. **TextRank** *(Fallback)* \u2014 build sentence similarity graph and run random walk.
+ *
+ * <video src="https://github.com/user-attachments/assets/73348d63-7671-4e20-8df9-29a13d5b0768"
+ *  width="550px" controls />
+ *
+ * @param docText - Plain text or HTML document to analyse.
+ * @param options - Optional tuning parameters (word limits, thresholds, query bias).
+ * @returns
+ *   - `optionSkipRanking: true` (default) \u2192 `KeyphraseEntry[]` sorted by weight.
+ *   - `optionSkipRanking: false` \u2192 `SEEKTOPICResult` with `topSentences`,
+ *     `keyphrases`, and the full `sentences` array.
+ *
+ * @example
+ * // Fast: just extract keyphrases
+ * const keyphrases = extractSEEKTOPIC(articleText, { phrasesModel });
+ * // \u2192 [{ keyphrase: "machine learning", weight: 84 }, ...]
+ *
+ * @example
+ * // Full: keyphrases + summary sentences, biased toward a search query
+ * const { topSentences, keyphrases } = extractSEEKTOPIC(articleText, {
+ *   phrasesModel,
+ *   optionSkipRanking: false,
+ *   heavyWeightQuery: "transformer attention",
+ *   limitTopSentences: 5,
+ * }) as SEEKTOPICResult;
+ *
+ * @author [ai-research-agent (2024)](https://airesearch.js.org)
+ * @category Topics
+ */
+export async function extractSEEKTOPIC(
+  docText: string,
+  options: SEEKTOPICOptions = {},
+): Promise<KeyphraseEntry[] | SEEKTOPICResult> {
+  if (typeof docText !== "string") throw new Error("docText must be a string");
+
+  const {
+    phrasesModel,
+    maxWords = 2,
+    minWords = 1,
+    minWordLength = 3,
+    topKeyphrasesPercent = 0.5,
+    limitTopSentences = 5,
+    limitTopKeyphrases = 10,
+    minKeyPhraseLength = 5,
+    heavyWeightQuery = "",
+    removeHTML = true,
+    optionSkipRanking = true,
+    getEnv = () => "",
+  } = options;
+
+  // \u2500\u2500 1. Normalize HTML \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  let text = docText
+    .replace(/</g, " <")
+    .replace(/>/g, "> ")
+    .replace(/&.{2,5};/g, ""); // strip &quot; &amp; &lt; &gt; &nbsp;
+  if (removeHTML) text = text.replace(/<[^>]*>/g, "");
+
+  // \u2500\u2500 2. Sentence segmentation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const sentencesArray = splitTextToSentences(text);
+
+  // \u2500\u2500 3. LLM Topic Extraction (Starting Step) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const first5kWords = text.split(/\s+/).slice(0, 5000).join(" ");
+  const llmPrompt = `Extract the 5-10 most important key topic phrases that best 
+  label this document. Return exclusively a comma-separated list of the key topic 
+  phrases. Document text: ${first5kWords}`;
+
+  let llmTopics: string[] = [];
+  try {
+    // `generate-language` moved into the chat-agent-toolkit workspace package.
+    // Load it lazily (and tolerate absence) since the LLM path is an optional
+    // enhancement — the n-gram fallback below covers extraction without it.
+    const { writeLanguageResponse } = await import("chat-agent-toolkit");
+    const aiResponse = await writeLanguageResponse({
+      query: llmPrompt,
+      provider: getEnv("LLM_PROVIDER") || "openai",
+      apiKey: getEnv("LLM_API_KEY") || getEnv("OPENAI_API_KEY") || "dummy",
+    });
+    if (aiResponse.content && !aiResponse.error) {
+      llmTopics = aiResponse.content
+        .split(",")
+        .map((s: string) => s.trim().replace(/^['"]|['"]$/g, ""))
+        .filter(Boolean);
+    }
+  } catch (e) {
+    console.error("LLM topic extraction failed", e);
+  }
+
+  // \u2500\u2500 4. Use vector-search.ts for top topic sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  if (llmTopics.length > 0) {
+    if (optionSkipRanking) {
+      return llmTopics.map((topic: string) => ({
+        keyphrase: topic,
+        sentences: [],
+        words: topic.split(/\s+/).length,
+        weight: 100,
+      }));
+    }
+
+    const { limitTopSentences = 3 } = options;
+    const allRelevant = await weighRelevanceConceptVectorMultiple(
+      sentencesArray,
+      llmTopics,
+      options,
+    );
+
+    const keyphraseObjects: any[] = [];
+    const topSentences: any[] = [];
+
+    for (const topic of llmTopics) {
+      // @ts-ignore
+      const topicRelevant = allRelevant[topic] || [];
+      const topX = topicRelevant.slice(0, limitTopSentences);
+
+      const topicSentences = topX.map((r: any) => ({
+        text: r.content,
+        similarity: r.similarity,
+      }));
+
+      keyphraseObjects.push({
+        keyphrase: topic,
+        topSentences: topicSentences,
+        words: topic.split(/\s+/).length,
+        weight: 100,
+      });
+
+      for (const r of topX) {
+        const sIdx = sentencesArray.indexOf(r.content);
+        topSentences.push({
+          text: r.content,
+          index: sIdx !== -1 ? sIdx : 0,
+          keyphrases: [{ keyphrase: topic, weight: r.similarity }],
+          weight: r.similarity,
+        });
+      }
+    }
+
+    // Deduplicate topSentences
+    const uniqueSentences = Array.from(
+      new Map(topSentences.map((s: any) => [s.text, s])).values(),
+    );
+
+    return {
+      topSentences: uniqueSentences.sort(
+        (a: any, b: any) => b.weight - a.weight,
+      ),
+      keyphrases: keyphraseObjects,
+      sentences: sentencesArray,
+    } as unknown as SEEKTOPICResult;
+  }
+
+  // \u2500\u2500 FALLBACK: Tokenise + extract n-grams \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const nGrams: NgramMap = {};
+  const sentenceKeysMap: SentenceEntry[] = [];
+
+  for (let idx = 0; idx < sentencesArray.length; idx++) {
+    const sentence = sentencesArray[idx];
+    const tokens = convertTextToTokens(sentence, { phrasesModel });
+
+    sentenceKeysMap.push({
+      text: sentence,
+      index: idx,
+      keyphrases: [],
+      weight: 0,
+    });
+
+    for (let i = 0; i < tokens.length; i++) {
+      for (let n = minWords; n <= maxWords; n++) {
+        extractNounEdgeGrams(n, tokens, i, nGrams, minWordLength, idx);
+      }
+    }
+  }
+
+  // \u2500\u2500 5. Score raw keyphrases \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  let keyphraseObjects: KeyphraseEntry[] = [];
+  for (let n = minWords; n <= maxWords; n++) {
+    if (!nGrams[n]) continue;
+    for (const [keyphrase, sentencesArr] of Object.entries(nGrams[n])) {
+      keyphraseObjects.push({
+        keyphrase,
+        sentences: sentencesArr,
+        words: n,
+        weight: sentencesArr.length * n,
+      });
+    }
+  }
+
+  // \u2500\u2500 6. Fold subphrases + deduplicate \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const folded = foldSubphrases(keyphraseObjects);
+  const seen = new Set<string>();
+  keyphraseObjects = folded
+    .sort((a, b) => b.weight - a.weight)
+    .filter((k) => !seen.has(k.keyphrase) && seen.add(k.keyphrase))
+    .map((k) => ({
+      ...k,
+      sentences: Array.isArray(k.sentences)
+        ? [...new Set(k.sentences as number[])]
+        : k.sentences,
+    }));
+
+  // \u2500\u2500 7. IDF + wiki-entity + heavy-query weighting \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  keyphraseObjects = weightKeyphrasesBySpecificity(
+    keyphraseObjects,
+    phrasesModel,
+    heavyWeightQuery,
+  )
+    .filter((k) => k.keyphrase.length > minKeyPhraseLength)
+    .sort((a, b) => b.weight - a.weight);
+
+  // Fast path: return keyphrases without TextRank
+  if (optionSkipRanking) return keyphraseObjects;
+
+  // \u2500\u2500 8. Attach keyphrases to sentences \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  for (const { keyphrase, sentences, weight } of keyphraseObjects) {
+    if (Array.isArray(sentences)) {
+      for (const si of sentences) {
+        sentenceKeysMap[si as number]?.keyphrases.push({ keyphrase, weight });
+      }
+    }
+  }
+
+  // \u2500\u2500 9. TextRank \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
+  const ranked = rankSentencesCentralToKeyphrase(sentenceKeysMap);
+
+  const topSentences = (ranked ?? [])
+    .sort((a, b) => b.weight - a.weight)
+    .slice(0, limitTopSentences)
+    .map((s) => ({
+      ...s,
+      keyphrases: [...new Set(s.keyphrases.map((k) => k.keyphrase))],
+    }));
+
+  const keyphrases = keyphraseObjects.slice(0, limitTopKeyphrases).map((k) => ({
+    ...k,
+    sentences: Array.isArray(k.sentences) ? k.sentences.join(",") : k.sentences,
+  }));
+
+  return { topSentences, keyphrases, sentences: sentencesArray };
+}

package/src/seektopic/types.ts

@@ -0,0 +1,92 @@
+/**
+ * @fileoverview Type definitions for the SEEKTOPIC engine and ranking utilities.
+ */
+import type {
+  TopicToken,
+  PhrasesModel,
+} from "../tokenize/text-to-topic-tokens";
+
+export type { TopicToken, PhrasesModel };
+
+/** Map from n-gram size \u2192 { phrase text \u2192 sentence indices } */
+export type NgramMap = Record<number, Record<string, number[]>>;
+
+/**
+ * A ranked keyphrase with its scoring metadata and the sentence indices it appears in.
+ */
+export interface KeyphraseEntry {
+  /** The keyphrase text */
+  keyphrase: string;
+  /** Indices of sentences containing this keyphrase (or comma-separated string form) */
+  sentences: number[] | string;
+  /** Sentences representing this keyphrase directly with their relevance similarity */
+  topSentences?: Array<{ text: string; similarity: number }>;
+  /** Number of words in the keyphrase */
+  words: number;
+  /** Composite ranking weight */
+  weight: number;
+  /** True if the phrase is a Wikipedia-linked entity */
+  wiki?: boolean;
+}
+
+/**
+ * A sentence node used during graph construction and TextRank scoring.
+ */
+export interface SentenceEntry {
+  text: string;
+  index: number;
+  keyphrases: Array<{ keyphrase: string; weight: number }>;
+  weight: number;
+}
+
+/**
+ * A sentence in the final SEEKTOPIC output \u2014 keyphrases are resolved to strings.
+ */
+export interface SentenceResult {
+  text: string;
+  index: number;
+  keyphrases: string[];
+  weight: number;
+}
+
+/** Options for {@link extractSEEKTOPIC} */
+export interface SEEKTOPICOptions {
+  /** Trie phrases model for wiki-phrase tokenization */
+  phrasesModel?: PhrasesModel;
+  /** Maximum words per keyphrase (default 2) */
+  maxWords?: number;
+  /** Minimum words per keyphrase (default 1) */
+  minWords?: number;
+  /** Minimum character length of any word in a keyphrase (default 3) */
+  minWordLength?: number;
+  /** Fraction of all keyphrases to use as the TextRank graph limit (default 0.5) */
+  topKeyphrasesPercent?: number;
+  /** Max sentences to return in full-ranking mode (default 5) */
+  limitTopSentences?: number;
+  /** Max keyphrases to return (default 10) */
+  limitTopKeyphrases?: number;
+  /** Minimum character length of the full keyphrase string (default 5) */
+  minKeyPhraseLength?: number;
+  /** Query string to bias keyphrase weights toward (default "") */
+  heavyWeightQuery?: string;
+  /** Strip HTML tags before processing (default true) */
+  removeHTML?: boolean;
+  /** Return only keyphrases without running TextRank (default true) */
+  optionSkipRanking?: boolean;
+  getEnv?: (key: string) => string | undefined;
+}
+
+/** Full SEEKTOPIC output when `optionSkipRanking` is false */
+export interface SEEKTOPICResult {
+  topSentences: SentenceResult[];
+  keyphrases: Array<Omit<KeyphraseEntry, "sentences"> & { sentences: string }>;
+  sentences: string[];
+}
+
+/** Options for {@link rankSentencesCentralToKeyphrase} */
+export interface RankOptions {
+  /** Number of random-walk steps (default 1000) */
+  iterations?: number;
+  /** Steps between forced random resets to avoid cluster traps (default 100) */
+  resetInterval?: number;
+}

package/src/seektopic/vector-search.ts

@@ -0,0 +1,232 @@
+import grab from "../utils/grab";
+
+/**
+ * Text embeddings convert words or phrases into numerical vectors in a high-dimensional
+ * space, where each dimension represents a semantic feature extracted by a model like
+ * MiniLM-L6-v2. In this concept space, words with similar meanings have vectors that
+ * are close together, allowing for quantitative comparisons of semantic similarity.
+ * These vector representations enable powerful applications in natural language processing,
+ * including semantic search, text classification, and clustering, by leveraging the
+ * geometric properties of the embedding space to capture and analyze the relationships
+ * between words and concepts.
+ * [Text Embeddings, Classification, and Semantic Search
+ *  (Youtube)](https://www.youtube.com/watch?v=sNa_uiqSlJo&t=129s)
+ *
+ * <img src="https://i.imgur.com/wtJqEqX.png" width="350" />
+ * @param {string} text - The text to embed.
+ * @param {Object} [options]
+ * @param {AutoTokenizer} options.pipeline
+ *  - The pipeline to use for embedding.
+ * @param {number} options.precision default=4 - The number of decimal places to round to.
+ * @returns {Promise<{embeddingsDict: Object.<string, number[]>, embedding: number[]}>}
+ * @category Similarity
+ */
+export async function convertTextToEmbedding(
+  text: string | string[],
+  options: any = {},
+) {
+  var { precision = 4, pipeline } = options;
+
+  if (!pipeline) pipeline = await getEmbeddingModel();
+
+  const embedding = await pipeline(text, { pooling: "mean", normalize: true });
+
+  // Check if input was an array to determine output format
+  if (Array.isArray(text)) {
+    return embedding
+      .tolist()
+      .map((vec) => vec.map((num) => parseFloat(num.toFixed(precision))));
+  } else {
+    const roundedEmbedding = Array.from((embedding as any).data).map(
+      (num: any) => parseFloat((num as any).toFixed(precision)),
+    );
+    return roundedEmbedding;
+  }
+}
+
+/**
+ * Calculate the semantic similarity between one text and a list of
+ * other sentences by comparing their embeddings.
+ * https://huggingface.co/docs/api-inference/detailed_parameters#sentence-similarity-task
+ *
+ * <img src="https://i.imgur.com/ex2UWnu.png" width="350px" />
+ * @param {string} source_sentence The string that you wish to
+ * compare the other strings with. This can be a phrase, sentence,
+ * or longer passage, depending on the model being used.
+ * @param {Array<string>} sentences A list of strings which will be compared
+ * against the source_sentence.
+ * @param {Object} [options]
+ * @param {string} options.model default="sentence-transformers/all-MiniLM-L6-v2"
+ * @param {string} options.HF_API_KEY Required https://huggingface.co/settings/tokens
+ * @returns array of 0-1 similarity scores for each sentence
+ * @category Similarity
+ */
+export async function weighRelevanceConceptVectorAPI(
+  source_sentence,
+  sentences,
+  options = {},
+) {
+  var { model = "sentence-transformers/all-MiniLM-L6-v2", HF_API_KEY = "" } =
+    options as any;
+
+  if (!HF_API_KEY) return { error: "No API key" };
+
+  const url = `https://api-inference.huggingface.co/models/${model}`;
+  try {
+    return await grab(url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${HF_API_KEY}`,
+      },
+      body: JSON.stringify({
+        inputs: {
+          source_sentence,
+          sentences,
+        },
+      }),
+    });
+  } catch (error) {
+    console.error("API request failed:", error);
+    return null;
+  }
+}
+
+/**
+ * Initialize HuggingFace Transformers pipeline for embedding text.
+ *
+ * <img src="https://i.imgur.com/3R5Tsrf.png" width="350px" />
+ * @param {Object} [options]
+ * @param {string} options.pipelineName default "feature-extraction",
+ * @param {string} options.modelName default="Xenova/all-MiniLM-L6-v2" -
+ * The name of the model to use
+ * @returns {Promise<import("@huggingface/transformers").AutoTokenizer>} The pipeline.
+ * @category Similarity
+ */
+export async function getEmbeddingModel(options: any = {}) {
+  const { pipeline } = await (import("@huggingface/transformers") as Promise<any>);
+  const {
+    pipelineName = "feature-extraction",
+    modelName = "Xenova/all-MiniLM-L6-v2",
+    quantized = true,
+    gpu = false,
+  } = options;
+
+  return await pipeline(pipelineName, modelName, {
+    quantized,
+    dtype: "fp32",
+    device: gpu ? "webgpu" : "cpu",
+  } as any);
+}
+
+/**
+ * [Cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) gets similarity of two
+ * vectors by whether they have the same direction (similar) or are poles apart. Cosine similarity
+ * is often used with text representations to compare how similar two documents or sentences
+ * are to each other. The output of cosine similarity ranges from -1 to 1, where -1 means the
+ * two vectors are completely dissimilar, and 1 indicates maximum similarity.
+ * @param {Array<number>} vectorA
+ * @param {Array<number>} vectorB
+ * @returns {number} -1 to 1 similarity score
+ */
+export function calculateCosineSimilarity(vectorA, vectorB) {
+  return (
+    vectorA.reduce((sum, a, i) => sum + a * vectorB[i], 0) /
+    (Math.sqrt(vectorA.reduce((sum, a) => sum + a * a, 0)) *
+      Math.sqrt(vectorB.reduce((sum, b) => sum + b * b, 0)))
+  );
+}
+
+///OLDER ======================
+/**
+ * Rerank documents's chunks based on relevance to query,
+ * based on cosine similarity of their concept vectors generated
+ * by a 20MB MiniLM transformer model downloaded locally.
+ *
+ * [A Complete Overview of Word Embeddings](https://www.youtube.com/watch?v=5MaWmXwxFNQ&t=323s)
+ * @param {Array<string>} documents
+ * @param {string} query
+ * @param {Object} [options]
+ * @returns {Promise<Array<{content: string, similarity: number}>>}
+ * @category Similarity
+ */
+export async function weighRelevanceConceptVector(
+  documents,
+  query,
+  options = {},
+) {
+  const docEmbeddings = await convertTextToEmbedding(documents, options);
+
+  const queryEmbedding = await convertTextToEmbedding(query, options);
+
+  let sortedDocs = docEmbeddings
+    .map((docEmbedding, i) => ({
+      index: i,
+      similarity: calculateCosineSimilarity(queryEmbedding, docEmbedding),
+    }))
+    .sort((a, b) => b.similarity - a.similarity)
+    .map(({ index, similarity }) => ({
+      content: documents[index],
+      similarity,
+    }));
+
+  return sortedDocs;
+}
+
+/**
+ * Rerank documents's chunks based on relevance to multiple queries,
+ * optimizing by embedding documents only once.
+ *
+ * @param {Array<string>} documents
+ * @param {Array<string>} queries
+ * @param {Object} [options]
+ * @returns {Promise<Object<string, Array<{content: string, similarity: number}>>>}
+ * @category Similarity
+ */
+export async function weighRelevanceConceptVectorMultiple(
+  documents,
+  queries,
+  options = {},
+) {
+  if (!documents || documents.length === 0 || !queries || queries.length === 0)
+    return {};
+
+  const pipeline = (options as any).pipeline;
+  const embedder = pipeline || (await getEmbeddingModel(options));
+
+  // Embed documents once
+  const docEmbeddings = await convertTextToEmbedding(documents, {
+    ...options,
+    pipeline: embedder,
+  });
+  // Embed all queries at once
+  const queryEmbeddings = await convertTextToEmbedding(queries, {
+    ...options,
+    pipeline: embedder,
+  });
+
+  // Ensure queryEmbeddings is a 2D array even if queries was a single string wrapped in an array
+  const qEmbeds = Array.isArray(queryEmbeddings[0])
+    ? queryEmbeddings
+    : [queryEmbeddings];
+
+  const resultsByQuery = {};
+
+  queries.forEach((query, qIdx) => {
+    const queryEmbedding = qEmbeds[qIdx];
+
+    let sortedDocs = docEmbeddings
+      .map((docEmbedding, i) => ({
+        index: i,
+        similarity: calculateCosineSimilarity(queryEmbedding, docEmbedding),
+      }))
+      .sort((a, b) => b.similarity - a.similarity)
+      .map(({ index, similarity }) => ({
+        content: documents[index],
+        similarity,
+      }));
+
+    resultsByQuery[query] = sortedDocs;
+  });
+
+  return resultsByQuery;
+}

package/src/seektopic/weight-keyphrases.ts

@@ -0,0 +1,59 @@
+/**
+ * @fileoverview Scoring logic for keyphrases based on Wiki-entities, specificity, and query bias.
+ * Implements the ranking refinement steps of the SEEKTOPIC pipeline.
+ */
+import type { KeyphraseEntry, PhrasesModel } from "./types";
+import { convertTextToTokens } from "../tokenize/text-to-topic-tokens";
+
+/**
+ * Applies two weighting passes to a keyphrase list:
+ *
+ * 1. **Wiki-entity bonus** \u2014 if any token in the phrase is a Wikipedia-linked
+ *    entity (POS tag 5), the keyphrase weight is doubled and `wiki` is set.
+ * 2. **IDF domain-specificity** \u2014 weight is multiplied by the average POS
+ *    uniqueness score across the phrase's tokens. Rare, domain-specific terms
+ *    (high uniqueness) receive a larger multiplier than common nouns.
+ * 3. **Heavy-query bias** (optional) \u2014 if `heavyWeightQuery` is set, keyphrases
+ *    that closely match the query words receive a large additive bonus, allowing
+ *    dynamic re-ranking when a user clicks a term or arrives via a search query.
+ *
+ * @param keyphrases - Keyphrases to weight; mutated in-place for efficiency.
+ * @param phrasesModel - Trie model passed to the tokenizer (may be undefined).
+ * @param heavyWeightQuery - Space-separated query string to bias ranking toward.
+ * @returns The same array with updated `weight` and `wiki` fields.
+ *
+ * @example
+ * const weighted = weightKeyphrasesBySpecificity(keyphrases, model, "self attention");
+ */
+export function weightKeyphrasesBySpecificity(
+  keyphrases: KeyphraseEntry[],
+  phrasesModel: PhrasesModel | undefined,
+  heavyWeightQuery: string,
+): KeyphraseEntry[] {
+  const querySplit = heavyWeightQuery ? heavyWeightQuery.split(" ") : [];
+
+  for (const kp of keyphrases) {
+    const tokens = convertTextToTokens(kp.keyphrase, { phrasesModel });
+
+    // Wiki-entity bonus: any token with category 5 doubles the weight
+    if (tokens.some(t => t[1] === 5)) {
+      kp.wiki = true;
+      kp.weight *= 2;
+    }
+
+    // IDF domain-specificity: scale by average token uniqueness score (index [1])
+    // Tokens without a score default to 4 (mid-range common noun)
+    const idfAvg = tokens.reduce((sum, t) => sum + (t[1] ?? 4), 0) / tokens.length;
+    kp.weight = Math.floor(kp.weight * idfAvg);
+
+    // Heavy-query bias: boost phrases that closely match the query
+    if (querySplit.length) {
+      const diffWords = querySplit.filter(w => !kp.keyphrase.includes(w)).length;
+      if (diffWords < 4 && diffWords < querySplit.length - 1) {
+        kp.weight += 4000 - diffWords * 1000;
+      }
+    }
+  }
+
+  return keyphrases;
+}