extract-webpage 1.2.5

package/src/tokenize/text-to-topic-tokens.ts

@@ -0,0 +1,175 @@
+/**
+ * @fileoverview Transformer for converting raw text queries into topic-weighted phrase tokens.
+ * Handles phrase detection, typo correction, and root word stemming.
+ */
+import { stemWordToRoot } from "./word-to-root-stem";
+import { isWordCommonIgnored } from "./word-is-ignored";
+
+/**
+ * Topic token tuple shape used by downstream ranking logic:
+ * [term, termCategory, uniqueness, metadata]
+ */
+export type TopicToken = [string, number, number, string];
+
+/**
+ * Trie structure for phrase completion lookup keyed by first two letters, then full token.
+ */
+type PhraseEntry = [string | null, number, number];
+
+export type PhrasesModel = Record<string, Record<string, PhraseEntry[]>>;
+
+export interface ConvertTextToTokensOptions {
+  phrasesModel: PhrasesModel;
+  typosModel?: Record<string, string>;
+  checkTypos?: 0 | 1;
+  ignoreStopWords?: 0 | 1;
+  checkRootWords?: 0 | 1;
+}
+/**
+ * @typedef {Object} Token
+ * @property {number} termCategory - The category of the term
+ * @property {number} uniqueness - The uniqueness score of the term
+ * @property {string} term - The actual term or phrase
+ */
+/**
+ * ### Convert Text Query to Topic Phrase Tokens
+ * <img width="350px"  src="https://i.imgur.com/NDrmSRQ.png" />
+ *
+ * Returns a list of phrases that are found in Wiki Titles/ dictionary phrases World Model
+ * that match the input phrase, or just the single word if found. Search results will be
+ *  more accurate if we infer likely phrases and search for those words occuring together and
+ *  not just split into words and find frequency. Examples are "white house" or "state of the art"
+ *  which should be searched as a phrase but would return different context if split into words.
+ *  As Led Zeppelin famously put it: \u266b "'Cause you know sometimes words have two meanings."
+ *
+ * @param {string} phrase
+ * @param {Object} [options]
+ * @param {Object} options.phrasesModel - remote model
+ * @param {Object} options.typosModel - remote model
+ * @param {number} options.checkTypos - check for typos
+ * @param {number} options.ignoreStopWords - ignore 300+ overused words
+ * @param {number} options.checkRootWords - check for word's root stem
+ * @returns {Array<{termCategory: number, uniqueness: number, term: string}>}
+ * @example
+ *   const result = convertTextToTokens("The president of the united states is in the white house", { phrasesModel, typosModel });
+ *   console.log(result);
+ *
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @category Topics
+ */
+export function convertTextToTokens(
+  phrase: string,
+  options: Partial<ConvertTextToTokensOptions> = {},
+): TopicToken[] {
+  let {
+    phrasesModel, //pass in remote model
+    typosModel,
+    checkTypos = 0,
+    checkRootWords = 1,
+    ignoreStopWords = 1,
+  } = options;
+
+  if (!phrasesModel) throw new Error("Missing phrasesModel ");
+
+  if (!phrase) throw new Error("Missing  phrase");
+
+  //strip non-alphanumeric characters from query an keep -'/
+  phrase = phrase.replace(/[^a-zA-Z0-9\s\-\'\/]/g, " ");
+
+  //split into words
+  var words = phrase.toLowerCase().split(/\W+/);
+
+  //check for typos
+
+  if (checkTypos && typosModel)
+    words = words.map((word) => typosModel[word] || word);
+
+  const topics: TopicToken[] = [];
+  for (var i = 0; i < words.length; i++) {
+    var word = words[i];
+
+    //ignore 300+ common stop words
+    if (ignoreStopWords && isWordCommonIgnored(word)) {
+      //todo include as ignored
+      topics.push([word, 0, 0, ""]);
+      continue;
+    }
+
+    //Find next word phrase completion list
+    var firstTwoLetters = word.slice(0, 2);
+    var possiblePhrases = phrasesModel[firstTwoLetters]
+      ? phrasesModel[firstTwoLetters][word]
+      : null;
+
+    //check for root words like "gaming" -> "game"
+    if (!possiblePhrases && checkRootWords) {
+      var rootWord = stemWordToRoot(word);
+      if (rootWord !== word)
+        possiblePhrases = phrasesModel[rootWord.slice(0, 2)]
+          ? phrasesModel[rootWord.slice(0, 2)][rootWord]
+          : null;
+
+      //TODO add label "root word"
+    }
+
+    //if word still not in dict, add it as a single word
+    if (!possiblePhrases) topics.push([word, 0, 0, ""]);
+
+    if (possiblePhrases) {
+      var maxPhraseLength = 1;
+      let singleWordObj: TopicToken | null = null;
+      var isPhraseFound = false;
+
+      //calculate max possible length of phrase of next words
+      for (var p of possiblePhrases)
+        if (p[0]?.length > maxPhraseLength) maxPhraseLength = p[0].length;
+
+      //grab that length of text from next words
+      var nextWords = "";
+      for (var j = 1; j < words.length - i; j++) {
+        nextWords += (words[i + j] || "") + " ";
+
+        if (nextWords.length >= maxPhraseLength) break;
+      }
+
+      for (const phraseEntry of possiblePhrases) {
+        //if no next phrase, preserve the single word
+        //it culd also be not in the dict first word
+        if (!phraseEntry[0] && !singleWordObj) {
+          const nextPhrase = phraseEntry.slice(1, 3) as [number, number];
+          singleWordObj = [word, nextPhrase[0], nextPhrase[1], ""];
+        } else {
+          //add next word to the phrase up to maxPhraseLength
+          const nextPhrasePart = phraseEntry[0];
+          if (
+            !isPhraseFound &&
+            typeof nextPhrasePart === "string" &&
+            nextWords.startsWith(nextPhrasePart)
+          ) {
+            var fullPhrase = word + " " + nextPhrasePart;
+            const nextPhrase = phraseEntry.slice(1, 3) as [number, number];
+            topics.push([fullPhrase, nextPhrase[0], nextPhrase[1], ""]); // remove first which is next words
+
+            //skip looping thru the next words added to phrase
+            i += nextPhrasePart.split(" ").length; //TODO fi
+
+            //suppress single-word "red" if "red wine" is found
+            isPhraseFound = true;
+
+            break;
+          }
+        }
+      }
+
+      //if no phrases then add the single word
+      if (!isPhraseFound) {
+        if (singleWordObj) {
+          // Could be starter token for phrases; keep it as standalone suggestion.
+          topics.push(singleWordObj);
+        }
+      }
+    }
+  }
+
+  return topics.filter(Boolean);
+}

package/src/tokenize/word-is-ignored.ts

@@ -0,0 +1,53 @@
+/**
+ * @fileoverview Utility for identifying common stop words (e.g., "the", "and", "is").
+ * Based on the SpaCy English stop word list.
+ */
+
+/**
+ * Checks word is in [320 commonly ignored "stop words 
+ * "](https://raw.githubusercontent.com/igorbrigadir/stopwords/master/en/spacy.txt) 
+ * in queries, using efficient JS Set method 
+ * @param {string} word
+ * @returns {Boolean}
+*/
+export function isWordCommonIgnored(word) {
+  return stopwords.has(word.toLowerCase());
+}
+
+//@ts-ignore
+const stopwords = new Set([
+  "a", "about", "above", "across", "after", "afterwards", "again", "against", "all",
+  "almost", "alone", "along", "already", "also", "although", "always", "am", "among",
+  "amongst", "amount", "an", "and", "another", "any", "anyhow", "anyone", "anything",
+  "anyway", "anywhere", "are", "around", "as", "at", "back", "be", "became", "because",
+  "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
+  "below", "beside", "besides", "between", "beyond", "both", "bottom", "but", "by",
+  "ca", "call", "can", "cannot", "could", "did", "do", "does", "doing", "done", "down",
+  "due", "during", "each", "eight", "either", "eleven", "else", "elsewhere", "empty",
+  "enough", "even", "ever", "every", "everyone", "everything", "everywhere", "except",
+  "few", "fifteen", "fifty", "first", "five", "for", "former", "formerly", "forty",
+  "four", "from", "front", "full", "further", "get", "give", "go", "had", "has",
+  "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon",
+  "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "i", "if",
+  "in", "indeed", "into", "is", "it", "its", "itself", "just", "keep", "last", "latter",
+  "latterly", "least", "less", "made", "make", "many", "may", "me", "meanwhile",
+  "might", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my",
+  "myself", "n't", "name", "namely", "neither", "never", "nevertheless", "next", "nine",
+  "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of",
+  "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others",
+  "otherwise", "our", "ours", "ourselves", "out", "over", "own", "part", "per",
+  "perhaps", "please", "put", "quite", "rather", "re", "really", "regarding", "same",
+  "say", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she",
+  "should", "show", "side", "since", "six", "sixty", "so", "some", "somehow",
+  "someone", "something", "sometime", "sometimes", "somewhere", "still", "such",
+  "take", "ten", "than", "that", "the", "their", "them", "themselves", "then",
+  "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon",
+  "these", "they", "third", "this", "those", "though", "three", "through", "throughout",
+  "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty",
+  "two", "under", "unless", "until", "up", "upon", "us", "used", "using", "various",
+  "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence",
+  "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon",
+  "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole",
+  "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you",
+  "your", "yours", "yourself", "yourselves", "'d", "'ll", "'m", "'re", "'s", "'ve"
+])

package/src/tokenize/word-to-root-stem.ts

@@ -0,0 +1,151 @@
+/**
+ * @fileoverview Implementation of the Porter Stemmer algorithm for word normalization.
+ * Used to reduce words to their root form (e.g., "running" to "run").
+ */
+/**
+ * Stems a word using the [Porter
+ * Stemmer](https://snowballstem.org/algorithms/porter/stemmer.html)
+ * for removing inflectional endings like "ing", "ist", "ize".
+ *
+ * @author [Porter, M. (1980)](https://tartarus.org/martin/PorterStemmer/)
+ * @param word - The word to be stemmed
+ * @returns The stemmed word
+ * @example const rootWord = stemWordToRoot("running"); // returns "run"
+ * @category Topics
+ */
+export function stemWordToRoot(word: string): string {
+  // Return short words (less than 3 characters) without stemming
+  if (word.length < 3) return word;
+
+  const SUFFIX_MAPS = {
+    step2: {
+      ational: "ate",
+      tional: "tion",
+      enci: "ence",
+      anci: "ance",
+      izer: "ize",
+      bli: "ble",
+      alli: "al",
+      entli: "ent",
+      eli: "e",
+      ousli: "ous",
+      ization: "ize",
+      ation: "ate",
+      ator: "ate",
+      alism: "al",
+      iveness: "ive",
+      fulness: "ful",
+      ousness: "ous",
+      aliti: "al",
+      iviti: "ive",
+      biliti: "ble",
+      logi: "log",
+    },
+    step3: {
+      icate: "ic",
+      ative: "",
+      alize: "al",
+      iciti: "ic",
+      ical: "ic",
+      ful: "",
+      ness: "",
+    },
+  };
+
+  const REGEX = {
+    consonant: /[^aeiou]/,
+    vowel: /[aeiouy]/,
+    consonants: /([^aeiou][^aeiouy]*)/,
+    vowels: /([aeiouy][aeiou]*)/,
+    gt0: /^([^aeiou][^aeiouy]*)?([aeiouy][aeiou]*)([^aeiou][^aeiouy]*)/,
+    eq1: /^([^aeiou][^aeiouy]*)?([aeiouy][aeiou]*)([^aeiou][^aeiouy]*)([aeiouy][aeiou]*)?$/,
+    gt1: /^([^aeiou][^aeiouy]*)?([aeiouy][aeiou]*[^aeiou][^aeiouy]*){2,}/,
+    vowelInStem: /^([^aeiou][^aeiouy]*)?[aeiouy]/,
+    consonantLike: /^([^aeiou][^aeiouy]*)[aeiouy][^aeiouwxy]$/,
+    sfxLl: /ll$/,
+    sfxE: /^(.+?)e$/,
+    sfxY: /^(.+?)y$/,
+    sfxIon: /^(.+?(s|t))(ion)$/,
+    sfxEdOrIng: /^(.+?)(ed|ing)$/,
+    sfxAtOrBlOrIz: /(at|bl|iz)$/,
+    sfxEED: /^(.+?)eed$/,
+    sfxS: /^.+?[^s]s$/,
+    sfxSsesOrIes: /^.+?(ss|i)es$/,
+    sfxMultiConsonantLike: /([^aeiouylsz])\1$/,
+    step2:
+      /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,
+    step3: /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,
+    step4:
+      /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,
+  };
+
+  let stem = word.toLowerCase();
+
+  // Special handling for words starting with 'y'
+  const firstCharWasY = stem[0] === "y";
+  if (firstCharWasY) stem = "Y" + stem.slice(1);
+
+  // Step 1a: Handle plural forms and -ed or -ing suffixes
+  stem = stem
+    .replace(REGEX.sfxSsesOrIes, "$1")
+    .replace(REGEX.sfxS, (s) => s.slice(0, -1));
+
+  // Step 1b: Handle -eed, -ed, -ing suffixes
+  let match;
+  if ((match = REGEX.sfxEED.exec(stem))) {
+    if (REGEX.gt0.test(match[1])) stem = stem.slice(0, -1);
+  } else if (
+    (match = REGEX.sfxEdOrIng.exec(stem)) &&
+    REGEX.vowelInStem.test(match[1])
+  ) {
+    stem = match[1];
+    if (REGEX.sfxAtOrBlOrIz.test(stem)) {
+      stem += "e";
+    } else if (REGEX.sfxMultiConsonantLike.test(stem)) {
+      stem = stem.slice(0, -1);
+    } else if (REGEX.consonantLike.test(stem)) {
+      stem += "e";
+    }
+  }
+
+  // Step 1c: Replace suffix y or Y by i if preceded by a non-vowel
+  if ((match = REGEX.sfxY.exec(stem)) && REGEX.vowelInStem.test(match[1])) {
+    stem = match[1] + "i";
+  }
+
+  // Step 2: Handle various suffixes
+  if ((match = REGEX.step2.exec(stem)) && REGEX.gt0.test(match[1])) {
+    stem = match[1] + SUFFIX_MAPS.step2[match[2]];
+  }
+
+  // Step 3: Handle more suffixes
+  if ((match = REGEX.step3.exec(stem)) && REGEX.gt0.test(match[1])) {
+    stem = match[1] + SUFFIX_MAPS.step3[match[2]];
+  }
+
+  // Step 4: Handle even more suffixes
+  if ((match = REGEX.step4.exec(stem))) {
+    if (REGEX.gt1.test(match[1])) stem = match[1];
+  } else if ((match = REGEX.sfxIon.exec(stem)) && REGEX.gt1.test(match[1])) {
+    stem = match[1];
+  }
+
+  // Step 5a: Remove final e
+  if (
+    (match = REGEX.sfxE.exec(stem)) &&
+    (REGEX.gt1.test(match[1]) ||
+      (REGEX.eq1.test(match[1]) && !REGEX.consonantLike.test(match[1])))
+  ) {
+    stem = match[1];
+  }
+
+  // Step 5b: Remove final ll
+  if (REGEX.sfxLl.test(stem) && REGEX.gt1.test(stem)) {
+    stem = stem.slice(0, -1);
+  }
+
+  // Restore initial Y if it was changed
+  if (firstCharWasY) stem = "y" + stem.slice(1);
+
+  return stem;
+}

package/src/types.d.ts

@@ -0,0 +1,130 @@
+import { Model } from "./models/types";
+
+type BaseUIConfigField = {
+  name: string;
+  key: string;
+  required: boolean;
+  description: string;
+  scope: "client" | "server";
+  env?: string;
+};
+
+type StringUIConfigField = BaseUIConfigField & {
+  type: "string";
+  placeholder?: string;
+  default?: string;
+};
+
+type SelectUIConfigFieldOptions = {
+  name: string;
+  value: string;
+};
+
+type SelectUIConfigField = BaseUIConfigField & {
+  type: "select";
+  default?: string;
+  options: SelectUIConfigFieldOptions[];
+};
+
+type PasswordUIConfigField = BaseUIConfigField & {
+  type: "password";
+  placeholder?: string;
+  default?: string;
+};
+
+type TextareaUIConfigField = BaseUIConfigField & {
+  type: "textarea";
+  placeholder?: string;
+  default?: string;
+};
+
+type SwitchUIConfigField = BaseUIConfigField & {
+  type: "switch";
+  default?: boolean;
+};
+
+type UIConfigField =
+  | StringUIConfigField
+  | SelectUIConfigField
+  | PasswordUIConfigField
+  | TextareaUIConfigField
+  | SwitchUIConfigField;
+
+type ConfigModelProvider = {
+  id: string;
+  name: string;
+  type: string;
+  chatModels: Model[];
+  config: { [key: string]: any };
+  hash: string;
+  isEnvBased?: boolean; // True if provider was created from environment variables (global keys)
+};
+
+type MCPServerConfig = {
+  id: string;
+  name: string;
+  type: string;
+  url?: string;
+  apiKey?: string;
+  config: { [key: string]: any };
+  enabled: boolean;
+  hash: string;
+};
+
+type Config = {
+  version: number;
+  setupComplete: boolean;
+  preferences: {
+    [key: string]: any;
+  };
+  personalization: {
+    [key: string]: any;
+  };
+  modelProviders: ConfigModelProvider[];
+  mcpServers: MCPServerConfig[];
+  search: {
+    [key: string]: any;
+  };
+};
+
+type EnvMap = {
+  [key: string]: {
+    fieldKey: string;
+    providerKey: string;
+  };
+};
+
+type ModelProviderUISection = {
+  name: string;
+  key: string;
+  fields: UIConfigField[];
+};
+
+type MCPServerUISection = {
+  name: string;
+  key: string;
+  fields: UIConfigField[];
+};
+
+type UIConfigSections = {
+  preferences: UIConfigField[];
+  personalization: UIConfigField[];
+  modelProviders: ModelProviderUISection[];
+  mcpServers: MCPServerUISection[];
+  search: UIConfigField[];
+};
+
+export type {
+  UIConfigField,
+  Config,
+  EnvMap,
+  UIConfigSections,
+  SelectUIConfigField,
+  StringUIConfigField,
+  ModelProviderUISection,
+  MCPServerUISection,
+  ConfigModelProvider,
+  MCPServerConfig,
+  TextareaUIConfigField,
+  SwitchUIConfigField,
+};