extract-webpage 1.2.5

package/src/suggest-next-words/autocomplete-ai.ts

@@ -0,0 +1,38 @@
+// import { pipeline, type TextGenerationSingle } from "@huggingface/transformers";
+
+// /**
+//  * Generates the next words for a given prompt using a text generation model.
+//  *
+//  * The DistilGPT2 model is a distilled, smaller version of OpenAI's GPT-2, featuring
+//  * around 82 million parameters (~80MB), and available in Hugging Face Transformers
+//  * for efficient next word (next-token) prediction tasks in English text. It is
+//  * well-suited for sequence generation and can be used as a next word prediction
+//  * model with code similar to GPT2, but offers faster and lighter inference compared
+//  * GPT-2 (124M parameter) model, preserving most capabilities but running
+//  * significantly faster and requiring less memory.
+//  * @see https://huggingface.co/Xenova/distilgpt2
+//  * @param {string} prompt - The input prompt to complete.
+//  * @param {Object} [options] - Generation options.
+//  * @param {number} [options.maxTokens=16] - Maximum number of new tokens to generate.
+//  * @param {string} [options.model='Xenova/distilgpt2'] - The model identifier to use.
+//  * @returns {Promise<string>} - The generated text completions.
+//  */
+// export async function predictNextWordsWithSmallLocalModel(
+//   prompt: string,
+//   { model = "Xenova/distilgpt2", maxTokens = 16, modelParams = {} } = {}
+// ): Promise<string> {
+//   if (!prompt) throw new Error("Prompt is required");
+//   return (
+//     await (
+//       await pipeline("text-generation", model, {
+//         dtype: "fp16",
+//         device: "cpu",
+//         ...modelParams,
+//       })
+//     )(prompt, { max_new_tokens: maxTokens })
+//   )
+//     .map((res) => (res as TextGenerationSingle).generated_text)
+//     .join(" ")
+//     .replace(prompt, "")
+//     .trim();
+// }

package/src/suggest-next-words/autocomplete-search-engines.ts

@@ -0,0 +1,435 @@
+/**
+ * Provides search query autocomplete/suggestions from various search engines.
+ */
+
+import { parseHTML } from "linkedom";
+
+// Some suggest APIs (notably Google) return 403 Forbidden for requests
+// without a browser-like User-Agent, especially from datacenter/Cloudflare
+// egress IPs. Sent by default; per-backend headers can still override it.
+const DEFAULT_USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+
+/**
+ * Fetch wrapper compatible with grab-url interface
+ */
+async function grab(url: string, options: any = {}): Promise<any> {
+  const timeout = options.timeout ? options.timeout * 1000 : 5000;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        "User-Agent": DEFAULT_USER_AGENT,
+        ...options.headers,
+      },
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    if (options.responseType === "json") {
+      return { data: await response.json() };
+    }
+    if (options.responseType === "arraybuffer") {
+      return await response.arrayBuffer();
+    }
+    if (options.responseType === "text") {
+      return await response.text();
+    }
+
+    // Default to json for backwards compatibility
+    try {
+      return { data: await response.json() };
+    } catch {
+      return await response.text();
+    }
+  } catch (error) {
+    clearTimeout(timeoutId);
+    throw error;
+  }
+}
+
+/**
+ * Autocomplete function type
+ */
+type AutocompleteFunction = (
+  query: string,
+  locale?: string,
+) => Promise<string[]>;
+
+/**
+ * Get autocomplete suggestions with error handling
+ */
+async function getSuggestions(url: string, options: any = {}): Promise<any> {
+  try {
+    const response = await grab(url, {
+      timeout: 3,
+      responseType: options.responseType || "json",
+      ...options,
+    });
+
+    if (typeof response === "object" && "data" in response) {
+      return response.data;
+    }
+    return response;
+  } catch (error) {
+    console.error("Autocomplete error:", error);
+    return null;
+  }
+}
+
+/**
+ * Baidu autocomplete
+ */
+export async function baidu(
+  query: string,
+  _locale?: string,
+): Promise<string[]> {
+  const params = new URLSearchParams({
+    ie: "utf-8",
+    json: "1",
+    prod: "pc",
+    wd: query,
+  });
+
+  const url = `https://www.baidu.com/sugrec?${params.toString()}`;
+  const data = await getSuggestions(url);
+  const results: string[] = [];
+
+  if (data && data.g) {
+    for (const item of data.g) {
+      results.push(item.q);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Brave autocomplete
+ */
+export async function brave(
+  query: string,
+  _locale?: string,
+): Promise<string[]> {
+  const params = new URLSearchParams({ q: query });
+  const url = `https://search.brave.com/api/suggest?${params.toString()}`;
+
+  const data = await getSuggestions(url, {
+    headers: {
+      Cookie: "country=all",
+    },
+  });
+
+  if (data && Array.isArray(data) && data.length > 1) {
+    return data[1];
+  }
+
+  return [];
+}
+
+/**
+ * DuckDuckGo autocomplete
+ */
+export async function duckduckgo(
+  query: string,
+  locale: string = "en-US",
+): Promise<string[]> {
+  // Extract region code (e.g., 'us-en' from 'en-US')
+  const region = locale.toLowerCase().split("-").reverse().join("-");
+
+  const params = new URLSearchParams({
+    q: query,
+    kl: region,
+  });
+
+  const url = `https://duckduckgo.com/ac/?type=list&${params.toString()}`;
+  const data = await getSuggestions(url);
+
+  if (data && Array.isArray(data) && data.length > 1) {
+    return data[1];
+  }
+
+  return [];
+}
+
+/**
+ * Google autocomplete
+ */
+export async function google(
+  query: string,
+  locale: string = "en",
+): Promise<string[]> {
+  // Map locale to Google subdomain
+  const subdomainMap: { [key: string]: string } = {
+    de: "google.de",
+    fr: "google.fr",
+    es: "google.es",
+    it: "google.it",
+    nl: "google.nl",
+    pt: "google.pt",
+    ru: "google.ru",
+    ja: "google.co.jp",
+    zh: "google.com.hk",
+    ko: "google.co.kr",
+  };
+
+  const lang = locale.split("-")[0];
+  const subdomain = subdomainMap[lang] || "google.com";
+
+  const params = new URLSearchParams({
+    q: query,
+    client: "gws-wiz",
+    hl: lang,
+  });
+
+  const url = `https://${subdomain}/complete/search?${params.toString()}`;
+  const response = await getSuggestions(url, { responseType: "text" });
+
+  const results: string[] = [];
+
+  if (response) {
+    try {
+      // Extract JSON from response
+      const jsonStart = response.indexOf("[");
+      const jsonEnd = response.lastIndexOf("]") + 1;
+
+      if (jsonStart >= 0 && jsonEnd > jsonStart) {
+        const jsonText = response.substring(jsonStart, jsonEnd);
+        const data = JSON.parse(jsonText);
+
+        if (data[0]) {
+          for (const item of data[0]) {
+            // Parse HTML entities
+            const { document } = parseHTML(item[0]);
+            const text = document.body?.textContent?.trim();
+            if (text) {
+              results.push(text);
+            }
+          }
+        }
+      }
+    } catch (e) {
+      // Ignore parse errors
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Qwant autocomplete
+ */
+export async function qwant(
+  query: string,
+  locale: string = "en_US",
+): Promise<string[]> {
+  const params = new URLSearchParams({
+    q: query,
+    locale: locale.replace("-", "_"),
+    version: "2",
+  });
+
+  const url = `https://api.qwant.com/v3/suggest?${params.toString()}`;
+  const data = await getSuggestions(url);
+  const results: string[] = [];
+
+  if (data && data.status === "success" && data.data && data.data.items) {
+    for (const item of data.data.items) {
+      results.push(item.value);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Startpage autocomplete
+ */
+export async function startpage(
+  query: string,
+  locale: string = "en",
+): Promise<string[]> {
+  const langMap: { [key: string]: string } = {
+    da: "dansk",
+    de: "deutsch",
+    en: "english",
+    es: "espanol",
+    fr: "francais",
+    nb: "norsk",
+    nl: "nederlands",
+    pl: "polski",
+    pt: "portugues",
+    sv: "svenska",
+  };
+
+  const baseLang = locale.split("-")[0];
+  const lui = langMap[baseLang] || "english";
+
+  const params = new URLSearchParams({
+    q: query,
+    format: "opensearch",
+    segment: "startpage.defaultffx",
+    lui: lui,
+  });
+
+  const url = `https://www.startpage.com/suggestions?${params.toString()}`;
+  const data = await getSuggestions(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    },
+  });
+
+  if (
+    data &&
+    Array.isArray(data) &&
+    data.length >= 2 &&
+    Array.isArray(data[1])
+  ) {
+    return data[1];
+  }
+
+  return [];
+}
+
+/**
+ * Wikipedia autocomplete
+ */
+export async function wikipedia(
+  query: string,
+  locale: string = "en",
+): Promise<string[]> {
+  const langMap: { [key: string]: string } = {
+    en: "en.wikipedia.org",
+    de: "de.wikipedia.org",
+    fr: "fr.wikipedia.org",
+    es: "es.wikipedia.org",
+    it: "it.wikipedia.org",
+    nl: "nl.wikipedia.org",
+    pt: "pt.wikipedia.org",
+    ru: "ru.wikipedia.org",
+    ja: "ja.wikipedia.org",
+    zh: "zh.wikipedia.org",
+    ar: "ar.wikipedia.org",
+    ko: "ko.wikipedia.org",
+  };
+
+  const lang = locale.split("-")[0];
+  const netloc = langMap[lang] || "en.wikipedia.org";
+
+  const params = new URLSearchParams({
+    action: "opensearch",
+    format: "json",
+    formatversion: "2",
+    search: query,
+    namespace: "0",
+    limit: "10",
+  });
+
+  const url = `https://${netloc}/w/api.php?${params.toString()}`;
+  const data = await getSuggestions(url);
+
+  if (data && Array.isArray(data) && data.length > 1) {
+    return data[1];
+  }
+
+  return [];
+}
+
+/**
+ * Yandex autocomplete
+ */
+export async function yandex(
+  query: string,
+  _locale?: string,
+): Promise<string[]> {
+  const params = new URLSearchParams({ part: query });
+  const url = `https://suggest.yandex.com/suggest-ff.cgi?${params.toString()}`;
+
+  const data = await getSuggestions(url);
+
+  if (data && Array.isArray(data) && data.length > 1) {
+    return data[1];
+  }
+
+  return [];
+}
+
+/**
+ * Available autocomplete backends
+ */
+export const backends: { [key: string]: AutocompleteFunction } = {
+  baidu,
+  brave,
+  duckduckgo,
+  google,
+  qwant,
+  startpage,
+  wikipedia,
+  yandex,
+};
+
+/**
+ * Get autocomplete suggestions from a specific backend
+ *
+ * @param backendName - Name of the autocomplete backend
+ * @param query - Search query
+ * @param locale - Locale/language code (e.g., 'en-US', 'de-DE')
+ * @returns Array of suggestion strings
+ */
+export async function searchAutocomplete(
+  backendName: string,
+  query: string,
+  locale: string = "en-US",
+): Promise<string[]> {
+  const backend = backends[backendName];
+
+  if (!backend) {
+    console.warn(`Autocomplete backend '${backendName}' not found`);
+    return [];
+  }
+
+  try {
+    return await backend(query, locale);
+  } catch (error) {
+    console.error(`Autocomplete error for ${backendName}:`, error);
+    return [];
+  }
+}
+
+/**
+ * Get autocomplete suggestions from multiple backends and merge them
+ *
+ * @param backendNames - Array of backend names to query
+ * @param query - Search query
+ * @param locale - Locale/language code
+ * @returns Merged and deduplicated array of suggestions
+ */
+export async function searchAutocompleteMulti(
+  backendNames: string[],
+  query: string,
+  locale: string = "en-US",
+): Promise<string[]> {
+  const promises = backendNames.map((name) =>
+    searchAutocomplete(name, query, locale),
+  );
+  const results = await Promise.all(promises);
+
+  // Merge and deduplicate
+  const merged = new Set<string>();
+  for (const result of results) {
+    for (const suggestion of result) {
+      merged.add(suggestion);
+    }
+  }
+
+  return Array.from(merged);
+}

package/src/tokenize/suggest-complete-word.ts

@@ -0,0 +1,137 @@
+/**
+ * @fileoverview Utility for suggesting word and phrase completions based on a trie model.
+ * Used for search autocomplete and real-time query suggestions.
+ */
+export interface SuggestCompletionsOptions {
+  phrasesModel: Record<string, Record<string, Array<[string | null, number, number]>>>;
+  limitMaxResults?: number;
+  numberOfLastWordsToCheck?: number;
+  optionShowFullQuery?: boolean;
+}
+
+export interface SuggestionResult {
+  name?: string;
+  word?: string;
+  phrase?: string;
+}
+
+/**
+ * ### Autocomplete Topic Phrase Completions
+ * <img width="350px"  src="https://i.imgur.com/0k5mO76.png" />
+ *
+ * Completes the query with the most likely next words for phrases.
+ * If typing 2+ letters of a word, returns all possible words matching those few letters.
+ *
+ *
+ * @param {string} query - The input query which can be pertial words or phrases.
+ * @param {Object} [options]
+ * @param {Object} options.phrasesModel - A custom phrases model to use for autocomplete suggestions.
+ * @param {number} options.limitMaxResults default=10 - The maximum number of autocomplete suggestions to return.
+ * @param {number} options.numberOfLastWordsToCheck default=5 - The number of last words in the query to check for phrase completions.
+ * @returns {Promise<Array<Object>>} An array of autocomplete suggestions, each containing either a 'phrase' or 'word' property.
+ * @example
+ * // Basic usage
+ * const suggestions = await suggestNextWordCompletions("self att");
+ * // Possible output: [{ phrase: "self attention" }, { phrase: "self attract" }, { phrase: "self attack" }]
+ *
+ * @example
+ * // Using options
+ * const customModel = await import("./custom-phrases-model.json");
+ * const suggestions = await suggestNextWordCompletions("artificial int", {
+ *   phrasesModel: customModel,
+ *   limitMaxResults: 5,
+ *   numberOfLastWordsToCheck: 3
+ * });
+ * // Possible output: [{ phrase: "artificial intelligence" }, { phrase: "artificial interpretation" }]
+ *
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @category Topics
+ */
+export async function suggestNextWordCompletions(
+  query: string,
+  options: Partial<SuggestCompletionsOptions> = {},
+): Promise<SuggestionResult[] | undefined> {
+  var {
+    phrasesModel, //pass in remote model
+    limitMaxResults = 10, //limit the number of results
+    numberOfLastWordsToCheck = 5, // check last few words for their phrase completions
+    optionShowFullQuery = true, //show full query in the result
+  } = options;
+
+  if (!phrasesModel) {
+    throw new Error("Missing phrasesModel");
+  }
+
+  // if (!phrasesModel)
+  //   phrasesModel = await import ( "../wordlists/wiki-phrases-model-240k.json");
+
+  //strip non-alphanumeric characters from query and -'
+  query = query.trim().replace(/[^a-zA-Z0-9\s\-\']/g, "");
+
+  //split into words
+  var words = query.toLowerCase().split(/\W+/);
+
+  if (query.length < 1) return;
+
+  const autocompleteGroups: SuggestionResult[][] = [];
+
+  const lastWords = words.slice(-numberOfLastWordsToCheck);
+
+  for (var i = 0; i < lastWords.length; i++) {
+    // Iterate over the tail-window we just sliced; using the full `words` array here
+    // skews matching when `numberOfLastWordsToCheck` is smaller than query length.
+    var word = lastWords[i];
+
+    //Find next word query completion list
+    var firstTwoLetters = word.slice(0, 2);
+    const possiblePhrases = phrasesModel[firstTwoLetters]
+      ? phrasesModel[firstTwoLetters][word]
+      : null;
+
+    if (possiblePhrases) {
+      var nextWords = words
+        .slice(i + 1)
+        .join(" ")
+        .trim();
+
+      const phraseMatches = possiblePhrases
+        .filter((phrase) =>
+          phrase[0] ? phrase[0].startsWith(nextWords) : false
+        )
+        .map((phrase) => ({ phrase: word + " " + phrase[0] }));
+
+      const possiblePhrasesInput = JSON.parse(JSON.stringify(phraseMatches));
+      autocompleteGroups.push(possiblePhrasesInput);
+    }
+  }
+
+  //if typing 2 letters of first word,
+  // return all possible first words matched what is typed in
+  let lastWord = words[words.length - 1];
+  autocompleteGroups.push(
+    Object.keys(phrasesModel[lastWord.slice(0, 2)] || {})
+      .filter((phrase) => phrase.startsWith(lastWord))
+      .map((phrase) => ({ word: phrase }))
+  );
+
+  let autocompletes: SuggestionResult[] = autocompleteGroups.flat(2).slice(0, limitMaxResults);
+
+  if (optionShowFullQuery)
+    autocompletes = autocompletes
+      .map((s) => {
+        var name;
+        if (s.word) name = words.slice(0, -1).join(" ") + " " + s.word;
+        if (s.phrase) {
+          name =
+            words
+              .join(" ")
+              .slice(0, words.join(" ").lastIndexOf(s.phrase.split(" ")[0])) +
+            " " +
+            s.phrase;
+        }
+        return { name };
+      })
+      .filter(Boolean);
+
+  return autocompletes;
+}