extract-webpage 1.2.49 → 1.2.51

package/dist/index.d.ts

@@ -6,8 +6,6 @@
  * @license AGPL-3.0 Organizations should email grokthiscontact@gmail.com
  * to get a dual-use commercial license to remove the GPL requirements.
  */
-export * from './search/search-web';
-export * from './search';
 export * from './tokenize/word-to-root-stem';
 export * from './tokenize/suggest-complete-word';
 export * from './tokenize/text-to-topic-tokens';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-webpage",
-  "version": "1.2.49",
+  "version": "1.2.51",
   "module": "./dist/extract-webpage.es.js",
   "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -18,12 +18,6 @@
       "import": "./dist/extract-webpage.es.js",
       "require": "./dist/extract-webpage.cjs.js"
     },
-    "./search": {
-      "types": "./src/search/index.ts",
-      "react-server": "./src/search/index.ts",
-      "import": "./src/search/index.ts",
-      "require": "./src/search/index.ts"
-    },
     "./*": {
       "types": "./src/*.ts",
       "react-server": "./src/*",
@@ -78,12 +72,12 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "ai": "^5.0.0",
-    "chat-agent-toolkit": "^1.2.48",
+    "chat-agent-toolkit": "^1.2.50",
     "chrono-node": "^2.9.0",
     "drizzle-orm": "^0.45.1",
-    "extract-pdf": "^0.1.36",
+    "extract-pdf": "^0.1.38",
     "extract-youtube": "^1.0.36",
-    "highlight.js": "^11.11.1",
+    "prismjs": "^1.30.0",
     "html-entities": "^2.6.0",
     "js-yaml": "^4.1.1",
     "jsdom": "^28.1.0",
@@ -92,7 +86,6 @@
     "marked": "^17.0.4",
     "node-fetch": "^3.3.2",
     "qwksearch-api-client": "^0.0.12",
-    "tldts": "^7.0.25",
     "zod": "^4.3.6"
   },
   "keywords": [

package/src/html-to-content/html-utils.ts

@@ -139,7 +139,10 @@ export function convertURLToAbsoluteURL(base, relative) {
 }
 
 import { marked } from "marked";
-import hljs from "highlight.js";
+import Prism from "prismjs";
+import loadLanguages from "prismjs/components/index.js";
+
+loadLanguages();
 
 /**
  * Converts Markdown text to HTML. It handles the following Markdown elements:
@@ -169,53 +172,14 @@ import hljs from "highlight.js";
 export function convertMarkdownToHTML(content, toHtml = true) {
   if (!toHtml) return convertHTMLToMarkdown(content);
 
-  // const md = new MarkdownIt({
-  //   highlight: function (str, lang) {
-  //     // If a language is provided and it's recognized by hljs
-  //     if (lang && hljs.getLanguage(lang)) {
-  //       try {
-  //         return (
-  //           '<pre><code class="hljs">' +
-  //           hljs.highlight(str, { language: lang, ignoreIllegals: true }).value
-  //           + '</code></pre>'
-  //         );
-  //       } catch (__) {}
-  //     }
-
-  //     // Default fallback for unsupported or no language
-  //     return (
-  //       '<pre><code class="hljs">' + md.utils.escapeHtml(str) + '</code></pre>'
-  //     );
-  //   },
-  // }).use(function (md) {
-  //   // Override the default fence rule for handling code blocks
-  //   const fence = md.renderer.rules.fence || function (tokens, idx, options, env, slf) {
-  //     const token = tokens[idx];
-  //     const code = token.content
-  //       .trim() // Trim leading/trailing whitespace
-  //       .replace(/^[ \t]*/gm, '') // Remove leading whitespace while preserving relative indentation
-  //       .replace(/&/g, '&amp;') // Encode HTML special characters
-  //       .replace(/</g, '&lt;')
-  //       .replace(/>/g, '&gt;')
-  //       .replace(/"/g, '&quot;')
-  //       .replace(/'/g, '&#39;');
-
-  //     // Wrap code in a blockquote and ignore the language name
-  //     return `<blockquote class="custom-code-block"><pre><code>${code}</code></pre></blockquote>`;
-  //   };
-
-  //   md.renderer.rules.fence = fence;
-  // });
-
-  // // Render markdown content
-  // return md.render(content);
 
   marked.setOptions({
     highlight: function (code, lang) {
-      const language = hljs.getLanguage(lang) ? lang : "plaintext";
-      return hljs.highlight(code, { language }).value;
+      const language = Prism.languages[lang] ? lang : "plaintext";
+      if (language === "plaintext") return code;
+      return Prism.highlight(code, Prism.languages[language], language);
     },
-    langPrefix: "hljs language-",
+    langPrefix: "language-",
   });
 
   return content?.length ? marked.parse(content) : "";

package/src/index.ts

@@ -6,9 +6,6 @@
  * @license AGPL-3.0 Organizations should email grokthiscontact@gmail.com
  * to get a dual-use commercial license to remove the GPL requirements.
  */
-export * from "./search/search-web";
-// Re-export MetaSearchAgent and search handlers (with search functions) for backward compatibility
-export * from "./search";
 export * from "./tokenize/word-to-root-stem";
 export * from "./tokenize/suggest-complete-word";
 export * from "./tokenize/text-to-topic-tokens";

package/dist/search/index.d.ts

@@ -1,14 +0,0 @@
-/**
- * @module extract-webpage/search
- * @description Re-exports MetaSearchAgent from agent-toolkit with search functions
- */
-export declare const searchHandlers: {
-    webSearch: import('chat-agent-toolkit').MetaSearchAgent;
-    academicSearch: import('chat-agent-toolkit').MetaSearchAgent;
-    writingAssistant: import('chat-agent-toolkit').MetaSearchAgent;
-    wolframAlphaSearch: import('chat-agent-toolkit').MetaSearchAgent;
-    youtubeSearch: import('chat-agent-toolkit').MetaSearchAgent;
-    redditSearch: import('chat-agent-toolkit').MetaSearchAgent;
-};
-export { MetaSearchAgent, generateSuggestions, groupAndSummarizeDocs, buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, registerUploadFileLoader, splitTextIntoChunks, LineOutputParser, LineListOutputParser, formatChatHistoryAsString, } from 'chat-agent-toolkit';
-export type { UploadFileLoader, MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample, Document, } from 'chat-agent-toolkit';

package/dist/search/meta-search-agent-reexport.d.ts

@@ -1,8 +0,0 @@
-/**
- * @fileoverview Re-exports MetaSearchAgent from chat-agent-toolkit
- * @deprecated Import from 'chat-agent-toolkit' instead
- */
-export { MetaSearchAgent as default, searchHandlers, generateSuggestions, groupAndSummarizeDocs, buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, splitTextIntoChunks, } from 'chat-agent-toolkit';
-export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample, Document, } from 'chat-agent-toolkit';
-export { LineOutputParser, LineListOutputParser, formatChatHistoryAsString, } from 'chat-agent-toolkit';
-export { webSearchResponsePrompt, webSearchRetrieverPrompt, webSearchRetrieverFewShots, writingAssistantPrompt, } from 'chat-agent-toolkit';

package/dist/search/public-searxng.d.ts

@@ -1,47 +0,0 @@
-/**
- * Search Web via SearXNG metasearch of all major search engines.
- */
-export declare function searchWeb(query: string, options?: SearchOptions): Promise<SearxngSearchResult[] | SearchResponse>;
-interface SearxngSearchOptions {
-    categories?: string[];
-    engines?: string[];
-    language?: string;
-    pageno?: number;
-}
-export declare const searchSearxng: (query: string, opts?: SearxngSearchOptions) => Promise<{
-    results: SearxngSearchResult[];
-    suggestions: string[];
-}>;
-interface SearchOptions {
-    category?: string | number;
-    recency?: string;
-    privateSearxng?: string | boolean | null;
-    maxRetries?: number;
-    page?: number;
-    safesearch?: boolean;
-    lang?: string;
-    proxy?: string | null;
-    useProxy?: boolean;
-}
-export interface SearxngSearchResult {
-    title: string;
-    url: string;
-    snippet?: string;
-    domain?: string;
-    favicon?: string;
-    score?: number;
-    source?: string;
-    date?: string;
-    img_src?: string;
-    thumbnail_src?: string;
-    thumbnail?: string;
-    content?: string;
-    author?: string;
-    iframe_src?: string;
-}
-export interface SearchResponse {
-    results: SearxngSearchResult[];
-    suggestions: string[];
-    infoboxes?: any[];
-}
-export {};

package/dist/search/search-web.d.ts

@@ -1,33 +0,0 @@
-/**
- * Search Web via SearXNG metasearch of all major search engines.
- * Options are 10 search categories, recency, and how many
- * times to retry other domains if first time fails.
- * SearXNG is a free internet metasearch engine which aggregates results from
- *  more than [180+ search sources](https://docs.searxng.org/user/configured_engines.html).
- *
- * [Searxng Overview](https://medium.com/@elmo92/search-in-peace-with-searxng-an-alternative-search-engine-that-keeps-your-searches-private-accd8cddd6fc)
- * [Searxng Installation Guide](https://github.com/searxng/searxng-docker/tree/master)
- *
- * ![google_dead](https://i.imgur.com/6rRpaY1.png)
- * @param {string} query - The search query string.
- * @param {Object} [options]
- * @param {string} options.category default=general - ["general", "news", "videos", "images",
- *  "science","it", "files", "social+media",  "map", "music"]
- * @param {string} options.recency default=all - ["all", "day", "week", "month", "year"]
- * @param {string|boolean} options.privateSearxng default=null - Use your custom domain SearXNG
- * @param {number} options.maxRetries default=3 - Maximum number of retry attempts if the initial search fails.
- * @param {number} options.page default=1 - The page number to retrieve.
- * @param {boolean} options.safesearch default=false - Whether to block adult content.
- * @param {string} options.lang default="en-US" - The language to use for the search.
- * @param {string} options.proxy default=false - Use corsproxy.io to access in frontend JS
- * @returns {Promise<Array<{title: string, url: string, snippet: string, domain: string, favicon: string, path: string, engines: string[]}>>} An array of search result objects.
- * @example  const advancedResults = await searchWeb('Node.js', {
- *   category: 2,
- *   recency: 1,
- *   maxRetries: 5
- * });
- * @category Search
- * @author [vtempest (2025)](https://github.com/vtempest)
- * [Heiser, M., Tauber, A., Flament, A., et al. (2014-)](https://github.com/searxng/searxng/graphs/contributors)
- */
-export declare function searchWeb(query: any, options?: any): Promise<any>;

package/dist/search/tavily.d.ts

@@ -1,20 +0,0 @@
-interface TavilySearchOptions {
-    searchDepth?: "basic" | "advanced";
-    maxResults?: number;
-    includeDomains?: string[];
-    excludeDomains?: string[];
-}
-interface TavilySearchResult {
-    title: string;
-    url: string;
-    content: string;
-    score: number;
-    raw_content?: string;
-}
-export declare const searchTavily: (query: string, opts?: TavilySearchOptions) => Promise<{
-    results: TavilySearchResult[];
-    suggestions: string[];
-}>;
-export declare const getTavilyApiKey: () => any;
-export declare const isTavilyConfigured: () => boolean;
-export {};

package/dist/search/url-to-html.d.ts

@@ -1,62 +0,0 @@
-/**
- * ### Tardigrade the Web Crawler
- *
- * 1. **Use Fetch API, check for bot detection.
- * Scrape  any domain's URL to get its HTML, JSON, or arraybuffer.<br />
- * Scraping internet pages is a [free speech right
- * ](https://blog.apify.com/is-web-scraping-legal/).
- * 2. Features: timeout, redirects, default UA, referer as google, and bot
- * detection checking. <br />
- * 3. If fetch method does not get needed HTML, use Docker proxy as backup.
- *
- * 4. [Setup Docker](https://github.com/vtempest/ai-research-agent/tree/master/src/crawler)
- *  container with NodeJS server API renders with puppeteer DOM to get all HTML loaded by
- *  secondary in-page API requests after the initial page request, including user login and cookie storage.
- * 5. Bypass Cloudflare bot check: A webpage proxy that request
- * through Chromium (puppeteer) - can be used to bypass Cloudflare
- * anti bot using cookie id javascript method.
- * 6. Send your request to the server with the port 3000 and add your URL to the "url"
- *  query string like this: `http://localhost:3000/?url=https://example.org`
- *
- * 7. Optional: Setup residential IP proxy to access sites that IP-block datacenters
- *  and manage rotation with [Scrapoxy](https://scrapoxy.io). Recommended:
- * [Hypeproxy](https://hypeproxy.io/products/static-residential-proxies)
- * [NinjasProxy](https://ninjasproxy.com/residential-proxies/)
- * [Proxy-Cheap](https://app.proxy-cheap.com/order)
- * [LiveProxies](https://liveproxies.io/rotating-residential-proxies-pricing)
- *
- * @param {string} url - any domain's URL
- * @param {Object} [options]
- * @param {number} options.timeout default=5 -  abort request if not retrived, in seconds
- * @param {number} options.maxRedirects default=3 - max redirects to follow
- * @param {number} options.checkBotDetection default=true - check for bot detection messages
- * @param {number} options.changeReferer default=true - set referer as google
- * @param {number} options.userAgentIndex default=0 - index of [google bot, default chrome]
- * @param {string} options.proxy default=false - use proxy url
- * @param {boolean} options.checkRobotsAllowed default=false - check robots.txt rules
- * @returns {Promise<string>} -  HTML, JSON, arraybuffer, or error object
- * @category Extract
- * @example await scrapeURL("https://hckrnews.com", {timeout: 5, userAgentIndex: 1})
- * @author [vtempest (2025)](https://github.com/vtempest)
- * @license MIT
- */
-export declare function scrapeURL(url: any, options?: any): Promise<any>;
-/**
- * As backup, scrape with JINA to get html
- * @param {string} url
- * @returns {Promise<string>}
- */
-export declare function scrapeJINA(url: any, timeout?: number): Promise<string>;
-/**
- * Fetches and parses the robots.txt file for a given URL.
- * @param {string} url - The base URL to fetch the robots.txt from.
- * @returns {Promise<Object>} A JSON object representing the parsed robots.txt.
- */
-export declare function fetchScrapingRules(url: any): Promise<{
-    directives: {};
-    crawlDelay: {};
-    sitemaps: any[];
-    preferredHost: any;
-} | {
-    error: string;
-}>;