extract-webpage 1.2.360 → 1.2.362

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-webpage",
-  "version": "1.2.360",
+  "version": "1.2.362",
   "module": "./dist/extract-webpage.es.js",
   "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -74,11 +74,11 @@
   "dependencies": {
     "@huggingface/transformers": "^4.2.0",
     "ai": "^7.0.94",
-    "chat-agent-toolkit": "^1.2.366",
+    "chat-agent-toolkit": "^1.2.368",
     "chrono-node": "^2.10.1",
     "drizzle-orm": "^0.45.2",
-    "extract-pdf": "^0.1.350",
-    "extract-youtube": "^1.0.354",
+    "extract-pdf": "^0.1.352",
+    "extract-youtube": "^1.0.356",
     "html-entities": "^2.6.0",
     "js-yaml": "^5.4.1",
     "jsdom": "^30.0.1",

package/src/html-to-content/html-utils.d.ts

@@ -0,0 +1,131 @@
+/**
+ * @module research/extractor/html-to-content/html-utils
+ * @description Research library module.
+ */
+/**
+ * Converts URL-safe escaped HTML codes like &"'`’ & to standard HTML or in reverse.
+ * @param {string} str - The string to process.
+ * @param {boolean} toStandardHTML  default=true - If true, converts url-safe codes
+ * to standard HTML. If false, converts standard HTML to url-safe codes.
+ * @return {string} The processed string.
+ * @category HTML Utilities
+ * @example
+ * var normalHTML = convertURLSafeHTMLToHTML('<p>This & that © 2023 '+
+ * '"Quotes"'Apostrophes' €100 ☺</p>', true)
+ * console.log(normalHTML) // "<p>This & that \u00a9 2023 "Quotes" 'Apostrophes' \u20ac100 \u263a</p>"
+ */
+export declare function convertURLSafeHTMLToHTML(str: any, toStandardHTML?: boolean): any;
+/**
+ * Convert relative URL to absolute URL using base URL.
+ * @param {string} base base url of the domain
+ * @param {string} relative partial urls like ../images/image.jpg #hash
+ * @returns {string} absolute URL
+ * @example
+ * var absoluteURL = convertURLToAbsoluteURL('https://example.com', 'images/image.jpg')
+ * console.log(absoluteURL) // Returns: "https://example.com/images/image.jpg"
+ * var absoluteURL = convertURLToAbsoluteURL('https://example.com', '//images/image.jpg')
+ * console.log(absoluteURL) // Returns: "https:images/image.jpg"
+ * @category HTML Utilities
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function convertURLToAbsoluteURL(base: any, relative: any): any;
+/**
+ * Converts Markdown text to HTML. It handles the following Markdown elements:
+ * - Headers (h1 to h6)
+ * - Bold text
+ * - Italic text
+ * - Unordered lists
+ * - Ordered lists
+ * - Paragraphs
+ * - Images
+ * - Links
+ * - Code blocks
+ * @param {string} content - The Markdown or HTML content to be converted.
+ * @param {boolean} toHtml - default=true - If true, converts Markdown to HTML.
+ *                          If false, converts HTML to Markdown.
+ * @returns {string} The resulting HTML string.
+ * @category HTML Utilities
+ * @example
+ * const markdown = "# Header\n\nThis is **bold** and *italic* text.\n\n* List item 1\n* List item 2";
+ * const html = convertMarkdownToHTML(markdown);
+ * console.log(html);
+ * // Output:
+ * // <h1>Header</h1>
+ * // <p>This is <strong>bold</strong> and <em>italic</em> text.</p>
+ * // <ul><li>List item 1</li><li>List item 2</li></ul>
+ */
+export declare function convertMarkdownToHTML(content: any, toHtml?: boolean): any;
+/**
+ * Detect whether a string is Markdown (rather than HTML or plain text) using
+ * regexp checks. Content that is dominated by HTML tags is never treated as
+ * Markdown, so real scraped pages pass through untouched; text needs at least
+ * two distinct Markdown syntax signals (or several links/images in Markdown
+ * form) to qualify. Used to catch scraper responses (e.g. the JINA reader or
+ * proxies wrapping it) that return Markdown in place of HTML, so it can be
+ * converted before main-content extraction — otherwise the article panel
+ * renders raw `[text](url)` syntax.
+ *
+ * @param {string} text - The content to test.
+ * @returns {boolean} True when the content should be parsed as Markdown.
+ * @category HTML Utilities
+ * @example
+ * detectMarkdown("# Title\n\nSome **bold** text.") // true
+ * detectMarkdown("<html><body><p>Hi</p></body></html>") // false
+ */
+export declare function detectMarkdown(text: any): boolean;
+/**
+ * Remove extra non-article content from a Markdown extraction using regexp
+ * checks: JINA reader metadata lines, cookie/consent and navigation phrases,
+ * and runs of consecutive link-only lines (menus, breadcrumbs, "related"
+ * link farms) whose targets are mostly relative site navigation. Standalone
+ * links inside prose are kept.
+ *
+ * @param {string} markdown - The Markdown content to clean.
+ * @returns {string} The cleaned Markdown.
+ * @category HTML Utilities
+ * @example
+ * removeMarkdownNavigation("Title: Page\n[Skip to content](#main)\nReal text")
+ * // => "Real text"
+ */
+export declare function removeMarkdownNavigation(markdown: any): string;
+/**
+ * Convert a Markdown document to formatted HTML using regular expressions to
+ * detect Markdown syntax. Unlike {@link convertMarkdownToHTML} (which relies on
+ * the `marked` library), this is a dependency-free, self-contained converter
+ * intended for post-processing content returned as Markdown (e.g. from the
+ * JINA reader fallback in the scraper).
+ *
+ * Supported block elements: ATX headers (`#`..`######`), setext headers
+ * (`===`/`---` underlines), fenced code blocks (```lang), blockquotes (`>`),
+ * unordered lists (`-`, `*`, `+`), ordered lists (`1.`, `1)`), pipe tables,
+ * horizontal rules (`---`, `***`, `___`) and paragraphs.
+ * Supported inline elements: bold, italic, strikethrough, inline code, images,
+ * links, linked images (`[![alt](src)](href)`) and autolinks (`<https://…>`).
+ *
+ * @param {string} markdown - The Markdown content to convert.
+ * @returns {string} The resulting formatted HTML string.
+ * @category HTML Utilities
+ * @example
+ * convertMarkdownToFormattedHTML("# Title\n\nSome **bold** text.");
+ * // => "<h1>Title</h1>\n<p>Some <strong>bold</strong> text.</p>"
+ */
+export declare function convertMarkdownToFormattedHTML(markdown: any): string;
+export declare function convertHTMLToMarkdown(html: any): any;
+/**
+ * Copy HTML to clipboard. When pasting into rich text field,
+ * pastes rich text. When pasting into plain text field, pastes:
+ * plain text, html, or markdown.
+ *
+ * @param {string} html - The HTML content to be copied.
+ * @param {object} options - The options object.
+ * @param {number} options.pastePlainFormat -
+ * default=0
+ * 0 - plain text
+ * 1 - markdown
+ * 2 - html
+ * @returns {Promise<void>} - A promise that resolves when
+ * the HTML is copied to the clipboard.
+ * @category HTML Utilities
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function copyHTMLToClipboard(html: any, options?: {}): Promise<void>;

package/src/html-to-content/prism-global.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @module html-to-content/prism-global
+ * @description Owns the Prism instance: publishes it on `globalThis`, then
+ * registers the language grammars that prismjs' own entry point leaves out —
+ * without ever depending on module evaluation order.
+ *
+ * The `prismjs/components/*` files are plain browser scripts, not modules:
+ * `prism-markup.js` literally opens with `Prism.languages.markup = {...}`,
+ * resolving `Prism` as a free variable off the global object. Someone has to
+ * put it there first. prismjs' own entry point does that for `window`
+ * (browser) and `global` (Node); on Cloudflare Workers / edge runtimes it sees
+ * neither, which is what the assignment below is for.
+ *
+ * Publishing it is only half the job, though: it has to happen *before* the
+ * grammar scripts run, and a static `import "prismjs/components/..."` cannot
+ * promise that once a bundler is in the loop. Those files declare no
+ * dependency on Prism — reading a global is invisible to the module graph — so
+ * nothing pins them after whoever publishes it, and a bundler is free to hoist
+ * them, split them into another chunk, or drop the publishing statement as a
+ * dead `globalThis` write. When that happens the chunk dies on load with
+ * `ReferenceError: Prism is not defined`, which takes down the whole route
+ * that lazily imported it rather than just the syntax highlighting.
+ *
+ * So the grammars are loaded with `import()` from inside `loadPrismGrammars()`
+ * instead. The global is published by a statement earlier in that same
+ * function body, which makes the ordering a runtime fact rather than a promise
+ * the bundler has to keep.
+ */
+import Prism from "prismjs";
+/**
+ * Registers the grammars above, once per runtime. Cheap to call repeatedly —
+ * callers are meant to invoke it wherever they are about to highlight, so that
+ * the work does not hinge on a module-level statement surviving tree-shaking.
+ *
+ * Never rejects: a grammar that fails to load just leaves its language out of
+ * `Prism.languages`, which every caller already treats as "render this block
+ * unhighlighted".
+ */
+export declare function loadPrismGrammars(): Promise<void>;
+export default Prism;

package/src/suggest-next-words/autocomplete-search-engines.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Provides search query autocomplete/suggestions from various search engines.
+ */
+/**
+ * Autocomplete function type
+ */
+type AutocompleteFunction = (query: string, locale?: string) => Promise<string[]>;
+/**
+ * Baidu autocomplete
+ */
+export declare function baidu(query: string, _locale?: string): Promise<string[]>;
+/**
+ * Brave autocomplete
+ */
+export declare function brave(query: string, _locale?: string): Promise<string[]>;
+/**
+ * DuckDuckGo autocomplete
+ */
+export declare function duckduckgo(query: string, locale?: string): Promise<string[]>;
+/**
+ * Google autocomplete
+ */
+export declare function google(query: string, locale?: string): Promise<string[]>;
+/**
+ * Qwant autocomplete
+ */
+export declare function qwant(query: string, locale?: string): Promise<string[]>;
+/**
+ * Startpage autocomplete
+ */
+export declare function startpage(query: string, locale?: string): Promise<string[]>;
+/**
+ * Wikipedia autocomplete
+ */
+export declare function wikipedia(query: string, locale?: string): Promise<string[]>;
+/**
+ * Yandex autocomplete
+ */
+export declare function yandex(query: string, _locale?: string): Promise<string[]>;
+/**
+ * Available autocomplete backends
+ */
+export declare const backends: {
+    [key: string]: AutocompleteFunction;
+};
+/**
+ * Get autocomplete suggestions from a specific backend
+ *
+ * @param backendName - Name of the autocomplete backend
+ * @param query - Search query
+ * @param locale - Locale/language code (e.g., 'en-US', 'de-DE')
+ * @returns Array of suggestion strings
+ */
+export declare function searchAutocomplete(backendName: string, query: string, locale?: string): Promise<string[]>;
+/**
+ * Get autocomplete suggestions from multiple backends and merge them
+ *
+ * @param backendNames - Array of backend names to query
+ * @param query - Search query
+ * @param locale - Locale/language code
+ * @returns Merged and deduplicated array of suggestions
+ */
+export declare function searchAutocompleteMulti(backendNames: string[], query: string, locale?: string): Promise<string[]>;
+export {};