extract-webpage 1.2.5

package/dist/html-to-cite/extract-author.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Extracts the author from the document and validates it as a human name
+ *
+ * @param {Document} document
+ * @returns {object|null} author_cite, author_short, author_type - or null if no valid author found
+*/
+export declare function extractAuthor(document: any): {
+    author_cite: any;
+    author_short: any;
+    author_type: number;
+};

package/dist/html-to-cite/extract-cite.d.ts

@@ -0,0 +1,33 @@
+export interface ExtractCiteResult {
+    author?: string;
+    author_cite?: string;
+    date?: string;
+    title?: string;
+    source?: string;
+}
+export interface ExtractCiteOptions {
+    url?: string;
+}
+/**
+ * ### \u1f4da\u1f48e Extract Expert Excerpt
+ * <img width="350px" src="https://i.imgur.com/4GOOM9s.jpeg" />
+ *
+ * Extract author, date, source, and title from HTML using meta tags
+ * and common class names. Validates human name from author string to check
+ * against common list of 90k first names, last names,and organizations to infer
+ * if it should be reversed starting by author last name (accounting for affixes/titles),
+ * since organizations are not reversed.
+ * [Article Extraction Benchmark](https://github.com/scrapinghub/article-extraction-benchmark?tab=readme-ov-file#results)
+ * @param {Document | string} document dom object or html string with article content
+ * @param {ExtractCiteOptions} [options={}]
+ * @returns {ExtractCiteResult | null} An object containing extracted citation information.
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function extractCite(document: Document | string, options?: ExtractCiteOptions): {
+    author: string;
+    author_cite: any;
+    date: string;
+    title: string;
+    source: string;
+};

package/dist/html-to-cite/extract-date/date-extractors.d.ts

@@ -0,0 +1,40 @@
+declare const FAST_PREPEND = "";
+declare const MIN_SEGMENT_LEN = 6;
+declare const MAX_SEGMENT_LEN = 52;
+declare const DATE_EXPRESSIONS: string;
+declare const SLOW_PREPEND = "";
+declare const FREE_TEXT_EXPRESSIONS = ".//*[self::div or self::h2 or self::h3 or self::h4 or self::li or self::p or self::span or self::time or self::ul]/text()";
+declare const THREE_COMP_REGEX_A: RegExp;
+declare const THREE_COMP_REGEX_B: RegExp;
+declare const TWO_COMP_REGEX: RegExp;
+declare const YEAR_PATTERN: RegExp;
+declare const COPYRIGHT_PATTERN: RegExp;
+declare const THREE_PATTERN: RegExp;
+declare const THREE_CATCH: RegExp;
+declare const THREE_LOOSE_PATTERN: RegExp;
+declare const THREE_LOOSE_CATCH: RegExp;
+declare const SELECT_YMD_PATTERN: RegExp;
+declare const SELECT_YMD_YEAR: RegExp;
+declare const YMD_YEAR: RegExp;
+declare const DATESTRINGS_PATTERN: RegExp;
+declare const DATESTRINGS_CATCH: RegExp;
+declare const SLASHES_PATTERN: RegExp;
+declare const SLASHES_YEAR: RegExp;
+declare const YYYYMM_PATTERN: RegExp;
+declare const YYYYMM_CATCH: RegExp;
+declare const MMYYYY_PATTERN: RegExp;
+declare const MMYYYY_YEAR: RegExp;
+declare const SIMPLE_PATTERN: RegExp;
+declare const YMD_PATTERN: RegExp;
+declare const TIMESTAMP_PATTERN: RegExp;
+declare function discard_unwanted(tree: any): any[];
+declare function extract_url_date(testurl: any, options: any): string;
+declare function regex_parse(string: any): Date;
+declare function custom_parse(string: any, outputformat: any, min_date: any, max_date: any): any;
+declare function external_date_parser(string: any, outputformat: any): string;
+declare function try_date_expr(string: any, outputformat: any, extensive_search: any, min_date: any, max_date: any): any;
+declare function img_search(tree: any, options: any): string;
+declare function pattern_search(text: any, date_pattern: any, options: any): any;
+declare function json_search(tree: any, options: any): any;
+declare function idiosyncrasies_search(htmlstring: any, options: any): any;
+export { discard_unwanted, extract_url_date, regex_parse, custom_parse, external_date_parser, try_date_expr, img_search, pattern_search, json_search, idiosyncrasies_search, DATE_EXPRESSIONS, FAST_PREPEND, SLOW_PREPEND, FREE_TEXT_EXPRESSIONS, MAX_SEGMENT_LEN, MIN_SEGMENT_LEN, YEAR_PATTERN, YMD_PATTERN, COPYRIGHT_PATTERN, TIMESTAMP_PATTERN, THREE_PATTERN, THREE_CATCH, THREE_LOOSE_PATTERN, THREE_LOOSE_CATCH, SELECT_YMD_PATTERN, SELECT_YMD_YEAR, YMD_YEAR, DATESTRINGS_PATTERN, DATESTRINGS_CATCH, SLASHES_PATTERN, SLASHES_YEAR, YYYYMM_PATTERN, YYYYMM_CATCH, MMYYYY_PATTERN, MMYYYY_YEAR, SIMPLE_PATTERN, THREE_COMP_REGEX_A, THREE_COMP_REGEX_B, TWO_COMP_REGEX, };

package/dist/html-to-cite/extract-date/date-validators.d.ts

@@ -0,0 +1,15 @@
+/**
+ * @fileoverview Validation and filtering logic for extracted date candidates.
+ * Ensures dates fall within plausible ranges and meet format requirements.
+ */
+declare function is_valid_date(date_input: any, outputformat: any, earliest: any, latest: any): boolean;
+declare function is_valid_format(outputformat: any): boolean;
+declare function plausible_year_filter(htmlstring: any, pattern: any, yearpat: any, earliest: any, latest: any, incomplete?: boolean): Map<any, any>;
+declare function compare_values(reference: any, attempt: any, options: any): any;
+declare function filter_ymd_candidate(bestmatch: any, pattern: any, original_date: any, copyear: any, outputformat: any, min_date: any, max_date: any): any;
+declare function convert_date(datestring: any, inputformat: any, outputformat: any): any;
+declare function check_extracted_reference(reference: any, options: any): string;
+declare function check_date_input(date_object: any, default_date: any): any;
+declare function get_min_date(min_date: any): any;
+declare function get_max_date(max_date: any): any;
+export { is_valid_date, is_valid_format, plausible_year_filter, compare_values, filter_ymd_candidate, convert_date, check_extracted_reference, check_date_input, get_min_date, get_max_date };

package/dist/html-to-cite/extract-date/extract-date-quick.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Extract date from document using various methods
+ *
+ * @param {Document} document - DOM object with article content
+ * @param {string} url - URL of the page
+ * @returns {string|null} Extracted date or null if not found
+ */
+export declare function extractDateQuick(document: any, url: any): any;

package/dist/html-to-cite/extract-date/extract-date.d.ts

@@ -0,0 +1,26 @@
+declare const DATE_ATTRIBUTES: Set<string>;
+declare const NAME_MODIFIED: Set<string>;
+declare const PROPERTY_MODIFIED: Set<string>;
+declare const ITEMPROP_ATTRS_ORIGINAL: Set<string>;
+declare const ITEMPROP_ATTRS_MODIFIED: Set<string>;
+declare const ITEMPROP_ATTRS: Set<string>;
+declare const CLASS_ATTRS: Set<string>;
+declare const NON_DIGITS_REGEX: RegExp;
+export { DATE_ATTRIBUTES, NAME_MODIFIED, PROPERTY_MODIFIED, ITEMPROP_ATTRS_ORIGINAL, ITEMPROP_ATTRS_MODIFIED, ITEMPROP_ATTRS, CLASS_ATTRS, NON_DIGITS_REGEX, };
+/**
+ * Extract date from document using various methods
+ *
+ * @param {Document} htmlobject - DOM object with article content
+ * @param {boolean} [extensive_search=true] - perform extensive search if true
+ * @param {boolean} [original_date=false] - return original date if true
+ * @param {string} [outputformat="%Y-%m-%d"] - output format
+ * @param {string} [url=null] - URL of the page
+ * @param {boolean} [verbose=false] - log debug messages if true
+ * @param {Date} [min_date=null] - minimum date to consider
+ * @param {Date} [max_date=null] - maximum date to consider
+ * @param {boolean} [deferred_url_extractor=false] - if true, do not extract date from URL
+ * @returns {string|null} Extracted date or null if not found
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * Based on [Barbaresi (2020)](https://github.com/adbar/htmldate/)
+ */
+export declare function extractDate(htmlobject: any, extensive_search?: boolean, original_date?: boolean, outputformat?: string, url?: any, verbose?: boolean, min_date?: any, max_date?: any, deferred_url_extractor?: boolean): any;

package/dist/html-to-cite/extract-source.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Extract source from document using common class names
+ *
+ * @param {document} document document or dom object with article content
+ * @returns {object} source
+*/
+export declare function extractSource(document: any): any;

package/dist/html-to-cite/extract-title.d.ts

@@ -0,0 +1,11 @@
+/**
+ * @fileoverview Utility for identifying, extracting, and normalizing document titles from HTML.
+ * Handles metadata, selectors, and breadcrumb cleaning.
+ */
+/**
+ * Extract and clean title from document
+ *
+ * @param {Document} document - DOM object with article content
+ * @returns {string} Extracted and cleaned title
+ */
+export declare function extractTitle(document: any): string;

package/dist/html-to-cite/human-names-recognize.d.ts

@@ -0,0 +1,16 @@
+export interface ExtractHumanNameOptions {
+    formatCiteShortenAuthor?: boolean;
+    maxAuthorsBeforeEtAl?: number;
+}
+/**
+ * Validates and formats author names properly handling multiple authors and multi-word names
+ *
+ * @param {string} author - The author name string(s) to be processed
+ * @param {ExtractHumanNameOptions} [options={}] - Configuration options
+ * @returns {object} Formatted author information for citation
+ */
+export declare function extractHumanName(author: string, options?: ExtractHumanNameOptions): {
+    author_cite: any;
+    author_short: any;
+    author_type: number;
+};

package/dist/html-to-cite/metadata-to-cite.d.ts

@@ -0,0 +1,12 @@
+export interface CiteMetadata {
+    author?: string;
+    date?: string;
+    title?: string;
+    source?: string;
+}
+/**
+ * Extract cite info from common property names in webpage's metadata
+ * @param {Document} doc dom object of document
+ * @returns {CiteMetadata} author, date, title, source
+ */
+export declare function extractCiteFromMetadata(doc: Document): CiteMetadata;

package/dist/html-to-cite/url-to-domain.d.ts

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview Utility for extracting and normalizing domain names from URLs.
+ * Handles subdomains and TLD cleaning for source attribution.
+ */
+/**
+ * Extract TLD and hostname from domain in Regex. There's [two or more part
+ * TLDs](https://en.wikipedia.org/wiki/List_of_Internet_top-level_domains)
+ * so it is hard to tell if host.secondTLD.tld or host.tld is correct way
+ * to get root domain (e.g. abc.go.jp, abc.co.uk)
+ * @param {string} domain
+ * @returns {string} rootDomain
+ */
+export declare function convertURLToDomain(domain: any): any;
+/**
+ * Checks if a string is a valid URL.
+ * @param {string} string
+ * @returns {boolean} true if the string is a valid URL
+ * @private
+ */
+export declare function isURLValid(string: any): boolean;

package/dist/html-to-content/extract-content/extract-content-mercury-utils.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @module research/extractor/html-to-content/extract-content/extract-content-mercury-utils
+ * @description Research library module.
+ */
+declare function normalizeSpaces(text: any): any;
+declare function paragraphize(node: any, document: any, br?: boolean): any;
+declare function getAttrs(node: any): unknown;
+declare function convertNodeTo(node: any, document: any, tag?: string): any;
+declare function brsToPs(document: any): any;
+declare function convertToParagraphs(document: any): any;
+declare function cleanImages(article: any, document: any): any;
+declare function stripJunkTags(article: any, document: any, tags?: any[]): any;
+declare function cleanHOnes(article: any, document: any): any;
+declare function cleanAttributes(article: any, document: any): any;
+declare function removeEmpty(article: any): any;
+declare function removeUnlessContent(node: any, weight: any): void;
+declare function rewriteTopLevel(article: any, document: any): any;
+declare function textLength(text: any): any;
+declare function linkDensity(node: any): number;
+declare function stripTags(text: any, document: any): any;
+declare function stripUnlikelyCandidates(document: any): any;
+declare function withinComment(node: any): boolean;
+declare function nodeIsSufficient(node: any): boolean;
+declare function isWordpress(document: any): boolean;
+declare function setAttr(node: any, attr: any, val: any): any;
+declare function setAttrs(node: any, attrs: any): any;
+export { normalizeSpaces, paragraphize, getAttrs, convertNodeTo, brsToPs, convertToParagraphs, cleanImages, stripJunkTags, cleanHOnes, cleanAttributes, removeEmpty, rewriteTopLevel, textLength, linkDensity, stripTags, stripUnlikelyCandidates, withinComment, nodeIsSufficient, isWordpress, setAttr, removeUnlessContent, setAttrs, };

package/dist/html-to-content/extract-content/extract-content-mercury.d.ts

@@ -0,0 +1,61 @@
+/**
+ *  ### HTML-to-Main-Content Extractor #2
+ *
+ * 1. The algorithm starts by loading the HTML content using linkedom, a lightweight DOM parser for Node.js.
+ * 2. It then applies a series of cleaning and scoring techniques to identify the main content of
+ * the page, starting with stripping unlikely candidates (e.g., elements with class names like "comment"
+ *  or "sidebar").
+ * 3. The HTML is converted into a series of paragraph elements, which are then scored based on various
+ *  factors such as text length, number of commas, and the presence of certain class names or IDs.
+ * 4. The algorithm assigns scores to parent and grandparent elements based on the scores of their
+ * children, with parents receiving the full score and grandparents receiving half.
+ * 5. After scoring, the algorithm finds the top candidate element by selecting the node with the
+ * highest score.
+ * 6. The top candidate's siblings are then examined to see if they should be included in the main
+ * content, based on their scores and other factors like link density.
+ * 7. The algorithm then cleans the selected content by removing unnecessary tags, attributes, and empty
+ * elements.
+ * 8. It also handles special cases like cleaning up header tags, images, and other potentially irrelevant
+ *  content.
+ * 9. Throughout the process, the algorithm uses various regular expressions and scoring heuristics to
+ * identify positive and negative indicators of content relevance.
+ * 10. Finally, the cleaned and extracted content is returned as an HTML string, representing the main
+ * body of the article or webpage.
+ *
+ * [Article Extraction Benchmark](https://trafilatura.readthedocs.io/en/latest/evaluation.html)
+ *
+ * @param {string} html - The HTML content to extract from.
+ * @param {Object} [opts] - The options for content extraction.
+ * @param {boolean} opts.stripUnlikelyCandidates default=true - Remove elements that match non-article-
+ * like criteria first (e.g., elements with a classname of "comment").
+ * @param {boolean} opts.weightNodes default=true - Modify an element's score based on certain classNames or
+ * IDs (e.g., subtract if a node has a className of 'comment', add if a node has an ID of 'entry-content').
+ * @param {boolean} opts.cleanConditionally default=true - Clean the node to remove superfluous content
+ *  like forms, ads, etc. Initially, pass in the most restrictive options which will return the highest
+ * quality content. On each failure, retry with slightly more lax options.
+ * @returns {string} The extracted content as an HTML string, or null if extraction fails.
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * Based on [Postlight Mercury Parser (2017-)](https://github.com/postlight/parser/tree/main/src)
+ * @example var url =  "https://en.wikipedia.org/wiki/David_Hilbert"
+ * var html = await (await fetch(url)).text();
+ * var content = extractMainContentFromHTML(html);
+ * console.log(content); // HTML content of main article body
+ * @category Extract
+ */
+export declare function extractMainContentFromHTML2(html: any, opts: any): any;
+/**
+ * Sets the score attribute of a node.
+ * @param {Node} node - The node to set the score on.
+ * @param {Document} document - The document object.
+ * @param {number} score - The score to set.
+ * @returns {Node} The node with the set score.
+ * @private
+ */
+export declare function setScore(node: any, document: any, score: any): any;
+/**
+ * Scores a paragraph node.
+ * @param {Node} node - The paragraph node to score.
+ * @private
+ * @returns {number} The score of the paragraph.
+ */
+export declare function scoreParagraph(node: any): number;

package/dist/html-to-content/extract-content/extract-content-readability.d.ts

@@ -0,0 +1,101 @@
+interface Candidate {
+    score: number;
+    elem: any;
+}
+/**
+ * ### HTML-to-Main-Content Extractor #1
+ * The function extracts main content with regex patterns, cleaning HTML, scoring nodes
+ *  based on content indicators like paragraphs and id/class names, selecting
+ *  the top candidate, extracting it, and cleaning up content around it.
+ *
+ *
+ * 1. Define regular expressions:
+ *    - Various regex patterns are defined to identify content and non-content areas.
+ *
+ * 2. Define helper functions:
+ *    - normalizeSpaces: Normalizes whitespace in a string.
+ *    - stripTags: Removes all HTML tags from a string.
+ *    - getTextLength: Calculates the length of text after stripping tags.
+ *    - calculateLinkDensity: Calculates the ratio of link text to total text.
+ *
+ * 3. Clean HTML:
+ *    - Remove unlikely candidates (e.g., ads, sidebars) from the HTML.
+ *
+ * 4. Define scoring function:
+ *    - scoreNode: Assigns a score to an HTML node based on content and attributes.
+ *    - Increases score for positive indicators (e.g., article, body, content tags).
+ *    - Decreases score for negative indicators (e.g., hidden, footer, sidebar tags).
+ *    - Adds to score based on paragraph tags and text length.
+ *
+ * 5. Find and score candidate nodes:
+ *    - Identify potential content nodes in the cleaned HTML.
+ *    - Score each node using the scoreNode function.
+ *
+ * 6. Select top candidate:
+ *    - Sort candidates by score and select the highest-scoring node.
+ *
+ * 7. Extract content:
+ *    - Use regex to extract content around the top candidate node.
+ *
+ * 8. Clean up extracted content:
+ *    - Remove script and style tags and their contents.
+ *    - Process anchor tags based on content density.
+ *    - Keep only specific HTML tags (a, p, img, h1-h6, ul, ol, li).
+ *    - Remove excess whitespace from the final content.
+ *
+ * [Article Extraction Benchmark](https://trafilatura.readthedocs.io/en/latest/evaluation.html)
+ *
+ * @example
+ * var url = "https://www.nytimes.com/2024/08/28/business/telegram-ceo-pavel-durov-charged.html"
+ * const html = await (await fetch(url)).text();
+ * var articleContent = extractMainContentFromHTML(html);
+ * @param {Object} [options]
+ * @param {number} options.minContentLength default=140 - Minimum length of content to be considered valid
+ * @param {number} options.minScore default=20 - Minimum score for content to be considered valid
+ * @param {number} options.minTextLength default=25 - Minimum length of text to be considered valid
+ * @param {number} options.retryLength default=250 - Length to retry content extraction if initial attempt fails
+ * @returns {string} Extracted HTML string of main content
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * Based on [Mozilla Readability (2015)](https://github.com/mozilla/readability)
+ * @category Extract
+ */
+export declare function extractMainContentFromHTML(html: string, options?: {
+    minContentLength?: number;
+    minScore?: number;
+    minTextLength?: number;
+    retryLength?: number;
+}): string;
+/**
+ * Calculates the link density of an element.
+ * @param {Element} elem - The element to calculate link density for
+ * @returns {number} The link density (ratio of link text length to total text length)
+ */
+export declare function getLinkDensity(elem: any): number;
+/**
+ * Calculates the weight of an element based on its class and id attributes.
+ * @param {Element} elem - The element to calculate weight for
+ * @param {RegExp} positiveRe - Regular expression for positive indicators
+ * @param {RegExp} negativeRe - Regular expression for negative indicators
+ * @returns {number} The calculated weight
+ */
+export declare function classWeight(elem: any, positiveRe: RegExp, negativeRe: RegExp): number;
+/**
+ * Scores a node based on its tag name and attributes.
+ * @param {Element} elem - The element to score
+ * @param {RegExp} positiveRe - Regular expression for positive indicators
+ * @param {RegExp} negativeRe - Regular expression for negative indicators
+ * @returns {Object} An object containing the score and the element
+ */
+export declare function scoreNode(elem: any, positiveRe: RegExp, negativeRe: RegExp): Candidate;
+/**
+ * Sanitizes the content by removing unwanted elements and cleaning remaining elements.
+ * @param {Element} node - The node to sanitize
+ * @param {Object} candidates - Object containing scored candidates
+ * @param {RegExp} videoRe - Regular expression for video URLs
+ * @param {RegExp} positiveRe - Regular expression for positive indicators
+ * @param {RegExp} negativeRe - Regular expression for negative indicators
+ * @param {number} minTextLength - Minimum text length to consider
+ * @returns {Element} The sanitized node
+ */
+export declare function sanitize(node: any, candidates: Record<string, Candidate>, videoRe: RegExp, positiveRe: RegExp, negativeRe: RegExp, minTextLength: number): any;
+export {};

package/dist/html-to-content/html-to-basic-html.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Strip HTML to ~30 basic markup HTML tags, lists, tables, images.
+ * Convert anchors and relative urls to absolute urls. Basic HTML supports the same
+ * elements as Markdown, which is used in writing plain text. Markdown is converted
+ * to HTML anyways to display it, and it is better to edit basic HTML in a rich text editor.
+ *
+ * [Mozilla DOM Reference](https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model) <br />
+ * [Source Code of Browser HTML DOM](https://chromium.googlesource.com/chromium/src/+/HEAD/third_party/blink/renderer/core/dom/) <br />
+ * [RegExp JS V8 Code](https://github.com/v8/v8/blob/94cde7c7f3fffc62f621e43f65be3d517b8a9f3d/src/regexp/regexp-compiler.cc#L3827)
+ * @param {string} html Any page's HTML to process
+ * @param {Object} [options]
+ * @param {boolean} options.images default=true - Whether to include images
+ * @param {boolean} options.links default=true - Whether to include links
+ * @param {boolean} options.videos default=true - Whether to include videos or not
+ * @param {boolean} options.formatting default=true - Whether to include formatting
+ * @param {string} options.url  base URL for converting relative URLs to absolute
+ * @param {string} options.allowTags default="br,p,u,b,i ,em,strong,h1,h2,h3,h4, h5,h6,blockquote,
+ * code,ul,ol,li,dd,dl, table,th,tr,td,sub,sup" - Comma-separated list of allowed HTML tags.
+ * @param {string} options.allowedAttributes default="text,tag,href, src,type,width, height,id,data"
+ *   List of allowed HTML attributes
+ * @returns {string} basic text formatting html
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @category HTML Utilities
+ */
+export declare function convertHTMLToBasicHTML(html: any, options?: {}): any[];
+/**
+ * Convert html string to array of JSON Objects tokens to translate,
+ * convert, or filter all elements.
+ * Flat array is faster than DOMParser which uses nested trees.
+ * @param {string} html
+ * @returns {array}  Example [{"tag": "img","src": ""}, ...]
+ 
+ * @private
+ */
+export declare function convertHTMLToTokens(html: any): any[];
+export declare function addDOMFunctions(domObject: any): any;

package/dist/html-to-content/html-to-content.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Extracts the main content and citation information from a document or HTML string
+ * @param {string|object} documentOrHTML - The document or HTML string to extract content from
+ * @param {Object} options - Optional configuration options
+ * @param {boolean} options.images default=true - Whether to include images in the extracted content
+ * @param {boolean} options.links default=true - Whether to include links in the extracted content
+ * @param {boolean} options.formatting default=true - Whether to preserve formatting in the extracted content
+ * @param {string} options.url The URL of the original document, if available, for absolutify-ing URLs
+ * @param {boolean} options.useExtractor2 default=false -
+ *    false uses Mozilla Readability, true uses Postlight Mercury.
+ *    then use the alternate if the first returns less than 200 characters
+ * @returns {Object} The extracted content and citation information
+ * @property {string} title - The title of the document
+ * @property {string} author_cite - The full citation for the author
+ * @property {string} author_short - A shortened version of the author's name
+ * @property {string} author - The author's name
+ * @property {string} date - The publication date
+ * @property {string} source - The source of the document
+ * @property {string} html - The extracted HTML content
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function extractContentAndCite(documentOrHTML: any, options?: {}): {
+    error: string;
+    title?: undefined;
+    author_cite?: undefined;
+    author_short?: undefined;
+    author?: undefined;
+    date?: undefined;
+    source?: undefined;
+    html?: undefined;
+} | {
+    title: string;
+    author_cite: any;
+    author_short: any;
+    author: string;
+    date: string;
+    source: string;
+    html: any;
+    error?: undefined;
+};
+/**
+ * @typedef {Object} ExtractedContent
+ * @property {string} title - The title of the content
+ * @property {string} author_cite - The full citation for the author
+ * @property {string} author_short - A shortened version of the author's name
+ * @property {string} author - The author's name
+ * @property {string} date - The publication date
+ * @property {string} source - The source of the content
+ * @property {string} html - The extracted main content in HTML format
+ * @private
+ */

package/dist/html-to-content/html-utils.d.ts

@@ -0,0 +1,76 @@
+/**
+ * @module research/extractor/html-to-content/html-utils
+ * @description Research library module.
+ */
+/**
+ * Converts URL-safe escaped HTML codes like &"'`’ & to standard HTML or in reverse.
+ * @param {string} str - The string to process.
+ * @param {boolean} toStandardHTML  default=true - If true, converts url-safe codes
+ * to standard HTML. If false, converts standard HTML to url-safe codes.
+ * @return {string} The processed string.
+ * @category HTML Utilities
+ * @example
+ * var normalHTML = convertURLSafeHTMLToHTML('<p>This & that © 2023 '+
+ * '"Quotes"'Apostrophes' €100 ☺</p>', true)
+ * console.log(normalHTML) // "<p>This & that \u00a9 2023 "Quotes" 'Apostrophes' \u20ac100 \u263a</p>"
+ */
+export declare function convertURLSafeHTMLToHTML(str: any, toStandardHTML?: boolean): any;
+/**
+ * Convert relative URL to absolute URL using base URL.
+ * @param {string} base base url of the domain
+ * @param {string} relative partial urls like ../images/image.jpg #hash
+ * @returns {string} absolute URL
+ * @example
+ * var absoluteURL = convertURLToAbsoluteURL('https://example.com', 'images/image.jpg')
+ * console.log(absoluteURL) // Returns: "https://example.com/images/image.jpg"
+ * var absoluteURL = convertURLToAbsoluteURL('https://example.com', '//images/image.jpg')
+ * console.log(absoluteURL) // Returns: "https:images/image.jpg"
+ * @category HTML Utilities
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function convertURLToAbsoluteURL(base: any, relative: any): any;
+/**
+ * Converts Markdown text to HTML. It handles the following Markdown elements:
+ * - Headers (h1 to h6)
+ * - Bold text
+ * - Italic text
+ * - Unordered lists
+ * - Ordered lists
+ * - Paragraphs
+ * - Images
+ * - Links
+ * - Code blocks
+ * @param {string} content - The Markdown or HTML content to be converted.
+ * @param {boolean} toHtml - default=true - If true, converts Markdown to HTML.
+ *                          If false, converts HTML to Markdown.
+ * @returns {string} The resulting HTML string.
+ * @category HTML Utilities
+ * @example
+ * const markdown = "# Header\n\nThis is **bold** and *italic* text.\n\n* List item 1\n* List item 2";
+ * const html = convertMarkdownToHTML(markdown);
+ * console.log(html);
+ * // Output:
+ * // <h1>Header</h1>
+ * // <p>This is <strong>bold</strong> and <em>italic</em> text.</p>
+ * // <ul><li>List item 1</li><li>List item 2</li></ul>
+ */
+export declare function convertMarkdownToHTML(content: any, toHtml?: boolean): any;
+export declare function convertHTMLToMarkdown(html: any): any;
+/**
+ * Copy HTML to clipboard. When pasting into rich text field,
+ * pastes rich text. When pasting into plain text field, pastes:
+ * plain text, html, or markdown.
+ *
+ * @param {string} html - The HTML content to be copied.
+ * @param {object} options - The options object.
+ * @param {number} options.pastePlainFormat -
+ * default=0
+ * 0 - plain text
+ * 1 - markdown
+ * 2 - html
+ * @returns {Promise<void>} - A promise that resolves when
+ * the HTML is copied to the clipboard.
+ * @category HTML Utilities
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function copyHTMLToClipboard(html: any, options?: {}): Promise<void>;

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @fileoverview Research Agent Library entry point.
+ * Exports various specialized agents, tools, and utilities for AI-driven research.
+ *
+ * @author vtempest <grokthiscontact@gmail.com>
+ * @license AGPL-3.0 Organizations should email grokthiscontact@gmail.com
+ * to get a dual-use commercial license to remove the GPL requirements.
+ */
+export * from './search/search-web';
+export * from './search';
+export * from './tokenize/word-to-root-stem';
+export * from './tokenize/suggest-complete-word';
+export * from './tokenize/text-to-topic-tokens';
+export * from './tokenize/text-to-sentences';
+export * from './tokenize/text-to-chunks';
+export * from './url-to-content/url-to-content';
+export * from './url-to-content/url-to-html';
+export * from './html-to-cite/url-to-domain';
+export * from './url-to-content/youtube-to-text';
+export * from './url-to-content/docx-to-content';
+export * from './html-to-content/html-to-content';
+export * from './html-to-content/extract-content/extract-content-readability';
+export * from './html-to-content/extract-content/extract-content-mercury';
+export * from './html-to-content/html-to-basic-html';
+export * from './html-to-cite/extract-cite';
+export * from './html-to-content/html-utils';

package/dist/search/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * @module extract-webpage/search
+ * @description Re-exports MetaSearchAgent from agent-toolkit with search functions
+ */
+export declare const searchHandlers: {
+    webSearch: import('chat-agent-toolkit').MetaSearchAgent;
+    academicSearch: import('chat-agent-toolkit').MetaSearchAgent;
+    writingAssistant: import('chat-agent-toolkit').MetaSearchAgent;
+    wolframAlphaSearch: import('chat-agent-toolkit').MetaSearchAgent;
+    youtubeSearch: import('chat-agent-toolkit').MetaSearchAgent;
+    redditSearch: import('chat-agent-toolkit').MetaSearchAgent;
+};
+export { MetaSearchAgent, generateSuggestions, groupAndSummarizeDocs, buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, splitTextIntoChunks, LineOutputParser, LineListOutputParser, formatChatHistoryAsString, } from 'chat-agent-toolkit';
+export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample, Document, } from 'chat-agent-toolkit';

package/dist/search/meta-search-agent-reexport.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @fileoverview Re-exports MetaSearchAgent from chat-agent-toolkit
+ * @deprecated Import from 'chat-agent-toolkit' instead
+ */
+export { MetaSearchAgent as default, searchHandlers, generateSuggestions, groupAndSummarizeDocs, buildFallbackDocs, rerankDocs, processDocs, normalizeSourcesOutput, splitTextIntoChunks, } from 'chat-agent-toolkit';
+export type { MetaSearchAgentType, Config, ChatTurnMessage, SearchingEvent, FewShotExample, Document, } from 'chat-agent-toolkit';
+export { LineOutputParser, LineListOutputParser, formatChatHistoryAsString, } from 'chat-agent-toolkit';
+export { webSearchResponsePrompt, webSearchRetrieverPrompt, webSearchRetrieverFewShots, writingAssistantPrompt, } from 'chat-agent-toolkit';