extract-webpage 1.2.5

package/dist/tokenize/text-to-chunks.d.ts

@@ -0,0 +1,48 @@
+/**
+ * @fileoverview Utility for splitting text into semantic chunks for RAG and NLP tasks.
+ * Uses complex regex patterns to identify structural elements like lists, tables, and code.
+ *
+ * ### Split Text by Semantic Characters
+ * <img width="350px"  src="https://i.imgur.com/RpXf5as.png" />
+ *
+ *
+ * Splits document text into semantic chunks based on various textual and structural
+    * elements like HTML, markdown, and paragraphs.
+ *
+ * This function performs a comprehensive tokenization of the input text, considering a wide range
+    * of semantic elements and structural patterns commonly found in documents.It uses regular
+        * expressions to identify and separate the following elements:
+ *
+ * 1. Headings(Setext - style, Markdown, and HTML - style)
+    * 2. Citations(e.g., [1])
+        * 3. List items(bulleted, numbered, lettered, or task lists, including nested up to three levels)
+            * 4. Block quotes(including nested quotes and citations, up to three levels)
+                * 5. Code blocks(fenced, indented, or HTML pre / code tags)
+                    * 6. Tables(Markdown, grid tables, and HTML tables)
+                        * 7. Horizontal rules(Markdown and HTML hr tag)
+                            * 8. Standalone lines or phrases(including single - line blocks and HTML elements)
+                                * 9. Sentences or phrases ending with punctuation(including ellipsis and Unicode punctuation)
+                                * 10. Quoted text, parenthetical phrases, or bracketed content
+                                    * 11. Paragraphs
+                                        * 12. HTML - like tags and their content(including self - closing tags and attributes)
+                                            * 13. LaTeX - style math expressions(inline and block)
+                                                * 14. Any remaining content(fallback)
+                                                    *
+ * The function applies various length constraints to each type of element to ensure reasonable
+    * chunk sizes.It also handles nested structures and special cases like code blocks and math
+        * expressions.
+ *
+ * [Sentence RAG Benchmarks](https://superlinked.com/vectorhub/articles/evaluation-rag-retrieval-chunking-methods)
+ *
+ * @author[Jina AI(2024)](https://gist.github.com/hanxiao/3f60354cf6dc5ac698bc9154163b4e6a)
+            * @param { string } text - The input text to be split into semantic chunks.
+ * @param { Object }[options = {}] - Optional configuration options(currently unused).
+ * @returns { Array.<string> } An array of text chunks, each representing a semantic unit of the document.
+ * @category Topics
+        * @example
+        * const text = "# Heading\n\nThis is a paragraph.\n\n- List item 1\n- List item 2\n\n";
+ * const chunks = splitTextSemanticChars(text);
+ * console.log(chunks);
+ * // Output: ['# Heading', 'This is a paragraph.', '- List item 1', '- List item 2']
+ */
+export declare function splitTextSemanticChars(text: any, options?: {}): unknown[];

package/dist/tokenize/text-to-sentences.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @fileoverview
+ * Splits text into sentences, handling 220+ common abbreviations,
+ * and inferring acronyms, numbers, URLs, times, names, etc.
+ *
+ * @param inputText - The text to be split into sentences.
+ * @param options - Configuration options for sentence splitting.
+ * @returns An array of sentences.
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @license MIT
+ * @example
+ * ```ts
+ * const text = "Dr. Smith went to the U.S. He met Mr. Jones.";
+ * const sentences = splitTextToSentences(text);
+ * ["Dr. Smith went to the U.S.", "He met Mr. Jones."]
+ * ```
+ */
+export declare function splitTextToSentences(inputText: string, options?: SplitSentencesOptions): string[];
+export type SplitSentencesOptions = {
+    /**
+     * Split on HTML tags like P, DIV, UL, OL.
+     * @default true
+     */
+    splitOnHtmlTags?: boolean;
+    /**
+     * Minimum size for a sentence.
+     * @default 20
+     */
+    minSize?: number;
+    /**
+     * Maximum size for a sentence.
+     * @default 500
+     */
+    maxSize?: number;
+};

package/dist/tokenize/text-to-topic-tokens.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Topic token tuple shape used by downstream ranking logic:
+ * [term, termCategory, uniqueness, metadata]
+ */
+export type TopicToken = [string, number, number, string];
+/**
+ * Trie structure for phrase completion lookup keyed by first two letters, then full token.
+ */
+type PhraseEntry = [string | null, number, number];
+export type PhrasesModel = Record<string, Record<string, PhraseEntry[]>>;
+export interface ConvertTextToTokensOptions {
+    phrasesModel: PhrasesModel;
+    typosModel?: Record<string, string>;
+    checkTypos?: 0 | 1;
+    ignoreStopWords?: 0 | 1;
+    checkRootWords?: 0 | 1;
+}
+/**
+ * @typedef {Object} Token
+ * @property {number} termCategory - The category of the term
+ * @property {number} uniqueness - The uniqueness score of the term
+ * @property {string} term - The actual term or phrase
+ */
+/**
+ * ### Convert Text Query to Topic Phrase Tokens
+ * <img width="350px"  src="https://i.imgur.com/NDrmSRQ.png" />
+ *
+ * Returns a list of phrases that are found in Wiki Titles/ dictionary phrases World Model
+ * that match the input phrase, or just the single word if found. Search results will be
+ *  more accurate if we infer likely phrases and search for those words occuring together and
+ *  not just split into words and find frequency. Examples are "white house" or "state of the art"
+ *  which should be searched as a phrase but would return different context if split into words.
+ *  As Led Zeppelin famously put it: \u266b "'Cause you know sometimes words have two meanings."
+ *
+ * @param {string} phrase
+ * @param {Object} [options]
+ * @param {Object} options.phrasesModel - remote model
+ * @param {Object} options.typosModel - remote model
+ * @param {number} options.checkTypos - check for typos
+ * @param {number} options.ignoreStopWords - ignore 300+ overused words
+ * @param {number} options.checkRootWords - check for word's root stem
+ * @returns {Array<{termCategory: number, uniqueness: number, term: string}>}
+ * @example
+ *   const result = convertTextToTokens("The president of the united states is in the white house", { phrasesModel, typosModel });
+ *   console.log(result);
+ *
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @category Topics
+ */
+export declare function convertTextToTokens(phrase: string, options?: Partial<ConvertTextToTokensOptions>): TopicToken[];
+export {};

package/dist/tokenize/word-is-ignored.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @fileoverview Utility for identifying common stop words (e.g., "the", "and", "is").
+ * Based on the SpaCy English stop word list.
+ */
+/**
+ * Checks word is in [320 commonly ignored "stop words
+ * "](https://raw.githubusercontent.com/igorbrigadir/stopwords/master/en/spacy.txt)
+ * in queries, using efficient JS Set method
+ * @param {string} word
+ * @returns {Boolean}
+*/
+export declare function isWordCommonIgnored(word: any): boolean;

package/dist/tokenize/word-to-root-stem.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @fileoverview Implementation of the Porter Stemmer algorithm for word normalization.
+ * Used to reduce words to their root form (e.g., "running" to "run").
+ */
+/**
+ * Stems a word using the [Porter
+ * Stemmer](https://snowballstem.org/algorithms/porter/stemmer.html)
+ * for removing inflectional endings like "ing", "ist", "ize".
+ *
+ * @author [Porter, M. (1980)](https://tartarus.org/martin/PorterStemmer/)
+ * @param word - The word to be stemmed
+ * @returns The stemmed word
+ * @example const rootWord = stemWordToRoot("running"); // returns "run"
+ * @category Topics
+ */
+export declare function stemWordToRoot(word: string): string;

package/dist/url-to-content/docx-to-content.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Converts a DOCX document to HTML
+ *
+ * @param {string|File|Blob|ArrayBuffer|Buffer|Uint8Array} input - DOCX input to convert
+ * @param {DocxOptions} [options] - Conversion options
+ * @returns {Promise<string>} The converted HTML
+ * @throws {Error} If conversion fails
+ * @category Extract
+ * @example
+ * const html = await convertDOCXToHTML('https://example.com/doc.docx');
+ * const html = await convertDOCXToHTML(fileInput.files[0]);
+ */
+export declare function convertDOCXToHTML(input: any, options?: {}): Promise<string>;
+/**
+ * Detects if a binary buffer is a DOCX file by checking the file signature
+ * DOCX files are ZIP archives with specific internal structure
+ *
+ * @param {ArrayBuffer|Buffer|Uint8Array} buffer - Binary buffer to check
+ * @returns {boolean} True if buffer appears to be a DOCX file
+ * @category Extract
+ */
+export declare function isBufferDOCX(buffer: any): boolean;

package/dist/url-to-content/is-url-adult.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @module research/extractor/url-to-content/is-url-porn
+ * @description Research library module.
+ */
+export interface IsURLPornOptions {
+    url?: string;
+    title?: string;
+    threshold?: number;
+}
+/**
+ * Determines if content is likely adult/porn based on configurable threshold
+ *
+ * @param {Object} options - Configuration object
+ * @param {string} [options.url] - URL to analyze (optional)
+ * @param {string} [options.title] - Page title to analyze (optional)
+ * @param {number} [options.threshold] - Probability threshold (0.5 default)
+ * @returns {boolean} True if likelihood exceeds threshold, false otherwise
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @example
+ * isURLPorn({
+ *   title: "Hot deals on sexy cars",
+ *   threshold: 0.8
+ * });
+ * console.log(isPorn2); // false (low confidence)
+ */
+export declare function isURLPorn(options?: IsURLPornOptions): boolean;

package/dist/url-to-content/url-to-content.d.ts

@@ -0,0 +1,127 @@
+export interface ExtractContentOptions {
+    images?: boolean;
+    links?: boolean;
+    formatting?: boolean;
+    absoluteURLs?: boolean;
+    timeout?: number;
+    proxy?: string | null;
+    citeFormatMonthFull?: boolean;
+    citeFormatAuthorFull?: boolean;
+    url?: string;
+    useThirdPartyBackup?: boolean;
+    /** Preferred transcript languages when extracting YouTube videos. */
+    languages?: string[];
+}
+export interface ExtractedArticle {
+    cite?: string;
+    html?: string;
+    url?: string;
+    author?: string;
+    author_cite?: string;
+    author_short?: string;
+    author_type?: number | string;
+    date?: string;
+    title?: string;
+    source?: string;
+    word_count?: number;
+    format?: string;
+    error?: string | number;
+}
+type UrlLikeDocument = {
+    location?: {
+        href?: string;
+    };
+    querySelectorAll?: (selector: string) => {
+        length: number;
+    } | ArrayLike<unknown>;
+};
+/**
+ * @typedef {Object} Article
+ * @property {string} cite - Cite in APA Format with Author name in Last, First Initial format
+ * @property {string} html - The Basic HTML content of the article
+ * @property {string} url - The URL of the article
+ * @property {string} author - The full name of the author of the article
+ * @property {string} author_cite - Author name in Last, First Initial format
+ * @property {string} author_short - Author name in Last format
+ * @property {number} author_type - Author type ["single", "two-author", "more-than-two", "organization"]
+ * @property {string} date - The publication date of the article
+ * @property {string} title - The title of the article
+ * @property {string} source - The source or publisher of the article
+ * @property {number} word_count - The word count of the full text (without HTML tags)
+ * @category Extract
+ */
+/**
+ * ### 🚜 Tractor the Text Extractor
+ * <img width="350px"  src="https://i.imgur.com/o8NTXxY.png" />
+ *
+ * 1. Main Content Detection: Extract the main content from a URL by combining
+ * Mozilla Readability and Postlight Mercury algorithms, utilizing over 100
+ * custom adapters for major sites for article, author, date HTML classes.
+ * 2. Basic HTML Standardization: Transform complex HTML into a simplified
+ * reading-mode format of basic HTML, making it ideal for research note archival
+ *  and focused reading, with headings, images and links.
+ * 3. YouTube Transcript Processing: When a YouTube video URL is detected,
+ * retrieve the complete video transcript including both manual captions and
+ * auto-generated subtitles, maintaining proper timestamp synchronization and
+ * speaker identification where available.
+ * 4. PDF to HTML: Process PDF documents by extracting
+ *  formatted text while intelligently handling line breaks, page headers,
+ *  footnotes. The system analyzes text height statistics to automatically
+ *  infer heading levels, creating a properly structured document hierarchy
+ *  based on standard deviation from mean text size.
+ * 5. DOCX Binary Buffer Processing: Accept DOCX files as binary buffers
+ * (ArrayBuffer, Buffer, or Uint8Array) and automatically detect and convert
+ * them to HTML while preserving formatting, styles, and document structure.
+ * 6. Citation Information Extraction: Identify and extract citation metadata
+ *  including author names, publication dates, sources, and titles using HTML
+ *  meta tags and common class name patterns. The system validates author names
+ *  against a comprehensive database of 90,000 first and last names,
+ * distinguishing between personal and organizational authors to properly
+ * format citations.
+ * 7. Author Name Formatting: Process author names by checking against
+ * known name databases, handling affixes and titles correctly, and determining
+ *  whether to reverse the name order based on whether it's a personal or
+ * organizational author, ensuring proper citation formatting.
+ * @param {document|string|ArrayBuffer|Buffer|Uint8Array} urlOrDoc - url, dom object with article content, or binary buffer (DOCX)
+ * @param {Object} [options]
+ * @param {boolean} options.images default=true - include images
+ * @param {boolean} options.links default=true - include links
+ * @param {boolean} options.formatting default=true - preserve formatting
+ * @param {boolean} options.absoluteURLs default=true - convert URLs to absolute
+ * @param {number} options.timeout default=5 - http request timeout
+ * @returns {{
+ *  title: string,
+ *  author_cite: string,
+ *  cite: string,
+ *  author: string,
+ *  date: string,
+ *  source: string,
+ *  html: string,
+ *  word_count: number
+ * }}
+ * cite - Cite in APA Format with Author name in Last, First Initial format
+ * url - The URL of the article
+ *  html - The HTML content of the article
+ *  author - The author of the article
+ *  author_cite - Author name in Last, First Middle format
+ *  author_short - Author name in Last format
+ *  author_type - Author type ["single", "two-author", "more-than-two", "organization"]
+ *  date - The publication date of the article
+ *  title - The title of the article
+ *  source - The source or origin of the article
+ *  word_count - The word count of the full text (without HTML tags)
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * @example
+ * // Extract from URL
+ * const result1 = await extractContent('https://example.com/article');
+ *
+ * // Extract from DOCX binary buffer
+ * const docxBuffer = new Uint8Array([...]); // DOCX file bytes
+ * const result2 = await extractContent(docxBuffer);
+ *
+ * // Extract from DOM object
+ * const result3 = await extractContent(document);
+ */
+export declare function extractContent(urlOrDoc: string | Document | UrlLikeDocument | ArrayBuffer | Buffer | Uint8Array, options?: ExtractContentOptions): Promise<ExtractedArticle>;
+export {};

package/dist/url-to-content/url-to-html.d.ts

@@ -0,0 +1,60 @@
+/**
+ * ### Tardigrade the Web Crawler
+ * <img src="https://i.imgur.com/iuzpcvD.png" width="350px" />
+ *
+ * 1. **Use Fetch API, check for bot detection.** Scrape  any domain's URL to get its HTML, JSON, or arraybuffer.<br />
+ * Scraping internet pages is a [free speech right
+ * ](https://blog.apify.com/is-web-scraping-legal/).
+ * 2. Features: timeout, redirects, default UA, referer as google, and bot
+ * detection checking. <br />
+ * 3. If fetch method does not get needed HTML, use Docker proxy as backup.
+ *
+ * 4. [Setup Docker](https://github.com/vtempest/ai-research-agent/tree/master/src/crawler)
+ *  container with NodeJS server API renders with puppeteer DOM to get all HTML loaded by
+ *  secondary in-page API requests after the initial page request, including user login and cookie storage.
+ * 5. Bypass Cloudflare bot check: A webpage proxy that request through Chromium (puppeteer) - can be used
+ * to bypass Cloudflare anti bot using cookie id javascript method.
+ * 6. Send your request to the server with the port 3000 and add your URL to the "url"
+ *  query string like this: `http://localhost:3000/?url=https://example.org`
+ *
+ * 7. Optional: Setup residential IP proxy to access sites that IP-block datacenters
+ *  and manage rotation with [Scrapoxy](https://scrapoxy.io). Recommended:
+ * [Hypeproxy](https://hypeproxy.io/products/static-residential-proxies)
+ * [NinjasProxy](https://ninjasproxy.com/residential-proxies/)
+ * [Proxy-Cheap](https://app.proxy-cheap.com/order)
+ * [LiveProxies](https://liveproxies.io/rotating-residential-proxies-pricing)
+ *
+ * @param {string} url - any domain's URL
+ * @param {Object} [options]
+ * @param {number} options.timeout default=5 -  abort request if not retrived, in seconds
+ * @param {number} options.maxRedirects default=3 - max redirects to follow
+ * @param {number} options.checkBotDetection default=true - check for bot detection messages
+ * @param {number} options.changeReferer default=true - set referer as google
+ * @param {number} options.userAgentIndex default=0 - index of [google bot, default chrome]
+ * @param {string} options.proxy default=false - use proxy url
+ * @param {boolean} options.checkRobotsAllowed default=false - check robots.txt rules
+ * @returns {Promise<string>} -  HTML, JSON, arraybuffer, or error object
+ * @category Extract
+ * @example await scrapeURL("https://hckrnews.com", {timeout: 5, userAgentIndex: 1})
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function scrapeURL(url: any, options?: {}): Promise<any>;
+/**
+ * As backup, scrape with JINA to get html
+ * @param {string} url
+ * @returns {Promise<string>}
+ */
+export declare function scrapeJINA(url: any): Promise<any>;
+/**
+ * Fetches and parses the robots.txt file for a given URL.
+ * @param {string} url - The base URL to fetch the robots.txt from.
+ * @returns {Promise<Object>} A JSON object representing the parsed robots.txt.
+ */
+export declare function fetchScrapingRules(url: any): Promise<{
+    directives: {};
+    crawlDelay: {};
+    sitemaps: any[];
+    preferredHost: any;
+} | {
+    error: string;
+}>;

package/dist/url-to-content/youtube-helpers.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @fileoverview Adapter helpers bridging the `extract-youtube` transcript API to
+ * the `getURLYoutubeVideo` / `convertYoutubeToText` helpers expected by the URL
+ * content extractor. Keeps the extractor decoupled from the transcript library's
+ * concrete API surface.
+ */
+/**
+ * Extracts the 11-character YouTube video id from a URL, if present.
+ *
+ * @param {string} url - A URL that may point to a YouTube video.
+ * @returns {string | null} The video id, or null when the URL is not a YouTube link.
+ */
+export declare function getURLYoutubeVideo(url: string): string | null;
+/**
+ * Fetches a YouTube video transcript and returns it as a simple HTML document.
+ *
+ * @param {string} url - The YouTube video URL.
+ * @param {{ languages?: string[] }} [options] - Optional transcript languages.
+ * @returns {Promise<Record<string, any>>} An extraction response with `html`, or `{ error }`.
+ */
+export declare function convertYoutubeToText(url: string, options?: {
+    languages?: string[];
+}): Promise<Record<string, any>>;

package/dist/url-to-content/youtube-to-text.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Fetch youtube.com video's webpage HTML for embedded transcript.
+ * If blocked, use scraper of alternative sites providing transcripts.
+ * @param {string} videoUrl
+ * @param {Object} [options]
+ * @param {boolean} options.addTimestamps default=true -
+ * true to return timestamps, default true
+ * @param {boolean} options.timeout default=5 - http request timeout
+ * @return {{content: string, timestamps: string, word_count: number}}
+ * where content is the full text of the transcript,
+ * timestamps is a string of comma-separated [characterIndex, timeSeconds] pairs,
+ * and word_count is the number of words in the transcript.
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ */
+export declare function convertYoutubeToText(videoUrl: any, options?: {}): Promise<{
+    html: any;
+    word_count: any;
+    source: string;
+    date: any;
+    title: any;
+    author_cite: any;
+    length: number;
+}>;
+/**
+ * Test if URL is to youtube video and return video id if true
+ * @param {string} url - youtube video URL
+ * @returns {string|boolean} video ID or false
+ * @private
+ */
+export declare function getURLYoutubeVideo(url: any): any;
+/**
+ * Fetch-based scraper of youtubetotranscript.com
+ * @returns {Object} content, timestamps -  where content is the full text of
+ * the transcript, and timestamps is an array of [characterIndex, timeSeconds]
+ */
+export declare function fetchViaYoutubeToTranscriptCom(videoId: any, options?: {}): Promise<{
+    error: number;
+    content?: undefined;
+    title?: undefined;
+    author_cite?: undefined;
+    timestamps?: undefined;
+} | {
+    content: string;
+    title: any;
+    author_cite: any;
+    timestamps: any[];
+    error?: undefined;
+}>;
+/** ========== NOT WORKING ========== */
+/**
+ * Get YouTube transcript of most YouTube videos,
+ * except if disabled by uploader
+ * fetch-based scraper of youtubetranscript.com
+ *
+ * @param {string} videoUrl
+ * @returns {Object} where content is the full text of
+ * the transcript, and timestamps is an array of [characterIndex, timeSeconds]
+ * @private
+ */
+export declare function fetchViaYoutubeTranscript(videoId: any, options?: {}): Promise<{
+    error: number;
+    content?: undefined;
+    timestamps?: undefined;
+} | {
+    content: string;
+    timestamps: any[];
+    error?: undefined;
+}>;
+export declare function extractYouTubeInfo(videoId: any, options?: {}): Promise<{}>;

package/dist/utils/documents.d.ts

@@ -0,0 +1,4 @@
+import { Document } from 'chat-agent-toolkit';
+export declare const getDocumentsFromLinks: ({ links }: {
+    links: string[];
+}) => Promise<Document<Record<string, any>>[]>;

package/dist/utils/grab.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Fetch wrapper for grabbing binary content
+ * Replacement for grab-url package using standard fetch API
+ */
+export interface GrabOptions {
+    responseType?: "text" | "arraybuffer";
+    /** Timeout in seconds */
+    timeout?: number;
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+}
+export default function grab(url: string, options?: GrabOptions & {
+    responseType?: "text";
+}): Promise<string>;
+export default function grab(url: string, options: GrabOptions & {
+    responseType: "arraybuffer";
+}): Promise<ArrayBuffer>;

package/package.json

@@ -0,0 +1,109 @@
+{
+  "name": "extract-webpage",
+  "version": "1.2.5",
+  "module": "./dist/extract-webpage.es.js",
+  "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
+  "author": "vtempest <grokthiscontact@gmail.com>",
+  "license": "rights.institute/PROSPER",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/OpenSourceAGI/qwksearch-research-agent",
+    "directory": "packages/extract-webpage"
+  },
+  "main": "./dist/extract-webpage.cjs.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/extract-webpage.es.js",
+      "require": "./dist/extract-webpage.cjs.js"
+    },
+    "./search": {
+      "types": "./src/search/index.ts",
+      "react-server": "./src/search/index.ts",
+      "import": "./src/search/index.ts",
+      "require": "./src/search/index.ts"
+    },
+    "./*": {
+      "types": "./src/*.ts",
+      "react-server": "./src/*",
+      "import": "./src/*",
+      "require": "./src/*"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "typesVersions": {
+    "*": {
+      "*": [
+        "src/*"
+      ]
+    }
+  },
+  "scripts": {
+    "build": "vite build",
+    "test": "vitest",
+    "ship": "npm run build && npx standard-version --release-as patch; rm CHANGELOG.md; npm publish",
+    "test-ui": "vitest --ui --watch",
+    "make": "rm -rf dist/*; NODE_OPTIONS=--max-old-space-size=15192 BUN_JSC_forceRAMSize=15192  vite build "
+  },
+  "peerDependencies": {
+    "next": ">=15.0.0"
+  },
+  "peerDependenciesMeta": {
+    "next": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@tsconfig/svelte": "^5.0.8",
+    "@types/node": "^22.0.0",
+    "@vitest/ui": "^4.0.18",
+    "axios": "^1.13.6",
+    "clsx": "^2.1.1",
+    "next": "^16.2.10",
+    "react": "^19.2.4",
+    "react-dom": "^19.2.4",
+    "terser": "^5.46.0",
+    "typedoc-plugin-markdown": "^4.10.0",
+    "typescript": "^5.9.3",
+    "vinext": "1.0.0-beta.0",
+    "vite": "^8.1.3",
+    "vite-plugin-dts": "^5.0.3",
+    "vite-plugin-node-polyfills": "^0.28.0",
+    "vitest": "^4.0.18"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.8.1",
+    "ai": "^5.0.0",
+    "chat-agent-toolkit": "^1.2.3",
+    "chrono-node": "^2.9.0",
+    "drizzle-orm": "^0.45.1",
+    "extract-pdf": "^0.1.1",
+    "extract-youtube": "^1.0.3",
+    "highlight.js": "^11.11.1",
+    "html-entities": "^2.6.0",
+    "js-yaml": "^4.1.1",
+    "jsdom": "^28.1.0",
+    "jszip": "^3.10.1",
+    "linkedom": "^0.18.12",
+    "marked": "^17.0.4",
+    "node-fetch": "^3.3.2",
+    "qwksearch-api-client": "^0.0.12",
+    "tldts": "^7.0.25",
+    "youtube-po-token-generator": "^0.6.0",
+    "zod": "^4.3.6"
+  },
+  "keywords": [
+    "nlp",
+    "autocomplete",
+    "knowledge-graph",
+    "keywords",
+    "search-algorithm",
+    "hacktoberfest",
+    "mind-map",
+    "ai-search"
+  ]
+}

package/src/config/env.ts

@@ -0,0 +1,8 @@
+/**
+ * Runtime env-var accessor.
+ * Uses `process.env` for Next.js. In Cloudflare Workers with vinext,
+ * this would need to be adapted to use cloudflare:workers.
+ */
+export function getEnv(key: string): string | undefined {
+  return process.env[key];
+}