llm-search-tools 1.1.0

package/dist/types.d.ts

@@ -0,0 +1,174 @@
+export interface ProxyConfig {
+    type: "http" | "https" | "socks4" | "socks5";
+    host: string;
+    port: number;
+    auth?: {
+        username: string;
+        password: string;
+    };
+    url?: string;
+}
+export interface ScraperOptions {
+    limit?: number;
+    safeSearch?: boolean;
+    timeout?: number;
+    forcePuppeteer?: boolean;
+    proxy?: ProxyConfig | string;
+    antiBot?: {
+        enabled?: boolean;
+        maxRetries?: number;
+        retryDelay?: number;
+    };
+    searxngInstance?: string;
+    category?: "web" | "news" | "images" | "videos";
+}
+export interface SearchOptions {
+    limit?: number;
+    safeSearch?: boolean;
+    timeout?: number;
+    forcePuppeteer?: boolean;
+    proxy?: ProxyConfig | string;
+    antiBot?: {
+        enabled?: boolean;
+        maxRetries?: number;
+        retryDelay?: number;
+    };
+    searxngInstance?: string;
+}
+export interface SearchResult {
+    title: string;
+    url: string;
+    snippet?: string;
+    source: "google" | "duckduckgo" | "wikipedia" | "hackernews" | "searxng" | "google-news" | "duckduckgo-news" | "google-images" | "duckduckgo-images" | "searxng-images";
+}
+export interface ImageResult extends SearchResult {
+    imageUrl: string;
+    thumbnailUrl?: string;
+    width?: number;
+    height?: number;
+    source: "google-images" | "duckduckgo-images" | "searxng-images";
+}
+export interface NewsResult extends SearchResult {
+    sourceName?: string;
+    publishedAt?: string | Date;
+    imageUrl?: string;
+}
+export interface WikipediaResult extends SearchResult {
+    extract?: string;
+    thumbnail?: string;
+}
+export interface HackerNewsResult extends SearchResult {
+    id?: number;
+    points?: number;
+    author?: string;
+    comments?: number;
+    time?: Date;
+}
+export interface FinanceResult {
+    symbol: string;
+    shortName?: string;
+    longName?: string;
+    regularMarketPrice?: number;
+    regularMarketChange?: number;
+    regularMarketChangePercent?: number;
+    regularMarketTime?: Date;
+    currency?: string;
+    exchange?: string;
+    marketState?: string;
+    source: "yahoo-finance";
+}
+export interface WebpageContent {
+    title?: string;
+    content: string;
+    textContent: string;
+    length: number;
+    excerpt?: string;
+    siteName?: string;
+    favicon?: string;
+    markdown?: string;
+    imageUrls?: string[];
+    rawHtml?: string;
+}
+export interface SearchError {
+    message: string;
+    code: string;
+    originalError?: unknown;
+}
+export interface CrawlOptions extends ScraperOptions {
+    maxPages?: number;
+    maxDepth?: number;
+    crawlType?: "cheerio" | "puppeteer";
+    stayOnDomain?: boolean;
+    ignoreRobotsTxt?: boolean;
+}
+export interface CrawledPage extends WebpageContent {
+    url: string;
+    depth: number;
+}
+export type CrawlResult = CrawledPage[];
+export interface AutocompleteResult {
+    query: string;
+    suggestions: string[];
+    source: string;
+}
+export interface AutocompleteOptions {
+    limit?: number;
+    proxy?: ProxyConfig | string;
+    timeout?: number;
+}
+export interface Flight {
+    airline: string;
+    departureTime: string;
+    arrivalTime: string;
+    duration: string;
+    price: string;
+    stops: string;
+    origin?: string;
+    destination?: string;
+}
+export interface FlightResult {
+    flights: Flight[];
+    url: string;
+    source: "google-flights";
+}
+export interface FlightSearchOptions extends ScraperOptions {
+    departureDate?: string;
+    returnDate?: string;
+    from?: string;
+    to?: string;
+}
+export interface Event {
+    title: string;
+    date: string;
+    location: string;
+    link?: string;
+    description?: string;
+    image?: string;
+}
+export interface EventResult {
+    events: Event[];
+    url: string;
+    source: "google-events";
+}
+export interface EventSearchOptions extends ScraperOptions {
+    date?: "today" | "tomorrow" | "week" | "weekend" | "month" | "next_month";
+}
+export interface MediaResult {
+    title: string;
+    description?: string;
+    rating?: string;
+    releaseDate?: string;
+    cast?: string[];
+    genres?: string[];
+    posterUrl?: string;
+    watchProviders?: {
+        name: string;
+        type: "stream" | "rent" | "buy";
+    }[];
+    url: string;
+    source: "tmdb" | "thetvdb" | "anidb";
+    mediaType: "movie" | "tv" | "anime";
+}
+export interface MediaSearchOptions extends ScraperOptions {
+    type?: "movie" | "tv" | "anime";
+}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+// types.ts - all our shared types n stuff
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,0CAA0C"}

package/dist/utils/createTestDocx.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/createTestDocx.js

@@ -0,0 +1,58 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const jszip_1 = __importDefault(require("jszip"));
+const fs_1 = require("fs");
+async function createTestDocx() {
+    const zip = new jszip_1.default();
+    // Add required DOCX files
+    zip.file('word/document.xml', `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+     <w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+       <w:body>
+         <w:p>
+           <w:r>
+             <w:t>Sample Test Document</w:t>
+           </w:r>
+         </w:p>
+         <w:p>
+           <w:r>
+             <w:t>This is a test DOCX document for llm-kit parser testing.</w:t>
+           </w:r>
+         </w:p>
+         <w:p>
+           <w:r>
+             <w:t>It contains some sample paragraphs to verify DOCX parsing.</w:t>
+           </w:r>
+         </w:p>
+         <w:p>
+           <w:r>
+             <w:t>Each paragraph should be properly extracted and formatted.</w:t>
+           </w:r>
+         </w:p>
+       </w:body>
+     </w:document>`);
+    zip.file('[Content_Types].xml', `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+     <Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
+       <Default Extension="xml" ContentType="application/xml"/>
+       <Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+       <Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
+     </Types>`);
+    zip.file('_rels/.rels', `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+     <Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
+       <Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
+     </Relationships>`);
+    // Generate the DOCX file
+    const content = await zip.generateAsync({
+        type: 'nodebuffer',
+        compression: 'DEFLATE',
+        compressionOptions: {
+            level: 9
+        }
+    });
+    // Write to file
+    (0, fs_1.writeFileSync)('test/files/test.docx', content);
+}
+// Create the test DOCX file
+createTestDocx().catch(console.error);

package/dist/utils/htmlcleaner.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Process HTML to convert relative URLs to absolute ones
+ * @param html The HTML string
+ * @param baseUrl The base URL for resolving relative URLs
+ * @returns HTML with absolute URLs
+ */
+export declare function makeUrlsAbsolute(html: string, baseUrl: string): string;
+/**
+ * Cleans HTML by removing invisible elements and minifies it while preserving visible content
+ * @param html The HTML string to clean
+ * @returns Cleaned and minified HTML with visible content preserved
+ */
+export declare function cleanAndMinifyHtml(html: string): string;
+/**
+ * Process HTML by cleaning it and converting all URLs to absolute
+ * @param html The HTML string to process
+ * @param baseUrl The base URL for resolving relative URLs
+ * @returns Processed HTML
+ */
+export declare function processHtml(html: string, baseUrl: string): string;

package/dist/utils/htmlcleaner.js

@@ -0,0 +1,172 @@
+"use strict";
+// utils/htmlcleaner.ts - utilities for processing and cleaning html
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeUrlsAbsolute = makeUrlsAbsolute;
+exports.cleanAndMinifyHtml = cleanAndMinifyHtml;
+exports.processHtml = processHtml;
+const cheerio = __importStar(require("cheerio"));
+/**
+ * Process HTML to convert relative URLs to absolute ones
+ * @param html The HTML string
+ * @param baseUrl The base URL for resolving relative URLs
+ * @returns HTML with absolute URLs
+ */
+function makeUrlsAbsolute(html, baseUrl) {
+    const $ = cheerio.load(html);
+    // convert relative urls to absolute
+    ["href", "src"].forEach((attr) => {
+        $(`[${attr}]`).each((_, element) => {
+            const el = $(element);
+            const value = $(el).attr(attr);
+            if (value && !value.startsWith("data:") && !value.startsWith("http")) {
+                try {
+                    const absoluteUrl = new URL(value, baseUrl).toString();
+                    $(el).attr(attr, absoluteUrl);
+                }
+                catch {
+                    // keep original if URL parsing fails
+                }
+            }
+        });
+    });
+    return $.html();
+}
+/**
+ * Cleans HTML by removing invisible elements and minifies it while preserving visible content
+ * @param html The HTML string to clean
+ * @returns Cleaned and minified HTML with visible content preserved
+ */
+function cleanAndMinifyHtml(html) {
+    // Load HTML into Cheerio
+    const $ = cheerio.load(html);
+    // Remove invisible and non-content elements
+    const elementsToRemove = [
+        "script",
+        "style",
+        "meta",
+        'link[rel="stylesheet"]',
+        'link[rel="preload"]',
+        'link[rel="prefetch"]',
+        "iframe",
+        "noscript",
+        "svg",
+        "video",
+        "object",
+        "embed",
+        "canvas",
+        "template",
+        '[style*="display:none"]',
+        '[style*="display: none"]',
+        '[style*="visibility:hidden"]',
+        '[style*="visibility: hidden"]',
+        "[hidden]",
+        '[aria-hidden="true"]',
+    ];
+    // Remove all specified elements
+    elementsToRemove.forEach((selector) => {
+        $(selector).remove();
+    });
+    // Remove JavaScript event handlers
+    $("*").each((_, element) => {
+        const el = $(element);
+        const tagName = el.prop('tagName')?.toLowerCase();
+        // Get all attributes
+        const attributes = [];
+        el[0].attribs && Object.keys(el[0].attribs).forEach(attr => {
+            attributes.push(attr);
+        });
+        // Process each attribute
+        attributes.forEach(attr => {
+            // Remove JavaScript event handlers
+            if (attr.startsWith("on")) {
+                el.removeAttr(attr);
+                return;
+            }
+            // Keep essential attributes but remove others
+            const keepAttribute = 
+            // Keep structural attributes
+            attr === "href" ||
+                attr === "src" ||
+                attr === "alt" ||
+                attr === "title" ||
+                // Keep basic formatting
+                attr === "colspan" ||
+                attr === "rowspan" ||
+                // Keep semantic attributes
+                attr === "role" ||
+                attr === "aria-label" ||
+                // Keep header identification
+                (attr === "id" &&
+                    (tagName === "h1" ||
+                        tagName === "h2" ||
+                        tagName === "h3" ||
+                        tagName === "h4" ||
+                        tagName === "h5" ||
+                        tagName === "h6" ||
+                        tagName === "header"));
+            if (!keepAttribute) {
+                el.removeAttr(attr);
+            }
+        });
+    });
+    // Basic HTML minification
+    let result = $.html();
+    // Minify the HTML
+    result = result
+        // Remove comments
+        .replace(/<!--[\s\S]*?-->/g, "")
+        // Remove extra whitespace
+        .replace(/\s{2,}/g, " ")
+        // Remove whitespace between tags
+        .replace(/>\s+</g, "><")
+        // Remove whitespace at start/end of each line
+        .replace(/^\s+|\s+$/gm, "")
+        // Remove unnecessary line breaks while keeping some structure
+        .replace(/\n+/g, "\n");
+    return result;
+}
+/**
+ * Process HTML by cleaning it and converting all URLs to absolute
+ * @param html The HTML string to process
+ * @param baseUrl The base URL for resolving relative URLs
+ * @returns Processed HTML
+ */
+function processHtml(html, baseUrl) {
+    // first clean and minify
+    const cleaned = cleanAndMinifyHtml(html);
+    // then convert urls to absolute
+    return makeUrlsAbsolute(cleaned, baseUrl);
+}