@illuma-ai/agents 1.1.21 → 1.1.23

package/src/tools/search/search.js

@@ -1,561 +0,0 @@
-import axios from 'axios';
-import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
-import { getAttribution, createDefaultLogger } from './utils';
-const chunker = {
-    cleanText: (text) => {
-        if (!text)
-            return '';
-        /** Normalized all line endings to '\n' */
-        const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
-        /** Handle multiple backslashes followed by newlines
-         * This replaces patterns like '\\\\\\n' with a single newline */
-        const fixedBackslashes = normalizedText.replace(/\\+\n/g, '\n');
-        /** Cleaned up consecutive newlines, tabs, and spaces around newlines */
-        const cleanedNewlines = fixedBackslashes.replace(/[\t ]*\n[\t \n]*/g, '\n');
-        /** Cleaned up excessive spaces and tabs */
-        const cleanedSpaces = cleanedNewlines.replace(/[ \t]+/g, ' ');
-        return cleanedSpaces.trim();
-    },
-    splitText: async (text, options) => {
-        const chunkSize = options?.chunkSize ?? 150;
-        const chunkOverlap = options?.chunkOverlap ?? 50;
-        const separators = options?.separators || ['\n\n', '\n'];
-        const splitter = new RecursiveCharacterTextSplitter({
-            separators,
-            chunkSize,
-            chunkOverlap,
-        });
-        return await splitter.splitText(text);
-    },
-    splitTexts: async (texts, options, logger) => {
-        // Split multiple texts
-        const logger_ = logger || createDefaultLogger();
-        const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
-            logger_.error('Error splitting text:', error);
-            return [text];
-        }));
-        return Promise.all(promises);
-    },
-};
-function createSourceUpdateCallback(sourceMap) {
-    return (link, update) => {
-        const source = sourceMap.get(link);
-        if (source) {
-            sourceMap.set(link, {
-                ...source,
-                ...update,
-            });
-        }
-    };
-}
-const getHighlights = async ({ query, content, reranker, topResults = 5, logger, }) => {
-    const logger_ = logger || createDefaultLogger();
-    if (!content) {
-        logger_.warn('No content provided for highlights');
-        return;
-    }
-    if (!reranker) {
-        logger_.warn('No reranker provided for highlights');
-        return;
-    }
-    try {
-        const documents = await chunker.splitText(content);
-        if (Array.isArray(documents)) {
-            return await reranker.rerank(query, documents, topResults);
-        }
-        else {
-            logger_.error('Expected documents to be an array, got:', typeof documents);
-            return;
-        }
-    }
-    catch (error) {
-        logger_.error('Error in content processing:', error);
-        return;
-    }
-};
-const createSerperAPI = (apiKey) => {
-    const config = {
-        apiKey: apiKey ?? process.env.SERPER_API_KEY,
-        apiUrl: 'https://google.serper.dev/search',
-        timeout: 10000,
-    };
-    if (config.apiKey == null || config.apiKey === '') {
-        throw new Error('SERPER_API_KEY is required for SerperAPI');
-    }
-    const getSources = async ({ query, date, country, safeSearch, numResults = 8, type, }) => {
-        if (!query.trim()) {
-            return { success: false, error: 'Query cannot be empty' };
-        }
-        try {
-            const safe = ['off', 'moderate', 'active'];
-            const payload = {
-                q: query,
-                safe: safe[safeSearch ?? 1],
-                num: Math.min(Math.max(1, numResults), 10),
-            };
-            // Set the search type if provided
-            if (type) {
-                payload.type = type;
-            }
-            if (date != null) {
-                payload.tbs = `qdr:${date}`;
-            }
-            if (country != null && country !== '') {
-                payload['gl'] = country.toLowerCase();
-            }
-            // Determine the API endpoint based on the search type
-            let apiEndpoint = config.apiUrl;
-            if (type === 'images') {
-                apiEndpoint = 'https://google.serper.dev/images';
-            }
-            else if (type === 'videos') {
-                apiEndpoint = 'https://google.serper.dev/videos';
-            }
-            else if (type === 'news') {
-                apiEndpoint = 'https://google.serper.dev/news';
-            }
-            const response = await axios.post(apiEndpoint, payload, {
-                headers: {
-                    'X-API-KEY': config.apiKey,
-                    'Content-Type': 'application/json',
-                },
-                timeout: config.timeout,
-            });
-            const data = response.data;
-            const results = {
-                organic: data.organic,
-                images: data.images ?? [],
-                answerBox: data.answerBox,
-                topStories: data.topStories ?? [],
-                peopleAlsoAsk: data.peopleAlsoAsk,
-                knowledgeGraph: data.knowledgeGraph,
-                relatedSearches: data.relatedSearches,
-                videos: data.videos ?? [],
-                news: data.news ?? [],
-            };
-            return { success: true, data: results };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return { success: false, error: `API request failed: ${errorMessage}` };
-        }
-    };
-    return { getSources };
-};
-const createSearXNGAPI = (instanceUrl, apiKey) => {
-    const config = {
-        instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
-        apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
-        defaultLocation: 'all',
-        timeout: 10000,
-    };
-    if (config.instanceUrl == null || config.instanceUrl === '') {
-        throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
-    }
-    const getSources = async ({ query, numResults = 8, safeSearch, type, }) => {
-        if (!query.trim()) {
-            return { success: false, error: 'Query cannot be empty' };
-        }
-        try {
-            // Ensure the instance URL ends with /search
-            if (config.instanceUrl == null || config.instanceUrl === '') {
-                return { success: false, error: 'Instance URL is not defined' };
-            }
-            let searchUrl = config.instanceUrl;
-            if (!searchUrl.endsWith('/search')) {
-                searchUrl = searchUrl.replace(/\/$/, '') + '/search';
-            }
-            // Determine the search category based on the type
-            let category = 'general';
-            if (type === 'images') {
-                category = 'images';
-            }
-            else if (type === 'videos') {
-                category = 'videos';
-            }
-            else if (type === 'news') {
-                category = 'news';
-            }
-            // Prepare parameters for SearXNG
-            const params = {
-                q: query,
-                format: 'json',
-                pageno: 1,
-                categories: category,
-                language: 'all',
-                safesearch: safeSearch,
-                engines: 'google,bing,duckduckgo',
-            };
-            const headers = {
-                'Content-Type': 'application/json',
-            };
-            if (config.apiKey != null && config.apiKey !== '') {
-                headers['X-API-Key'] = config.apiKey;
-            }
-            const response = await axios.get(searchUrl, {
-                headers,
-                params,
-                timeout: config.timeout,
-            });
-            const data = response.data;
-            // Helper function to identify news results since SearXNG doesn't provide that classification by default
-            const isNewsResult = (result) => {
-                const url = result.url?.toLowerCase() ?? '';
-                const title = result.title?.toLowerCase() ?? '';
-                // News-related keywords in title/content
-                const newsKeywords = [
-                    'breaking news',
-                    'latest news',
-                    'top stories',
-                    'news today',
-                    'developing story',
-                    'trending news',
-                    'news',
-                ];
-                // Check if title/content contains news keywords
-                const hasNewsKeywords = newsKeywords.some((keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)
-                );
-                // Check if URL contains news-related paths
-                const hasNewsPath = url.includes('/news/') ||
-                    url.includes('/world/') ||
-                    url.includes('/politics/') ||
-                    url.includes('/breaking/');
-                return hasNewsKeywords || hasNewsPath;
-            };
-            // Transform SearXNG results to match SerperAPI format
-            const organicResults = (data.results ?? [])
-                .slice(0, numResults)
-                .map((result, index) => {
-                let attribution = '';
-                try {
-                    attribution = new URL(result.url ?? '').hostname;
-                }
-                catch {
-                    attribution = '';
-                }
-                return {
-                    position: index + 1,
-                    title: result.title ?? '',
-                    link: result.url ?? '',
-                    snippet: result.content ?? '',
-                    date: result.publishedDate ?? '',
-                    attribution,
-                };
-            });
-            const imageResults = (data.results ?? [])
-                .filter((result) => result.img_src)
-                .slice(0, 6)
-                .map((result, index) => ({
-                title: result.title ?? '',
-                imageUrl: result.img_src ?? '',
-                position: index + 1,
-                source: new URL(result.url ?? '').hostname,
-                domain: new URL(result.url ?? '').hostname,
-                link: result.url ?? '',
-            }));
-            // Extract news results from organic results
-            const newsResults = (data.results ?? [])
-                .filter(isNewsResult)
-                .map((result, index) => {
-                let attribution = '';
-                try {
-                    attribution = new URL(result.url ?? '').hostname;
-                }
-                catch {
-                    attribution = '';
-                }
-                return {
-                    title: result.title ?? '',
-                    link: result.url ?? '',
-                    snippet: result.content ?? '',
-                    date: result.publishedDate ?? '',
-                    source: attribution,
-                    imageUrl: result.img_src ?? '',
-                    position: index + 1,
-                };
-            });
-            const topStories = newsResults.slice(0, 5);
-            const relatedSearches = Array.isArray(data.suggestions)
-                ? data.suggestions.map((suggestion) => ({ query: suggestion }))
-                : [];
-            const results = {
-                organic: organicResults,
-                images: imageResults,
-                topStories: topStories, // Use first 5 extracted news as top stories
-                relatedSearches,
-                videos: [],
-                news: newsResults,
-                // Add empty arrays for other Serper fields to maintain parity
-                places: [],
-                shopping: [],
-                peopleAlsoAsk: [],
-                knowledgeGraph: undefined,
-                answerBox: undefined,
-            };
-            return { success: true, data: results };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                success: false,
-                error: `SearXNG API request failed: ${errorMessage}`,
-            };
-        }
-    };
-    return { getSources };
-};
-export const createSearchAPI = (config) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, } = config;
-    if (searchProvider.toLowerCase() === 'serper') {
-        return createSerperAPI(serperApiKey);
-    }
-    else if (searchProvider.toLowerCase() === 'searxng') {
-        return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
-    }
-    else {
-        throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`);
-    }
-};
-export const createSourceProcessor = (config = {}, scraperInstance) => {
-    if (!scraperInstance) {
-        throw new Error('Scraper instance is required');
-    }
-    const { topResults = 5, 
-    // strategies = ['no_extraction'],
-    // filterContent = true,
-    reranker, logger, } = config;
-    const logger_ = logger || createDefaultLogger();
-    const scraper = scraperInstance;
-    const webScraper = {
-        scrapeMany: async ({ query, links, onGetHighlights, }) => {
-            logger_.debug(`Scraping ${links.length} links`);
-            const promises = [];
-            try {
-                for (let i = 0; i < links.length; i++) {
-                    const currentLink = links[i];
-                    const promise = scraper
-                        .scrapeUrl(currentLink, {})
-                        .then(([url, response]) => {
-                        const attribution = getAttribution(url, response.data?.metadata, logger_);
-                        if (response.success && response.data) {
-                            const [content, references] = scraper.extractContent(response);
-                            return {
-                                url,
-                                references,
-                                attribution,
-                                content: chunker.cleanText(content),
-                            };
-                        }
-                        else {
-                            logger_.error(`Error scraping ${url}: ${response.error ?? 'Unknown error'}`);
-                        }
-                        return {
-                            url,
-                            attribution,
-                            error: true,
-                            content: '',
-                        };
-                    })
-                        .then(async (result) => {
-                        try {
-                            if (result.error != null) {
-                                logger_.error(`Error scraping ${result.url}: ${result.content}`);
-                                return {
-                                    ...result,
-                                };
-                            }
-                            const highlights = await getHighlights({
-                                query,
-                                reranker,
-                                content: result.content,
-                                logger: logger_,
-                            });
-                            if (onGetHighlights) {
-                                onGetHighlights(result.url);
-                            }
-                            return {
-                                ...result,
-                                highlights,
-                            };
-                        }
-                        catch (error) {
-                            logger_.error('Error processing scraped content:', error);
-                            return {
-                                ...result,
-                            };
-                        }
-                    })
-                        .catch((error) => {
-                        logger_.error(`Error scraping ${currentLink}:`, error);
-                        return {
-                            url: currentLink,
-                            error: true,
-                            content: '',
-                        };
-                    });
-                    promises.push(promise);
-                }
-                return await Promise.all(promises);
-            }
-            catch (error) {
-                logger_.error('Error in scrapeMany:', error);
-                return [];
-            }
-        },
-    };
-    const fetchContents = async ({ links, query, target, onGetHighlights, onContentScraped, }) => {
-        const initialLinks = links.slice(0, target);
-        // const remainingLinks = links.slice(target).reverse();
-        const results = await webScraper.scrapeMany({
-            query,
-            links: initialLinks,
-            onGetHighlights,
-        });
-        for (const result of results) {
-            if (result.error === true) {
-                continue;
-            }
-            const { url, content, attribution, references, highlights } = result;
-            onContentScraped?.(url, {
-                content,
-                attribution,
-                references,
-                highlights,
-            });
-        }
-    };
-    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, skipScraping = false, }) => {
-        try {
-            if (!result.data) {
-                return {
-                    organic: [],
-                    topStories: [],
-                    images: [],
-                    relatedSearches: [],
-                };
-            }
-            else if (!result.data.organic) {
-                return result.data;
-            }
-            // If content was already extracted directly (e.g., direct URL extraction), skip scraping
-            if (skipScraping) {
-                logger_.debug('Skipping additional scraping - content already extracted');
-                return result.data;
-            }
-            if (!proMode) {
-                const wikiSources = result.data.organic.filter((source) => source.link.includes('wikipedia.org'));
-                if (!wikiSources.length) {
-                    return result.data;
-                }
-                const wikiSourceMap = new Map();
-                wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);
-                const onContentScraped = createSourceUpdateCallback(wikiSourceMap);
-                await fetchContents({
-                    query,
-                    target: 1,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: [wikiSources[0].link],
-                });
-                for (let i = 0; i < result.data.organic.length; i++) {
-                    const source = result.data.organic[i];
-                    const updatedSource = wikiSourceMap.get(source.link);
-                    if (updatedSource) {
-                        result.data.organic[i] = {
-                            ...source,
-                            ...updatedSource,
-                        };
-                    }
-                }
-                return result.data;
-            }
-            const sourceMap = new Map();
-            const organicLinksSet = new Set();
-            // Collect organic links
-            const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
-            // Collect top story links, excluding any that are already in organic links
-            const topStories = result.data.topStories ?? [];
-            const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
-            if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
-                return result.data;
-            }
-            const onContentScraped = createSourceUpdateCallback(sourceMap);
-            const promises = [];
-            // Process organic links
-            if (organicLinks.length > 0) {
-                promises.push(fetchContents({
-                    query,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: organicLinks,
-                    target: numElements,
-                }));
-            }
-            // Process top story links
-            if (news && topStoryLinks.length > 0) {
-                promises.push(fetchContents({
-                    query,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: topStoryLinks,
-                    target: numElements,
-                }));
-            }
-            await Promise.all(promises);
-            if (result.data.organic.length > 0) {
-                updateSourcesWithContent(result.data.organic, sourceMap);
-            }
-            if (news && topStories.length > 0) {
-                updateSourcesWithContent(topStories, sourceMap);
-            }
-            return result.data;
-        }
-        catch (error) {
-            logger_.error('Error in processSources:', error);
-            return {
-                organic: [],
-                topStories: [],
-                images: [],
-                relatedSearches: [],
-                ...result.data,
-                error: error instanceof Error ? error.message : String(error),
-            };
-        }
-    };
-    return {
-        processSources,
-        topResults,
-    };
-};
-/** Helper function to collect links and update sourceMap */
-function collectLinks(sources, sourceMap, existingLinksSet) {
-    const links = [];
-    for (const source of sources) {
-        if (source.link) {
-            // For topStories, only add if not already in organic links
-            if (existingLinksSet && existingLinksSet.has(source.link)) {
-                continue;
-            }
-            links.push(source.link);
-            if (existingLinksSet) {
-                existingLinksSet.add(source.link);
-            }
-            sourceMap.set(source.link, source);
-        }
-    }
-    return links;
-}
-/** Helper function to update sources with scraped content */
-function updateSourcesWithContent(sources, sourceMap) {
-    for (let i = 0; i < sources.length; i++) {
-        const source = sources[i];
-        const updatedSource = sourceMap.get(source.link);
-        if (updatedSource) {
-            sources[i] = {
-                ...source,
-                ...updatedSource,
-            };
-        }
-    }
-}
-//# sourceMappingURL=search.js.map

package/src/tools/search/serper-scraper.js

@@ -1,126 +0,0 @@
-import axios from 'axios';
-import { createDefaultLogger } from './utils';
-/**
- * Serper scraper implementation
- * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages
- *
- * Features:
- * - Simple API with single endpoint
- * - Returns both text and markdown content
- * - Includes metadata from scraped pages
- * - Credits-based pricing model
- *
- * @example
- * ```typescript
- * const scraper = createSerperScraper({
- *   apiKey: 'your-serper-api-key',
- *   includeMarkdown: true,
- *   timeout: 10000
- * });
- *
- * const [url, response] = await scraper.scrapeUrl('https://example.com');
- * if (response.success) {
- *   const [content] = scraper.extractContent(response);
- *   console.log(content);
- * }
- * ```
- */
-export class SerperScraper {
-    apiKey;
-    apiUrl;
-    timeout;
-    logger;
-    includeMarkdown;
-    constructor(config = {}) {
-        this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';
-        this.apiUrl =
-            config.apiUrl ??
-                process.env.SERPER_SCRAPE_URL ??
-                'https://scrape.serper.dev';
-        this.timeout = config.timeout ?? 7500;
-        this.includeMarkdown = config.includeMarkdown ?? true;
-        this.logger = config.logger || createDefaultLogger();
-        if (!this.apiKey) {
-            this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');
-        }
-        this.logger.debug(`Serper scraper initialized with API URL: ${this.apiUrl}`);
-    }
-    /**
-     * Scrape a single URL
-     * @param url URL to scrape
-     * @param options Scrape options
-     * @returns Scrape response
-     */
-    async scrapeUrl(url, options = {}) {
-        if (!this.apiKey) {
-            return [
-                url,
-                {
-                    success: false,
-                    error: 'SERPER_API_KEY is not set',
-                },
-            ];
-        }
-        try {
-            const payload = {
-                url,
-                includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,
-            };
-            const response = await axios.post(this.apiUrl, payload, {
-                headers: {
-                    'X-API-KEY': this.apiKey,
-                    'Content-Type': 'application/json',
-                },
-                timeout: options.timeout ?? this.timeout,
-            });
-            return [url, { success: true, data: response.data }];
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return [
-                url,
-                {
-                    success: false,
-                    error: `Serper Scrape API request failed: ${errorMessage}`,
-                },
-            ];
-        }
-    }
-    /**
-     * Extract content from scrape response
-     * @param response Scrape response
-     * @returns Extracted content or empty string if not available
-     */
-    extractContent(response) {
-        if (!response.success || !response.data) {
-            return ['', undefined];
-        }
-        if (response.data.markdown != null) {
-            return [response.data.markdown, undefined];
-        }
-        if (response.data.text != null) {
-            return [response.data.text, undefined];
-        }
-        return ['', undefined];
-    }
-    /**
-     * Extract metadata from scrape response
-     * @param response Scrape response
-     * @returns Metadata object
-     */
-    extractMetadata(response) {
-        if (!response.success || !response.data || !response.data.metadata) {
-            return {};
-        }
-        return response.data.metadata;
-    }
-}
-/**
- * Create a Serper scraper instance
- * @param config Scraper configuration
- * @returns Serper scraper instance
- */
-export const createSerperScraper = (config = {}) => {
-    return new SerperScraper(config);
-};
-//# sourceMappingURL=serper-scraper.js.map