@illuma-ai/agents 1.1.20 → 1.1.22

package/src/tools/search/test.js

@@ -1,129 +0,0 @@
-/* eslint-disable no-console */
-// processWikipedia.ts
-import * as fs from 'fs';
-import * as path from 'path';
-import { processContent } from './content';
-/**
- * Process a Wikipedia article (HTML and Markdown) and create a referenced version
- */
-async function processWikipediaArticle() {
-    try {
-        console.log('Starting Wikipedia article processing...');
-        // Define file paths - adapt these to your specific file locations
-        const htmlPath = path.resolve('./test.html');
-        const markdownPath = path.resolve('./test.md');
-        const outputPath = path.resolve('./output.md');
-        // Check if input files exist
-        if (!fs.existsSync(htmlPath)) {
-            throw new Error(`Wikipedia HTML file not found at ${htmlPath}`);
-        }
-        if (!fs.existsSync(markdownPath)) {
-            throw new Error(`Wikipedia Markdown file not found at ${markdownPath}`);
-        }
-        console.log('Reading Wikipedia article files...');
-        const html = fs.readFileSync(htmlPath, 'utf-8');
-        const markdown = fs.readFileSync(markdownPath, 'utf-8');
-        // Extract article title for logging
-        const titleMatch = /<h1[^>]*>([^<]+)<\/h1>/i.exec(html);
-        const articleTitle = titleMatch
-            ? titleMatch[1].trim()
-            : 'Wikipedia article';
-        console.log(`Processing "${articleTitle}"...`);
-        // Measure processing time
-        const startTime = process.hrtime();
-        // Process content
-        const result = processContent(html, markdown);
-        // Calculate processing time
-        const elapsed = process.hrtime(startTime);
-        const timeInMs = elapsed[0] * 1000 + elapsed[1] / 1000000;
-        // Generate reference appendix
-        const appendix = generateReferenceAppendix(result);
-        // Create complete output with the processed content and appendix
-        const completeOutput = result.markdown + appendix;
-        // Write to output file
-        fs.writeFileSync(outputPath, completeOutput);
-        // Print processing statistics
-        console.log('\nWikipedia article processing complete! ✓');
-        console.log('-'.repeat(60));
-        console.log(`Article: ${articleTitle}`);
-        console.log(`Processing time: ${timeInMs.toFixed(2)}ms`);
-        console.log('Media references replaced:');
-        console.log(`  - Links: ${result.links.length}`);
-        console.log(`  - Images: ${result.images.length}`);
-        console.log(`  - Videos: ${result.videos.length}`);
-        console.log(`  - Total: ${result.links.length + result.images.length + result.videos.length}`);
-        console.log(`Output saved to: ${outputPath}`);
-        console.log('-'.repeat(60));
-        // Print sample of the transformation
-        const sampleLines = result.markdown.split('\n').slice(0, 10).join('\n');
-        console.log('\nSample of transformed content:');
-        console.log('-'.repeat(30));
-        console.log(sampleLines);
-        console.log('-'.repeat(30));
-        console.log('... (continued in output file)');
-    }
-    catch (error) {
-        console.error('Error processing Wikipedia article:', error);
-        process.exit(1);
-    }
-}
-/**
- * Generate a comprehensive reference appendix with all media links
- */
-function generateReferenceAppendix(result) {
-    let appendix = '\n\n' + '---'.repeat(10) + '\n\n';
-    appendix += '# References\n\n';
-    if (result.links.length > 0) {
-        appendix += '## Links\n\n';
-        result.links.forEach((link, index) => {
-            // Clean and format text for display
-            let displayText = '';
-            if (link.text != null && link.text.trim()) {
-                // Limit length for very long link text
-                let cleanText = link.text.trim();
-                if (cleanText.length > 50) {
-                    cleanText = cleanText.substring(0, 47) + '...';
-                }
-                displayText = ` - "${cleanText}"`;
-            }
-            appendix += `**link#${index + 1}**: ${link.originalUrl}${displayText}\n\n`;
-        });
-    }
-    if (result.images.length > 0) {
-        appendix += '## Images\n\n';
-        result.images.forEach((image, index) => {
-            const displayTitle = image.title != null && image.title.trim()
-                ? ` - ${image.title.trim()}`
-                : '';
-            appendix += `**image#${index + 1}**: ${image.originalUrl}${displayTitle}\n\n`;
-        });
-    }
-    if (result.videos.length > 0) {
-        appendix += '## Videos\n\n';
-        result.videos.forEach((video, index) => {
-            const displayTitle = video.title != null && video.title.trim()
-                ? ` - ${video.title.trim()}`
-                : '';
-            appendix += `**video#${index + 1}**: ${video.originalUrl}${displayTitle}\n\n`;
-        });
-    }
-    // Add a category breakdown to show what types of links were found
-    const totalRefs = result.links.length + result.images.length + result.videos.length;
-    appendix += '## Summary\n\n';
-    appendix += `Total references: **${totalRefs}**\n\n`;
-    appendix += `- Links: ${result.links.length}\n`;
-    appendix += `- Images: ${result.images.length}\n`;
-    appendix += `- Videos: ${result.videos.length}\n`;
-    return appendix;
-}
-// Using async IIFE to allow for better error handling
-(async () => {
-    try {
-        await processWikipediaArticle();
-    }
-    catch (error) {
-        console.error('Unhandled error:', error);
-        process.exit(1);
-    }
-})();
-//# sourceMappingURL=test.js.map

package/src/tools/search/tool.js

@@ -1,453 +0,0 @@
-import { tool } from '@langchain/core/tools';
-import { countrySchema, imagesSchema, videosSchema, querySchema, dateSchema, newsSchema, } from './schema';
-import { createSearchAPI, createSourceProcessor } from './search';
-import { createSerperScraper } from './serper-scraper';
-import { createFirecrawlScraper } from './firecrawl';
-import { expandHighlights } from './highlights';
-import { formatResultsForLLM } from './format';
-import { createDefaultLogger } from './utils';
-import { createReranker } from './rerankers';
-import { Constants } from '@/common';
-/**
- * URL regex pattern to detect direct URLs in query
- */
-const URL_PATTERN = /https?:\/\/[^\s<>"{}|\\^`[\]]+/gi;
-/**
- * Extracts URLs from a query string
- * @param query - The search query
- * @returns Array of URLs found in the query
- */
-function extractUrlsFromQuery(query) {
-    const matches = query.match(URL_PATTERN);
-    return matches ?? [];
-}
-/**
- * Checks if the query is primarily a URL request (contains URL and minimal other text)
- * @param query - The search query
- * @returns True if the query appears to be a direct URL request
- */
-function isDirectUrlRequest(query) {
-    const urls = extractUrlsFromQuery(query);
-    if (urls.length === 0) {
-        return false;
-    }
-    // Remove URLs from query and check remaining text
-    let remainingText = query;
-    for (const url of urls) {
-        remainingText = remainingText.replace(url, '');
-    }
-    // Clean up and check if remaining text is minimal (just filler words or questions about the URL)
-    remainingText = remainingText.trim().toLowerCase();
-    // If very little text remains, it's likely a direct URL request
-    if (remainingText.length < 50) {
-        return true;
-    }
-    return false;
-}
-/**
- * Directly extracts content from URLs using the scraper
- * @param urls - URLs to extract content from
- * @param scraper - The scraper instance to use
- * @param logger - Logger instance
- * @returns Search result with extracted content
- */
-async function extractDirectUrlContent({ urls, scraper, logger, }) {
-    try {
-        const results = [];
-        for (const url of urls) {
-            try {
-                logger.debug(`Direct URL extraction: ${url}`);
-                const [, response] = await scraper.scrapeUrl(url);
-                if (response.success && response.data) {
-                    const [content, references] = scraper.extractContent(response);
-                    const metadata = scraper.extractMetadata(response);
-                    // Helper to safely extract string from metadata
-                    const getString = (value) => {
-                        return typeof value === 'string' ? value : undefined;
-                    };
-                    results.push({
-                        position: results.length + 1,
-                        title: getString(metadata.title) ?? getString(metadata.ogTitle) ?? url,
-                        link: url,
-                        snippet: getString(metadata.description) ??
-                            getString(metadata.ogDescription) ??
-                            '',
-                        content: content,
-                        references: references,
-                        processed: true,
-                    });
-                }
-                else {
-                    logger.warn(`Failed to extract content from ${url}: ${response.error}`);
-                    // Still add the URL as a result, but without content
-                    results.push({
-                        position: results.length + 1,
-                        title: url,
-                        link: url,
-                        snippet: response.error ?? 'Failed to extract content',
-                        processed: false,
-                    });
-                }
-            }
-            catch (error) {
-                logger.error(`Error extracting URL ${url}:`, error);
-                results.push({
-                    position: results.length + 1,
-                    title: url,
-                    link: url,
-                    snippet: error instanceof Error ? error.message : String(error),
-                    processed: false,
-                });
-            }
-        }
-        return {
-            success: true,
-            data: {
-                organic: results,
-                topStories: [],
-                images: [],
-                videos: [],
-                relatedSearches: [],
-            },
-        };
-    }
-    catch (error) {
-        logger.error('Error in direct URL extraction:', error);
-        return {
-            success: false,
-            error: error instanceof Error ? error.message : String(error),
-        };
-    }
-}
-/**
- * Executes parallel searches and merges the results
- */
-async function executeParallelSearches({ searchAPI, query, date, country, safeSearch, images, videos, news, logger, }) {
-    // Prepare all search tasks to run in parallel
-    const searchTasks = [
-        // Main search
-        searchAPI.getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-        }),
-    ];
-    if (images) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'images',
-        })
-            .catch((error) => {
-            logger.error('Error fetching images:', error);
-            return {
-                success: false,
-                error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    if (videos) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'videos',
-        })
-            .catch((error) => {
-            logger.error('Error fetching videos:', error);
-            return {
-                success: false,
-                error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    if (news) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'news',
-        })
-            .catch((error) => {
-            logger.error('Error fetching news:', error);
-            return {
-                success: false,
-                error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    // Run all searches in parallel
-    const results = await Promise.all(searchTasks);
-    // Get the main search result (first result)
-    const mainResult = results[0];
-    if (!mainResult.success) {
-        throw new Error(mainResult.error ?? 'Search failed');
-    }
-    // Merge additional results with the main results
-    const mergedResults = { ...mainResult.data };
-    // Convert existing news to topStories if present
-    if (mergedResults.news !== undefined && mergedResults.news.length > 0) {
-        const existingNewsAsTopStories = mergedResults.news
-            .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')
-            .map((newsItem) => ({
-            title: newsItem.title ?? '',
-            link: newsItem.link ?? '',
-            source: newsItem.source ?? '',
-            date: newsItem.date ?? '',
-            imageUrl: newsItem.imageUrl ?? '',
-            processed: false,
-        }));
-        mergedResults.topStories = [
-            ...(mergedResults.topStories ?? []),
-            ...existingNewsAsTopStories,
-        ];
-        delete mergedResults.news;
-    }
-    results.slice(1).forEach((result) => {
-        if (result.success && result.data !== undefined) {
-            if (result.data.images !== undefined && result.data.images.length > 0) {
-                mergedResults.images = [
-                    ...(mergedResults.images ?? []),
-                    ...result.data.images,
-                ];
-            }
-            if (result.data.videos !== undefined && result.data.videos.length > 0) {
-                mergedResults.videos = [
-                    ...(mergedResults.videos ?? []),
-                    ...result.data.videos,
-                ];
-            }
-            if (result.data.news !== undefined && result.data.news.length > 0) {
-                const newsAsTopStories = result.data.news.map((newsItem) => ({
-                    ...newsItem,
-                    link: newsItem.link ?? '',
-                }));
-                mergedResults.topStories = [
-                    ...(mergedResults.topStories ?? []),
-                    ...newsAsTopStories,
-                ];
-            }
-        }
-    });
-    return { success: true, data: mergedResults };
-}
-function createSearchProcessor({ searchAPI, safeSearch, sourceProcessor, scraper, onGetHighlights, logger, }) {
-    return async function ({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false, }) {
-        try {
-            // Check if query contains direct URLs for extraction
-            const urls = extractUrlsFromQuery(query);
-            const isDirectUrl = isDirectUrlRequest(query);
-            let searchResult;
-            if (isDirectUrl && urls.length > 0) {
-                // Direct URL extraction mode - skip search API and extract directly
-                logger.debug(`Direct URL extraction mode for: ${urls.join(', ')}`);
-                searchResult = await extractDirectUrlContent({
-                    urls,
-                    scraper,
-                    logger,
-                });
-            }
-            else {
-                // Normal search mode - execute parallel searches and merge results
-                searchResult = await executeParallelSearches({
-                    searchAPI,
-                    query,
-                    date,
-                    country,
-                    safeSearch,
-                    images,
-                    videos,
-                    news,
-                    logger,
-                });
-            }
-            onSearchResults?.(searchResult);
-            const processedSources = await sourceProcessor.processSources({
-                query,
-                news,
-                result: searchResult,
-                proMode,
-                onGetHighlights,
-                numElements: maxSources,
-                // Skip additional scraping if we already extracted content directly
-                skipScraping: isDirectUrl,
-            });
-            return expandHighlights(processedSources);
-        }
-        catch (error) {
-            logger.error('Error in search:', error);
-            return {
-                organic: [],
-                topStories: [],
-                images: [],
-                videos: [],
-                news: [],
-                relatedSearches: [],
-                error: error instanceof Error ? error.message : String(error),
-            };
-        }
-    };
-}
-function createOnSearchResults({ runnableConfig, onSearchResults, }) {
-    return function (results) {
-        if (!onSearchResults) {
-            return;
-        }
-        onSearchResults(results, runnableConfig);
-    };
-}
-function createTool({ schema, search, onSearchResults: _onSearchResults, }) {
-    return tool(async (rawParams, runnableConfig) => {
-        const params = rawParams;
-        const { query, date, country: _c, images, videos, news } = params;
-        const country = typeof _c === 'string' && _c ? _c : undefined;
-        // Log the incoming query for debugging URL detection
-        const toolLogger = createDefaultLogger();
-        toolLogger.debug(`[web_search] Received query: "${query}"`);
-        const detectedUrls = extractUrlsFromQuery(query);
-        if (detectedUrls.length > 0) {
-            toolLogger.debug(`[web_search] Detected URLs in query: ${detectedUrls.join(', ')}`);
-        }
-        const searchResult = await search({
-            query,
-            date,
-            country,
-            images,
-            videos,
-            news,
-            onSearchResults: createOnSearchResults({
-                runnableConfig,
-                onSearchResults: _onSearchResults,
-            }),
-        });
-        const turn = runnableConfig.toolCall?.turn ?? 0;
-        const { output, references } = formatResultsForLLM(turn, searchResult);
-        const data = { turn, ...searchResult, references };
-        return [output, { [Constants.WEB_SEARCH]: data }];
-    }, {
-        name: Constants.WEB_SEARCH,
-        description: `Real-time web search and direct URL content extraction. Results have required citation anchors.
-
-**CAPABILITIES:**
-- Search: Query the web for information on any topic
-- Direct URL: Fetch and extract content from a specific URL for summarization or analysis
-
-**CRITICAL - URL HANDLING:**
-When user provides a URL (e.g., "summarize https://example.com/article"), you MUST include the FULL URL in the query parameter.
-- CORRECT: query = "https://example.com/article" or "summarize https://example.com/article"
-- WRONG: query = "example article summary" (do NOT convert URLs to search terms)
-
-**USAGE:**
-- For search: Use concise search terms as query
-- For URL extraction: Pass the complete URL in the query field
-
-Note: Use ONCE per reply unless instructed otherwise.
-
-Anchors:
-- \\ue202turnXtypeY
-- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
-
-Special Markers:
-- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
-- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
-
-**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
-Use anchor marker(s) immediately after the statement:
-- Standalone: "Pure functions produce same output. \\ue202turn0search0"
-- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
-- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
-- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Image: "See photo \\ue202turn0image0."
-
-**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**
-`.trim(),
-        schema: schema,
-        responseFormat: Constants.CONTENT_AND_ARTIFACT,
-    });
-}
-export const createSearchTool = (config = {}) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, scraperProvider = 'firecrawl', firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
-    const logger = config.logger || createDefaultLogger();
-    const schemaProperties = {
-        query: querySchema,
-        date: dateSchema,
-        images: imagesSchema,
-        videos: videosSchema,
-        news: newsSchema,
-    };
-    if (searchProvider === 'serper') {
-        schemaProperties.country = countrySchema;
-    }
-    const toolSchema = {
-        type: 'object',
-        properties: schemaProperties,
-        required: ['query'],
-    };
-    const searchAPI = createSearchAPI({
-        searchProvider,
-        serperApiKey,
-        searxngInstanceUrl,
-        searxngApiKey,
-    });
-    /** Create scraper based on scraperProvider */
-    let scraperInstance;
-    if (scraperProvider === 'serper') {
-        scraperInstance = createSerperScraper({
-            ...serperScraperOptions,
-            apiKey: serperApiKey,
-            timeout: scraperTimeout ?? serperScraperOptions?.timeout,
-            logger,
-        });
-    }
-    else {
-        scraperInstance = createFirecrawlScraper({
-            ...firecrawlOptions,
-            apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
-            apiUrl: firecrawlApiUrl,
-            version: firecrawlVersion,
-            timeout: scraperTimeout ?? firecrawlOptions?.timeout,
-            formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
-            logger,
-        });
-    }
-    const selectedReranker = createReranker({
-        rerankerType,
-        jinaApiKey,
-        jinaApiUrl,
-        cohereApiKey,
-        logger,
-    });
-    if (!selectedReranker) {
-        logger.warn('No reranker selected. Using default ranking.');
-    }
-    const sourceProcessor = createSourceProcessor({
-        reranker: selectedReranker,
-        topResults,
-        strategies,
-        filterContent,
-        logger,
-    }, scraperInstance);
-    const search = createSearchProcessor({
-        searchAPI,
-        safeSearch,
-        sourceProcessor,
-        scraper: scraperInstance,
-        onGetHighlights,
-        logger,
-    });
-    return createTool({
-        search,
-        schema: toolSchema,
-        onSearchResults: _onSearchResults,
-    });
-};
-//# sourceMappingURL=tool.js.map

package/src/tools/search/types.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=types.js.map

package/src/tools/search/utils.js

@@ -1,59 +0,0 @@
-/* eslint-disable no-console */
-/**
- * Singleton instance of the default logger
- */
-let defaultLoggerInstance = null;
-/**
- * Creates a default logger that maps to console methods
- * Uses a singleton pattern to avoid creating multiple instances
- * @returns A default logger that implements the Logger interface
- */
-export const createDefaultLogger = () => {
-    if (!defaultLoggerInstance) {
-        defaultLoggerInstance = {
-            error: console.error,
-            warn: console.warn,
-            info: console.info,
-            debug: console.debug,
-        };
-    }
-    return defaultLoggerInstance;
-};
-export const fileExtRegex = /\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\?.*)?$/i;
-export const getDomainName = (link, metadata, logger) => {
-    try {
-        const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
-        const domain = new URL(url).hostname.replace(/^www\./, '');
-        if (domain) {
-            return domain;
-        }
-    }
-    catch (e) {
-        // URL parsing failed
-        if (logger) {
-            logger.error('Error parsing URL:', e);
-        }
-        else {
-            console.error('Error parsing URL:', e);
-        }
-    }
-    return;
-};
-export function getAttribution(link, metadata, logger) {
-    if (!metadata)
-        return getDomainName(link, metadata, logger);
-    const twitterSite = metadata['twitter:site'];
-    const twitterSiteFormatted = typeof twitterSite === 'string' ? twitterSite.replace(/^@/, '') : undefined;
-    const possibleAttributions = [
-        metadata.ogSiteName,
-        metadata['og:site_name'],
-        metadata.title?.split('|').pop()?.trim(),
-        twitterSiteFormatted,
-    ];
-    const attribution = possibleAttributions.find((attr) => attr != null && typeof attr === 'string' && attr.trim() !== '');
-    if (attribution != null) {
-        return attribution;
-    }
-    return getDomainName(link, metadata, logger);
-}
-//# sourceMappingURL=utils.js.map

package/src/types/graph.js

@@ -1,24 +0,0 @@
-/**
- * Error thrown when the model refuses to produce structured output due to safety policies.
- */
-export class StructuredOutputRefusalError extends Error {
-    refusalText;
-    constructor(refusalText) {
-        super(`Model refused to produce structured output: ${refusalText}`);
-        this.refusalText = refusalText;
-        this.name = 'StructuredOutputRefusalError';
-    }
-}
-/**
- * Error thrown when the structured output response was truncated due to max_tokens.
- */
-export class StructuredOutputTruncatedError extends Error {
-    stopReason;
-    constructor(stopReason) {
-        super(`Structured output was truncated (stop_reason: ${stopReason}). ` +
-            'Increase max_tokens to allow the full JSON response to be generated.');
-        this.stopReason = stopReason;
-        this.name = 'StructuredOutputTruncatedError';
-    }
-}
-//# sourceMappingURL=graph.js.map