@illuma-ai/agents 1.0.90 → 1.0.94

package/src/tools/search/schema.js

@@ -0,0 +1,102 @@
+export var DATE_RANGE;
+(function (DATE_RANGE) {
+    DATE_RANGE["PAST_HOUR"] = "h";
+    DATE_RANGE["PAST_24_HOURS"] = "d";
+    DATE_RANGE["PAST_WEEK"] = "w";
+    DATE_RANGE["PAST_MONTH"] = "m";
+    DATE_RANGE["PAST_YEAR"] = "y";
+})(DATE_RANGE || (DATE_RANGE = {}));
+export const DEFAULT_QUERY_DESCRIPTION = `
+GUIDELINES:
+- Start broad, then narrow: Begin with key concepts, then refine with specifics
+- Think like sources: Use terminology experts would use in the field
+- Consider perspective: Frame queries from different viewpoints for better results
+- Quality over quantity: A precise 3-4 word query often beats lengthy sentences
+
+TECHNIQUES (combine for power searches):
+- EXACT PHRASES: Use quotes ("climate change report")
+- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)
+- SITE-SPECIFIC: Restrict to websites (site:edu research)
+- FILETYPE: Find specific documents (filetype:pdf study)
+- OR OPERATOR: Find alternatives (electric OR hybrid cars)
+- DATE RANGE: Recent information (data after:2020)
+- WILDCARDS: Use * for unknown terms (how to * bread)
+- SPECIFIC QUESTIONS: Use who/what/when/where/why/how
+- DOMAIN TERMS: Include technical terminology for specialized topics
+- CONCISE TERMS: Prioritize keywords over sentences
+`.trim();
+export const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
+Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
+Provide this when the search should return results specific to a particular country.
+Examples:
+- "us" for United States (default)
+- "de" for Germany
+- "in" for India
+`.trim();
+export const querySchema = {
+    type: 'string',
+    description: DEFAULT_QUERY_DESCRIPTION,
+};
+export const dateSchema = {
+    type: 'string',
+    enum: Object.values(DATE_RANGE),
+    description: 'Date range for search results.',
+};
+export const countrySchema = {
+    type: 'string',
+    description: DEFAULT_COUNTRY_DESCRIPTION,
+};
+export const imagesSchema = {
+    type: 'boolean',
+    description: 'Whether to also run an image search.',
+};
+export const videosSchema = {
+    type: 'boolean',
+    description: 'Whether to also run a video search.',
+};
+export const newsSchema = {
+    type: 'boolean',
+    description: 'Whether to also run a news search.',
+};
+/** Combined web search tool schema with all properties */
+export const WebSearchToolSchema = {
+    type: 'object',
+    properties: {
+        query: querySchema,
+        date: dateSchema,
+        country: countrySchema,
+        images: imagesSchema,
+        videos: videosSchema,
+        news: newsSchema,
+    },
+    required: ['query'],
+};
+export const WebSearchToolName = 'web_search';
+export const WebSearchToolDescription = `Real-time search. Results have required citation anchors.
+
+Note: Use ONCE per reply unless instructed otherwise.
+
+Anchors:
+- \\ue202turnXtypeY
+- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
+
+Special Markers:
+- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
+- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
+
+**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
+Use anchor marker(s) immediately after the statement:
+- Standalone: "Pure functions produce same output. \\ue202turn0search0"
+- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
+- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
+- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Image: "See photo \\ue202turn0image0."
+
+**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**`;
+export const WebSearchToolDefinition = {
+    name: WebSearchToolName,
+    description: WebSearchToolDescription,
+    schema: WebSearchToolSchema,
+};
+//# sourceMappingURL=schema.js.map

package/src/tools/search/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["schema.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,UAMX;AAND,WAAY,UAAU;IACpB,6BAAe,CAAA;IACf,iCAAmB,CAAA;IACnB,6BAAe,CAAA;IACf,8BAAgB,CAAA;IAChB,6BAAe,CAAA;AACjB,CAAC,EANW,UAAU,KAAV,UAAU,QAMrB;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG;;;;;;;;;;;;;;;;;;CAkBxC,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,CAAC,MAAM,2BAA2B,GACtC;;;;;;;CAOD,CAAC,IAAI,EAAE,CAAC;AAET,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,yBAAyB;CAC9B,CAAC;AAEX,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;IAC/B,WAAW,EAAE,gCAAgC;CACrC,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,2BAA2B;CAChC,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,SAAS;IACf,WAAW,EAAE,sCAAsC;CAC3C,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,SAAS;IACf,WAAW,EAAE,qCAAqC;CAC1C,CAAC;AAEX,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,SAAS;IACf,WAAW,EAAE,oCAAoC;CACzC,CAAC;AAEX,0DAA0D;AAC1D,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,KAAK,EAAE,WAAW;QAClB,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,aAAa;QACtB,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,YAAY;QACpB,IAAI,EAAE,UAAU;KACjB;IACD,QAAQ,EAAE,CAAC,OAAO,CAAC;CACX,CAAC;AAEX,MAAM,CAAC,MAAM,iBAAiB,GAAG,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,wBAAwB,GAAG;;;;;;;;;;;;;;;;;;;;;kFAqB0C,CAAC;AAEnF,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,wBAAwB;IACrC,MAAM,EAAE,mBAAmB;CACnB,CAAC"}

package/src/tools/search/search.js

@@ -0,0 +1,561 @@
+import axios from 'axios';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { getAttribution, createDefaultLogger } from './utils';
+const chunker = {
+    cleanText: (text) => {
+        if (!text)
+            return '';
+        /** Normalized all line endings to '\n' */
+        const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+        /** Handle multiple backslashes followed by newlines
+         * This replaces patterns like '\\\\\\n' with a single newline */
+        const fixedBackslashes = normalizedText.replace(/\\+\n/g, '\n');
+        /** Cleaned up consecutive newlines, tabs, and spaces around newlines */
+        const cleanedNewlines = fixedBackslashes.replace(/[\t ]*\n[\t \n]*/g, '\n');
+        /** Cleaned up excessive spaces and tabs */
+        const cleanedSpaces = cleanedNewlines.replace(/[ \t]+/g, ' ');
+        return cleanedSpaces.trim();
+    },
+    splitText: async (text, options) => {
+        const chunkSize = options?.chunkSize ?? 150;
+        const chunkOverlap = options?.chunkOverlap ?? 50;
+        const separators = options?.separators || ['\n\n', '\n'];
+        const splitter = new RecursiveCharacterTextSplitter({
+            separators,
+            chunkSize,
+            chunkOverlap,
+        });
+        return await splitter.splitText(text);
+    },
+    splitTexts: async (texts, options, logger) => {
+        // Split multiple texts
+        const logger_ = logger || createDefaultLogger();
+        const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
+            logger_.error('Error splitting text:', error);
+            return [text];
+        }));
+        return Promise.all(promises);
+    },
+};
+function createSourceUpdateCallback(sourceMap) {
+    return (link, update) => {
+        const source = sourceMap.get(link);
+        if (source) {
+            sourceMap.set(link, {
+                ...source,
+                ...update,
+            });
+        }
+    };
+}
+const getHighlights = async ({ query, content, reranker, topResults = 5, logger, }) => {
+    const logger_ = logger || createDefaultLogger();
+    if (!content) {
+        logger_.warn('No content provided for highlights');
+        return;
+    }
+    if (!reranker) {
+        logger_.warn('No reranker provided for highlights');
+        return;
+    }
+    try {
+        const documents = await chunker.splitText(content);
+        if (Array.isArray(documents)) {
+            return await reranker.rerank(query, documents, topResults);
+        }
+        else {
+            logger_.error('Expected documents to be an array, got:', typeof documents);
+            return;
+        }
+    }
+    catch (error) {
+        logger_.error('Error in content processing:', error);
+        return;
+    }
+};
+const createSerperAPI = (apiKey) => {
+    const config = {
+        apiKey: apiKey ?? process.env.SERPER_API_KEY,
+        apiUrl: 'https://google.serper.dev/search',
+        timeout: 10000,
+    };
+    if (config.apiKey == null || config.apiKey === '') {
+        throw new Error('SERPER_API_KEY is required for SerperAPI');
+    }
+    const getSources = async ({ query, date, country, safeSearch, numResults = 8, type, }) => {
+        if (!query.trim()) {
+            return { success: false, error: 'Query cannot be empty' };
+        }
+        try {
+            const safe = ['off', 'moderate', 'active'];
+            const payload = {
+                q: query,
+                safe: safe[safeSearch ?? 1],
+                num: Math.min(Math.max(1, numResults), 10),
+            };
+            // Set the search type if provided
+            if (type) {
+                payload.type = type;
+            }
+            if (date != null) {
+                payload.tbs = `qdr:${date}`;
+            }
+            if (country != null && country !== '') {
+                payload['gl'] = country.toLowerCase();
+            }
+            // Determine the API endpoint based on the search type
+            let apiEndpoint = config.apiUrl;
+            if (type === 'images') {
+                apiEndpoint = 'https://google.serper.dev/images';
+            }
+            else if (type === 'videos') {
+                apiEndpoint = 'https://google.serper.dev/videos';
+            }
+            else if (type === 'news') {
+                apiEndpoint = 'https://google.serper.dev/news';
+            }
+            const response = await axios.post(apiEndpoint, payload, {
+                headers: {
+                    'X-API-KEY': config.apiKey,
+                    'Content-Type': 'application/json',
+                },
+                timeout: config.timeout,
+            });
+            const data = response.data;
+            const results = {
+                organic: data.organic,
+                images: data.images ?? [],
+                answerBox: data.answerBox,
+                topStories: data.topStories ?? [],
+                peopleAlsoAsk: data.peopleAlsoAsk,
+                knowledgeGraph: data.knowledgeGraph,
+                relatedSearches: data.relatedSearches,
+                videos: data.videos ?? [],
+                news: data.news ?? [],
+            };
+            return { success: true, data: results };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return { success: false, error: `API request failed: ${errorMessage}` };
+        }
+    };
+    return { getSources };
+};
+const createSearXNGAPI = (instanceUrl, apiKey) => {
+    const config = {
+        instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
+        apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
+        defaultLocation: 'all',
+        timeout: 10000,
+    };
+    if (config.instanceUrl == null || config.instanceUrl === '') {
+        throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
+    }
+    const getSources = async ({ query, numResults = 8, safeSearch, type, }) => {
+        if (!query.trim()) {
+            return { success: false, error: 'Query cannot be empty' };
+        }
+        try {
+            // Ensure the instance URL ends with /search
+            if (config.instanceUrl == null || config.instanceUrl === '') {
+                return { success: false, error: 'Instance URL is not defined' };
+            }
+            let searchUrl = config.instanceUrl;
+            if (!searchUrl.endsWith('/search')) {
+                searchUrl = searchUrl.replace(/\/$/, '') + '/search';
+            }
+            // Determine the search category based on the type
+            let category = 'general';
+            if (type === 'images') {
+                category = 'images';
+            }
+            else if (type === 'videos') {
+                category = 'videos';
+            }
+            else if (type === 'news') {
+                category = 'news';
+            }
+            // Prepare parameters for SearXNG
+            const params = {
+                q: query,
+                format: 'json',
+                pageno: 1,
+                categories: category,
+                language: 'all',
+                safesearch: safeSearch,
+                engines: 'google,bing,duckduckgo',
+            };
+            const headers = {
+                'Content-Type': 'application/json',
+            };
+            if (config.apiKey != null && config.apiKey !== '') {
+                headers['X-API-Key'] = config.apiKey;
+            }
+            const response = await axios.get(searchUrl, {
+                headers,
+                params,
+                timeout: config.timeout,
+            });
+            const data = response.data;
+            // Helper function to identify news results since SearXNG doesn't provide that classification by default
+            const isNewsResult = (result) => {
+                const url = result.url?.toLowerCase() ?? '';
+                const title = result.title?.toLowerCase() ?? '';
+                // News-related keywords in title/content
+                const newsKeywords = [
+                    'breaking news',
+                    'latest news',
+                    'top stories',
+                    'news today',
+                    'developing story',
+                    'trending news',
+                    'news',
+                ];
+                // Check if title/content contains news keywords
+                const hasNewsKeywords = newsKeywords.some((keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)
+                );
+                // Check if URL contains news-related paths
+                const hasNewsPath = url.includes('/news/') ||
+                    url.includes('/world/') ||
+                    url.includes('/politics/') ||
+                    url.includes('/breaking/');
+                return hasNewsKeywords || hasNewsPath;
+            };
+            // Transform SearXNG results to match SerperAPI format
+            const organicResults = (data.results ?? [])
+                .slice(0, numResults)
+                .map((result, index) => {
+                let attribution = '';
+                try {
+                    attribution = new URL(result.url ?? '').hostname;
+                }
+                catch {
+                    attribution = '';
+                }
+                return {
+                    position: index + 1,
+                    title: result.title ?? '',
+                    link: result.url ?? '',
+                    snippet: result.content ?? '',
+                    date: result.publishedDate ?? '',
+                    attribution,
+                };
+            });
+            const imageResults = (data.results ?? [])
+                .filter((result) => result.img_src)
+                .slice(0, 6)
+                .map((result, index) => ({
+                title: result.title ?? '',
+                imageUrl: result.img_src ?? '',
+                position: index + 1,
+                source: new URL(result.url ?? '').hostname,
+                domain: new URL(result.url ?? '').hostname,
+                link: result.url ?? '',
+            }));
+            // Extract news results from organic results
+            const newsResults = (data.results ?? [])
+                .filter(isNewsResult)
+                .map((result, index) => {
+                let attribution = '';
+                try {
+                    attribution = new URL(result.url ?? '').hostname;
+                }
+                catch {
+                    attribution = '';
+                }
+                return {
+                    title: result.title ?? '',
+                    link: result.url ?? '',
+                    snippet: result.content ?? '',
+                    date: result.publishedDate ?? '',
+                    source: attribution,
+                    imageUrl: result.img_src ?? '',
+                    position: index + 1,
+                };
+            });
+            const topStories = newsResults.slice(0, 5);
+            const relatedSearches = Array.isArray(data.suggestions)
+                ? data.suggestions.map((suggestion) => ({ query: suggestion }))
+                : [];
+            const results = {
+                organic: organicResults,
+                images: imageResults,
+                topStories: topStories, // Use first 5 extracted news as top stories
+                relatedSearches,
+                videos: [],
+                news: newsResults,
+                // Add empty arrays for other Serper fields to maintain parity
+                places: [],
+                shopping: [],
+                peopleAlsoAsk: [],
+                knowledgeGraph: undefined,
+                answerBox: undefined,
+            };
+            return { success: true, data: results };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                success: false,
+                error: `SearXNG API request failed: ${errorMessage}`,
+            };
+        }
+    };
+    return { getSources };
+};
+export const createSearchAPI = (config) => {
+    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, } = config;
+    if (searchProvider.toLowerCase() === 'serper') {
+        return createSerperAPI(serperApiKey);
+    }
+    else if (searchProvider.toLowerCase() === 'searxng') {
+        return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
+    }
+    else {
+        throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`);
+    }
+};
+export const createSourceProcessor = (config = {}, scraperInstance) => {
+    if (!scraperInstance) {
+        throw new Error('Scraper instance is required');
+    }
+    const { topResults = 5, 
+    // strategies = ['no_extraction'],
+    // filterContent = true,
+    reranker, logger, } = config;
+    const logger_ = logger || createDefaultLogger();
+    const scraper = scraperInstance;
+    const webScraper = {
+        scrapeMany: async ({ query, links, onGetHighlights, }) => {
+            logger_.debug(`Scraping ${links.length} links`);
+            const promises = [];
+            try {
+                for (let i = 0; i < links.length; i++) {
+                    const currentLink = links[i];
+                    const promise = scraper
+                        .scrapeUrl(currentLink, {})
+                        .then(([url, response]) => {
+                        const attribution = getAttribution(url, response.data?.metadata, logger_);
+                        if (response.success && response.data) {
+                            const [content, references] = scraper.extractContent(response);
+                            return {
+                                url,
+                                references,
+                                attribution,
+                                content: chunker.cleanText(content),
+                            };
+                        }
+                        else {
+                            logger_.error(`Error scraping ${url}: ${response.error ?? 'Unknown error'}`);
+                        }
+                        return {
+                            url,
+                            attribution,
+                            error: true,
+                            content: '',
+                        };
+                    })
+                        .then(async (result) => {
+                        try {
+                            if (result.error != null) {
+                                logger_.error(`Error scraping ${result.url}: ${result.content}`);
+                                return {
+                                    ...result,
+                                };
+                            }
+                            const highlights = await getHighlights({
+                                query,
+                                reranker,
+                                content: result.content,
+                                logger: logger_,
+                            });
+                            if (onGetHighlights) {
+                                onGetHighlights(result.url);
+                            }
+                            return {
+                                ...result,
+                                highlights,
+                            };
+                        }
+                        catch (error) {
+                            logger_.error('Error processing scraped content:', error);
+                            return {
+                                ...result,
+                            };
+                        }
+                    })
+                        .catch((error) => {
+                        logger_.error(`Error scraping ${currentLink}:`, error);
+                        return {
+                            url: currentLink,
+                            error: true,
+                            content: '',
+                        };
+                    });
+                    promises.push(promise);
+                }
+                return await Promise.all(promises);
+            }
+            catch (error) {
+                logger_.error('Error in scrapeMany:', error);
+                return [];
+            }
+        },
+    };
+    const fetchContents = async ({ links, query, target, onGetHighlights, onContentScraped, }) => {
+        const initialLinks = links.slice(0, target);
+        // const remainingLinks = links.slice(target).reverse();
+        const results = await webScraper.scrapeMany({
+            query,
+            links: initialLinks,
+            onGetHighlights,
+        });
+        for (const result of results) {
+            if (result.error === true) {
+                continue;
+            }
+            const { url, content, attribution, references, highlights } = result;
+            onContentScraped?.(url, {
+                content,
+                attribution,
+                references,
+                highlights,
+            });
+        }
+    };
+    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, skipScraping = false, }) => {
+        try {
+            if (!result.data) {
+                return {
+                    organic: [],
+                    topStories: [],
+                    images: [],
+                    relatedSearches: [],
+                };
+            }
+            else if (!result.data.organic) {
+                return result.data;
+            }
+            // If content was already extracted directly (e.g., direct URL extraction), skip scraping
+            if (skipScraping) {
+                logger_.debug('Skipping additional scraping - content already extracted');
+                return result.data;
+            }
+            if (!proMode) {
+                const wikiSources = result.data.organic.filter((source) => source.link.includes('wikipedia.org'));
+                if (!wikiSources.length) {
+                    return result.data;
+                }
+                const wikiSourceMap = new Map();
+                wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);
+                const onContentScraped = createSourceUpdateCallback(wikiSourceMap);
+                await fetchContents({
+                    query,
+                    target: 1,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: [wikiSources[0].link],
+                });
+                for (let i = 0; i < result.data.organic.length; i++) {
+                    const source = result.data.organic[i];
+                    const updatedSource = wikiSourceMap.get(source.link);
+                    if (updatedSource) {
+                        result.data.organic[i] = {
+                            ...source,
+                            ...updatedSource,
+                        };
+                    }
+                }
+                return result.data;
+            }
+            const sourceMap = new Map();
+            const organicLinksSet = new Set();
+            // Collect organic links
+            const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
+            // Collect top story links, excluding any that are already in organic links
+            const topStories = result.data.topStories ?? [];
+            const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
+            if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
+                return result.data;
+            }
+            const onContentScraped = createSourceUpdateCallback(sourceMap);
+            const promises = [];
+            // Process organic links
+            if (organicLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: organicLinks,
+                    target: numElements,
+                }));
+            }
+            // Process top story links
+            if (news && topStoryLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: topStoryLinks,
+                    target: numElements,
+                }));
+            }
+            await Promise.all(promises);
+            if (result.data.organic.length > 0) {
+                updateSourcesWithContent(result.data.organic, sourceMap);
+            }
+            if (news && topStories.length > 0) {
+                updateSourcesWithContent(topStories, sourceMap);
+            }
+            return result.data;
+        }
+        catch (error) {
+            logger_.error('Error in processSources:', error);
+            return {
+                organic: [],
+                topStories: [],
+                images: [],
+                relatedSearches: [],
+                ...result.data,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    };
+    return {
+        processSources,
+        topResults,
+    };
+};
+/** Helper function to collect links and update sourceMap */
+function collectLinks(sources, sourceMap, existingLinksSet) {
+    const links = [];
+    for (const source of sources) {
+        if (source.link) {
+            // For topStories, only add if not already in organic links
+            if (existingLinksSet && existingLinksSet.has(source.link)) {
+                continue;
+            }
+            links.push(source.link);
+            if (existingLinksSet) {
+                existingLinksSet.add(source.link);
+            }
+            sourceMap.set(source.link, source);
+        }
+    }
+    return links;
+}
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent(sources, sourceMap) {
+    for (let i = 0; i < sources.length; i++) {
+        const source = sources[i];
+        const updatedSource = sourceMap.get(source.link);
+        if (updatedSource) {
+            sources[i] = {
+                ...source,
+                ...updatedSource,
+            };
+        }
+    }
+}
+//# sourceMappingURL=search.js.map