@illuma-ai/agents 1.1.21 → 1.1.23

package/src/tools/search/highlights.js

@@ -1,241 +0,0 @@
-// 2. Pre-compile all regular expressions (only do this once)
-// Group patterns by priority for early returns
-const priorityPatterns = [
-    // High priority patterns (structural)
-    [
-        { regex: /\n\n/g }, // Double newline (paragraph break)
-        { regex: /\n/g }, // Single newline
-        { regex: /={3,}\s*\n|-{3,}\s*\n/g }, // Section separators
-    ],
-    // Medium priority (semantic)
-    [
-        { regex: /[.!?][")\]]?\s/g }, // End of sentence
-        { regex: /;\s/g }, // Semicolon
-        { regex: /:\s/g }, // Colon
-    ],
-    // Low priority (any breaks)
-    [
-        { regex: /,\s/g }, // Comma
-        { regex: /\s-\s/g }, // Dash surrounded by spaces
-        { regex: /\s/g }, // Any space
-    ],
-];
-function findFirstMatch(text, regex) {
-    // Reset regex
-    regex.lastIndex = 0;
-    // For very long texts, try chunking
-    if (text.length > 10000) {
-        const chunkSize = 2000;
-        let position = 0;
-        while (position < text.length) {
-            const chunk = text.substring(position, position + chunkSize);
-            regex.lastIndex = 0;
-            const match = regex.exec(chunk);
-            if (match) {
-                return position + match.index;
-            }
-            // Move to next chunk with some overlap
-            position += chunkSize - 100;
-            if (position >= text.length)
-                break;
-        }
-        return -1;
-    }
-    // For shorter texts, normal regex search
-    const match = regex.exec(text);
-    return match ? match.index : -1;
-}
-// 3. Optimized boundary finding functions
-function findLastMatch(text, regex) {
-    // Reset regex state
-    regex.lastIndex = 0;
-    let lastIndex = -1;
-    let lastLength = 0;
-    let match;
-    // For very long texts, use a different approach to avoid regex engine slowdowns
-    if (text.length > 10000) {
-        // Try dividing the text into chunks for faster processing
-        const chunkSize = 2000;
-        let startPosition = Math.max(0, text.length - chunkSize);
-        while (startPosition >= 0) {
-            const chunk = text.substring(startPosition, startPosition + chunkSize);
-            regex.lastIndex = 0;
-            let chunkLastIndex = -1;
-            let chunkLastLength = 0;
-            while ((match = regex.exec(chunk)) !== null) {
-                chunkLastIndex = match.index;
-                chunkLastLength = match[0].length;
-            }
-            if (chunkLastIndex !== -1) {
-                return startPosition + chunkLastIndex + chunkLastLength;
-            }
-            // Move to previous chunk with some overlap
-            startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;
-            if (startPosition <= 0)
-                break;
-        }
-        return -1;
-    }
-    // For shorter texts, normal regex search
-    while ((match = regex.exec(text)) !== null) {
-        lastIndex = match.index;
-        lastLength = match[0].length;
-    }
-    return lastIndex === -1 ? -1 : lastIndex + lastLength;
-}
-// 4. Find the best boundary with priority groups
-function findBestBoundary(text, direction = 'backward') {
-    if (!text || text.length === 0)
-        return 0;
-    // Try each priority group
-    for (const patternGroup of priorityPatterns) {
-        for (const pattern of patternGroup) {
-            const position = direction === 'backward'
-                ? findLastMatch(text, pattern.regex)
-                : findFirstMatch(text, pattern.regex);
-            if (position !== -1) {
-                return position;
-            }
-        }
-    }
-    // No match found, use character boundary
-    return direction === 'backward' ? text.length : 0;
-}
-/**
- * Tracks references used in a highlight without changing their numbers
- */
-function trackReferencesInHighlight(text, sourceResult // Source containing the original references
-) {
-    // Track used references
-    const references = [];
-    if (!text || text.length === 0 || !text.includes('#')) {
-        return { references }; // Early return
-    }
-    // Quick check for reference markers
-    if (!text.includes('link#') &&
-        !text.includes('image#') &&
-        !text.includes('video#')) {
-        return { references };
-    }
-    // Get references from the source if available
-    const sourceRefs = sourceResult.references || {
-        links: [],
-        images: [],
-        videos: [],
-    };
-    // Find references but don't modify text
-    const refRegex = /\((link|image|video)#(\d+)(?:\s+"([^"]*)")?\)/g;
-    let match;
-    while ((match = refRegex.exec(text)) !== null) {
-        const [, type, indexStr] = match;
-        const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based
-        // Get the source array for this type
-        const refType = type;
-        const sourceArray = sourceRefs[`${refType}s`];
-        // Skip if invalid reference
-        if (!sourceArray ||
-            originalIndex < 0 ||
-            originalIndex >= sourceArray.length) {
-            continue; // Skip invalid references
-        }
-        // Get original reference
-        const reference = sourceArray[originalIndex];
-        // Track if not already tracked
-        const alreadyTracked = references.some((ref) => ref.type === refType && ref.originalIndex === originalIndex);
-        if (!alreadyTracked) {
-            references.push({
-                type: refType,
-                originalIndex,
-                reference,
-            });
-        }
-    }
-    return { references };
-}
-/**
- * Expand highlights in search results using smart boundary detection.
- *
- * This implementation finds natural text boundaries like paragraphs, sentences,
- * and phrases to provide context while maintaining readability.
- *
- * @param searchResults - Search results object
- * @param mainExpandBy - Primary expansion size on each side (default: 300)
- * @param separatorExpandBy - Additional range to look for separators (default: 150)
- * @returns Copy of search results with expanded highlights and tracked references
- */
-export function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy = 150) {
-    // Avoid deep copy - only copy what we modify
-    const resultCopy = { ...searchResults };
-    if (resultCopy.organic)
-        resultCopy.organic = [...resultCopy.organic];
-    if (resultCopy.topStories)
-        resultCopy.topStories = [...resultCopy.topStories];
-    // Process the results efficiently
-    const processResultTypes = ['organic', 'topStories'];
-    for (const resultType of processResultTypes) {
-        if (!resultCopy[resultType])
-            continue;
-        // Map results to new array with modified highlights
-        resultCopy[resultType] = resultCopy[resultType]?.map((result) => {
-            if (result.content == null ||
-                result.content === '' ||
-                !result.highlights ||
-                result.highlights.length === 0) {
-                return result; // No modification needed
-            }
-            // Create a shallow copy with expanded highlights
-            const resultCopy = { ...result };
-            const content = result.content;
-            const highlights = [];
-            // Process each highlight
-            for (const highlight of result.highlights) {
-                const { references } = trackReferencesInHighlight(highlight.text, result);
-                let startPos = content.indexOf(highlight.text);
-                let highlightLen = highlight.text.length;
-                if (startPos === -1) {
-                    // Try with stripped whitespace
-                    const strippedHighlight = highlight.text.trim();
-                    startPos = content.indexOf(strippedHighlight);
-                    if (startPos === -1) {
-                        highlights.push({
-                            text: highlight.text,
-                            score: highlight.score,
-                            references,
-                        });
-                        continue;
-                    }
-                    highlightLen = strippedHighlight.length;
-                }
-                // Calculate boundaries
-                const mainStart = Math.max(0, startPos - mainExpandBy);
-                const mainEnd = Math.min(content.length, startPos + highlightLen + mainExpandBy);
-                const separatorStart = Math.max(0, mainStart - separatorExpandBy);
-                const separatorEnd = Math.min(content.length, mainEnd + separatorExpandBy);
-                // Extract text segments
-                const headText = content.substring(separatorStart, mainStart);
-                const tailText = content.substring(mainEnd, separatorEnd);
-                // Find natural boundaries
-                const bestHeadBoundary = findBestBoundary(headText, 'backward');
-                const bestTailBoundary = findBestBoundary(tailText, 'forward');
-                // Calculate final positions
-                const finalStart = separatorStart + bestHeadBoundary;
-                const finalEnd = mainEnd + bestTailBoundary;
-                // Extract the expanded highlight
-                const expandedHighlightText = content
-                    .substring(finalStart, finalEnd)
-                    .trim();
-                highlights.push({
-                    text: expandedHighlightText,
-                    score: highlight.score,
-                    references,
-                });
-            }
-            resultCopy.highlights = highlights;
-            delete resultCopy.content;
-            delete resultCopy.references;
-            return resultCopy;
-        });
-    }
-    return resultCopy;
-}
-//# sourceMappingURL=highlights.js.map

package/src/tools/search/index.js

@@ -1,3 +0,0 @@
-export * from './tool';
-export * from './schema';
-//# sourceMappingURL=index.js.map

package/src/tools/search/jina-reranker.test.js

@@ -1,106 +0,0 @@
-import { JinaReranker, createReranker } from './rerankers';
-import { createDefaultLogger } from './utils';
-// Helper to access private apiUrl property for testing
-const getApiUrl = (reranker) => reranker.apiUrl;
-describe('JinaReranker', () => {
-    const mockLogger = createDefaultLogger();
-    describe('constructor', () => {
-        it('should use default API URL when no apiUrl is provided', () => {
-            const reranker = new JinaReranker({
-                apiKey: 'test-key',
-                logger: mockLogger,
-            });
-            // Access private property for testing
-            const apiUrl = getApiUrl(reranker);
-            expect(apiUrl).toBe('https://api.jina.ai/v1/rerank');
-        });
-        it('should use custom API URL when provided', () => {
-            const customUrl = 'https://custom-jina-endpoint.com/v1/rerank';
-            const reranker = new JinaReranker({
-                apiKey: 'test-key',
-                apiUrl: customUrl,
-                logger: mockLogger,
-            });
-            const apiUrl = getApiUrl(reranker);
-            expect(apiUrl).toBe(customUrl);
-        });
-        it('should use environment variable JINA_API_URL when available', () => {
-            const originalEnv = process.env.JINA_API_URL;
-            process.env.JINA_API_URL = 'https://env-jina-endpoint.com/v1/rerank';
-            const reranker = new JinaReranker({
-                apiKey: 'test-key',
-                logger: mockLogger,
-            });
-            const apiUrl = getApiUrl(reranker);
-            expect(apiUrl).toBe('https://env-jina-endpoint.com/v1/rerank');
-            // Restore original environment
-            if (originalEnv != null && originalEnv !== '') {
-                process.env.JINA_API_URL = originalEnv;
-            }
-            else {
-                delete process.env.JINA_API_URL;
-            }
-        });
-        it('should prioritize explicit apiUrl over environment variable', () => {
-            const originalEnv = process.env.JINA_API_URL;
-            process.env.JINA_API_URL = 'https://env-jina-endpoint.com/v1/rerank';
-            const customUrl = 'https://explicit-jina-endpoint.com/v1/rerank';
-            const reranker = new JinaReranker({
-                apiKey: 'test-key',
-                apiUrl: customUrl,
-                logger: mockLogger,
-            });
-            const apiUrl = getApiUrl(reranker);
-            expect(apiUrl).toBe(customUrl);
-            // Restore original environment
-            if (originalEnv != null && originalEnv !== '') {
-                process.env.JINA_API_URL = originalEnv;
-            }
-            else {
-                delete process.env.JINA_API_URL;
-            }
-        });
-    });
-    describe('rerank method', () => {
-        it('should log the API URL being used', async () => {
-            const customUrl = 'https://test-jina-endpoint.com/v1/rerank';
-            const reranker = new JinaReranker({
-                apiKey: 'test-key',
-                apiUrl: customUrl,
-                logger: mockLogger,
-            });
-            const logSpy = jest.spyOn(mockLogger, 'debug');
-            try {
-                await reranker.rerank('test query', ['document1', 'document2'], 2);
-            }
-            catch (_error) {
-                // Expected to fail due to missing API key, but we can check the log
-            }
-            expect(logSpy).toHaveBeenCalledWith(expect.stringContaining(`Reranking 2 chunks with Jina using API URL: ${customUrl}`));
-            logSpy.mockRestore();
-        });
-    });
-});
-describe('createReranker', () => {
-    it('should create JinaReranker with jinaApiUrl when provided', () => {
-        const customUrl = 'https://custom-jina-endpoint.com/v1/rerank';
-        const reranker = createReranker({
-            rerankerType: 'jina',
-            jinaApiKey: 'test-key',
-            jinaApiUrl: customUrl,
-        });
-        expect(reranker).toBeInstanceOf(JinaReranker);
-        const apiUrl = getApiUrl(reranker);
-        expect(apiUrl).toBe(customUrl);
-    });
-    it('should create JinaReranker with default URL when jinaApiUrl is not provided', () => {
-        const reranker = createReranker({
-            rerankerType: 'jina',
-            jinaApiKey: 'test-key',
-        });
-        expect(reranker).toBeInstanceOf(JinaReranker);
-        const apiUrl = getApiUrl(reranker);
-        expect(apiUrl).toBe('https://api.jina.ai/v1/rerank');
-    });
-});
-//# sourceMappingURL=jina-reranker.test.js.map

package/src/tools/search/rerankers.js

@@ -1,165 +0,0 @@
-import axios from 'axios';
-import { createDefaultLogger } from './utils';
-export class BaseReranker {
-    apiKey;
-    logger;
-    constructor(logger) {
-        // Each specific reranker will set its API key
-        this.logger = logger || createDefaultLogger();
-    }
-    getDefaultRanking(documents, topK) {
-        return documents
-            .slice(0, Math.min(topK, documents.length))
-            .map((doc) => ({ text: doc, score: 0 }));
-    }
-}
-export class JinaReranker extends BaseReranker {
-    apiUrl;
-    constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = process.env.JINA_API_URL || 'https://api.jina.ai/v1/rerank', logger, }) {
-        super(logger);
-        this.apiKey = apiKey;
-        this.apiUrl = apiUrl;
-    }
-    async rerank(query, documents, topK = 5) {
-        this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);
-        try {
-            if (this.apiKey == null || this.apiKey === '') {
-                this.logger.warn('JINA_API_KEY is not set. Using default ranking.');
-                return this.getDefaultRanking(documents, topK);
-            }
-            const requestData = {
-                model: 'jina-reranker-v2-base-multilingual',
-                query: query,
-                top_n: topK,
-                documents: documents,
-                return_documents: true,
-            };
-            const response = await axios.post(this.apiUrl, requestData, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    Authorization: `Bearer ${this.apiKey}`,
-                },
-            });
-            this.logger.debug('Jina API Model:', response.data?.model);
-            this.logger.debug('Jina API Usage:', response.data?.usage);
-            if (response.data && response.data.results.length) {
-                return response.data.results.map((result) => {
-                    const docIndex = result.index;
-                    const score = result.relevance_score;
-                    let text = '';
-                    // If return_documents is true, the document field will be present
-                    if (result.document != null) {
-                        const doc = result.document;
-                        if (typeof doc === 'object' && 'text' in doc) {
-                            text = doc.text;
-                        }
-                        else if (typeof doc === 'string') {
-                            text = doc;
-                        }
-                    }
-                    else {
-                        // Otherwise, use the index to get the document
-                        text = documents[docIndex];
-                    }
-                    return { text, score };
-                });
-            }
-            else {
-                this.logger.warn('Unexpected response format from Jina API. Using default ranking.');
-                return this.getDefaultRanking(documents, topK);
-            }
-        }
-        catch (error) {
-            this.logger.error('Error using Jina reranker:', error);
-            // Fallback to default ranking on error
-            return this.getDefaultRanking(documents, topK);
-        }
-    }
-}
-export class CohereReranker extends BaseReranker {
-    constructor({ apiKey = process.env.COHERE_API_KEY, logger, }) {
-        super(logger);
-        this.apiKey = apiKey;
-    }
-    async rerank(query, documents, topK = 5) {
-        this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);
-        try {
-            if (this.apiKey == null || this.apiKey === '') {
-                this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');
-                return this.getDefaultRanking(documents, topK);
-            }
-            const requestData = {
-                model: 'rerank-v3.5',
-                query: query,
-                top_n: topK,
-                documents: documents,
-            };
-            const response = await axios.post('https://api.cohere.com/v2/rerank', requestData, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    Authorization: `Bearer ${this.apiKey}`,
-                },
-            });
-            this.logger.debug('Cohere API ID:', response.data?.id);
-            this.logger.debug('Cohere API Meta:', response.data?.meta);
-            if (response.data && response.data.results.length) {
-                return response.data.results.map((result) => {
-                    const docIndex = result.index;
-                    const score = result.relevance_score;
-                    const text = documents[docIndex];
-                    return { text, score };
-                });
-            }
-            else {
-                this.logger.warn('Unexpected response format from Cohere API. Using default ranking.');
-                return this.getDefaultRanking(documents, topK);
-            }
-        }
-        catch (error) {
-            this.logger.error('Error using Cohere reranker:', error);
-            // Fallback to default ranking on error
-            return this.getDefaultRanking(documents, topK);
-        }
-    }
-}
-export class InfinityReranker extends BaseReranker {
-    constructor(logger) {
-        super(logger);
-        // No API key needed for the placeholder implementation
-    }
-    async rerank(query, documents, topK = 5) {
-        this.logger.debug(`Reranking ${documents.length} chunks with Infinity (placeholder)`);
-        // This would be replaced with actual Infinity reranker implementation
-        return this.getDefaultRanking(documents, topK);
-    }
-}
-/**
- * Creates the appropriate reranker based on type and configuration
- */
-export const createReranker = (config) => {
-    const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;
-    // Create a default logger if none is provided
-    const defaultLogger = logger || createDefaultLogger();
-    switch (rerankerType.toLowerCase()) {
-        case 'jina':
-            return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });
-        case 'cohere':
-            return new CohereReranker({
-                apiKey: cohereApiKey,
-                logger: defaultLogger,
-            });
-        case 'infinity':
-            return new InfinityReranker(defaultLogger);
-        case 'none':
-            defaultLogger.debug('Skipping reranking as reranker is set to "none"');
-            return undefined;
-        default:
-            defaultLogger.warn(`Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`);
-            return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });
-    }
-};
-// Example usage:
-// const jinaReranker = new JinaReranker();
-// const cohereReranker = new CohereReranker();
-// const infinityReranker = new InfinityReranker();
-//# sourceMappingURL=rerankers.js.map

package/src/tools/search/schema.js

@@ -1,102 +0,0 @@
-export var DATE_RANGE;
-(function (DATE_RANGE) {
-    DATE_RANGE["PAST_HOUR"] = "h";
-    DATE_RANGE["PAST_24_HOURS"] = "d";
-    DATE_RANGE["PAST_WEEK"] = "w";
-    DATE_RANGE["PAST_MONTH"] = "m";
-    DATE_RANGE["PAST_YEAR"] = "y";
-})(DATE_RANGE || (DATE_RANGE = {}));
-export const DEFAULT_QUERY_DESCRIPTION = `
-GUIDELINES:
-- Start broad, then narrow: Begin with key concepts, then refine with specifics
-- Think like sources: Use terminology experts would use in the field
-- Consider perspective: Frame queries from different viewpoints for better results
-- Quality over quantity: A precise 3-4 word query often beats lengthy sentences
-
-TECHNIQUES (combine for power searches):
-- EXACT PHRASES: Use quotes ("climate change report")
-- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)
-- SITE-SPECIFIC: Restrict to websites (site:edu research)
-- FILETYPE: Find specific documents (filetype:pdf study)
-- OR OPERATOR: Find alternatives (electric OR hybrid cars)
-- DATE RANGE: Recent information (data after:2020)
-- WILDCARDS: Use * for unknown terms (how to * bread)
-- SPECIFIC QUESTIONS: Use who/what/when/where/why/how
-- DOMAIN TERMS: Include technical terminology for specialized topics
-- CONCISE TERMS: Prioritize keywords over sentences
-`.trim();
-export const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
-Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
-Provide this when the search should return results specific to a particular country.
-Examples:
-- "us" for United States (default)
-- "de" for Germany
-- "in" for India
-`.trim();
-export const querySchema = {
-    type: 'string',
-    description: DEFAULT_QUERY_DESCRIPTION,
-};
-export const dateSchema = {
-    type: 'string',
-    enum: Object.values(DATE_RANGE),
-    description: 'Date range for search results.',
-};
-export const countrySchema = {
-    type: 'string',
-    description: DEFAULT_COUNTRY_DESCRIPTION,
-};
-export const imagesSchema = {
-    type: 'boolean',
-    description: 'Whether to also run an image search.',
-};
-export const videosSchema = {
-    type: 'boolean',
-    description: 'Whether to also run a video search.',
-};
-export const newsSchema = {
-    type: 'boolean',
-    description: 'Whether to also run a news search.',
-};
-/** Combined web search tool schema with all properties */
-export const WebSearchToolSchema = {
-    type: 'object',
-    properties: {
-        query: querySchema,
-        date: dateSchema,
-        country: countrySchema,
-        images: imagesSchema,
-        videos: videosSchema,
-        news: newsSchema,
-    },
-    required: ['query'],
-};
-export const WebSearchToolName = 'web_search';
-export const WebSearchToolDescription = `Real-time search. Results have required citation anchors.
-
-Note: Use ONCE per reply unless instructed otherwise.
-
-Anchors:
-- \\ue202turnXtypeY
-- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
-
-Special Markers:
-- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
-- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
-
-**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
-Use anchor marker(s) immediately after the statement:
-- Standalone: "Pure functions produce same output. \\ue202turn0search0"
-- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
-- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
-- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Image: "See photo \\ue202turn0image0."
-
-**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**`;
-export const WebSearchToolDefinition = {
-    name: WebSearchToolName,
-    description: WebSearchToolDescription,
-    schema: WebSearchToolSchema,
-};
-//# sourceMappingURL=schema.js.map