erosolar-cli 1.7.55 → 1.7.57

package/dist/bin/tools/webTools.js

@@ -1,480 +0,0 @@
-import * as https from 'node:https';
-import * as http from 'node:http';
-import { getSecretValue } from '../core/secretStore.js';
-export function createWebTools() {
-    return [
-        {
-            name: 'WebFetch',
-            description: `- Fetches content from a specified URL and processes it using an AI model
-- Takes a URL and a prompt as input
-- Fetches the URL content, converts HTML to markdown
-- Processes the content with the prompt using a small, fast model
-- Returns the model's response about the content
-- Use this tool when you need to retrieve and analyze web content
-
-Usage notes:
-  - IMPORTANT: If TAVILY_API_KEY is set, use WebExtract instead for better content extraction
-  - The URL must be a fully-formed valid URL
-  - HTTP URLs will be automatically upgraded to HTTPS
-  - The prompt should describe what information you want to extract from the page
-  - This tool is read-only and does not modify any files
-  - Results may be summarized if the content is very large
-  - Includes a self-cleaning 15-minute cache for faster responses when repeatedly accessing the same URL
-  - When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new WebFetch request with the redirect URL to fetch the content.`,
-            parameters: {
-                type: 'object',
-                properties: {
-                    url: {
-                        type: 'string',
-                        description: 'The URL to fetch content from',
-                    },
-                    prompt: {
-                        type: 'string',
-                        description: 'The prompt to run on the fetched content',
-                    },
-                },
-                required: ['url', 'prompt'],
-            },
-            handler: async (args) => {
-                const url = args['url'];
-                const prompt = args['prompt'];
-                if (!url || !prompt) {
-                    return 'Error: url and prompt parameters are required.';
-                }
-                try {
-                    // Upgrade HTTP to HTTPS
-                    const targetUrl = url.replace(/^http:\/\//, 'https://');
-                    const content = await fetchUrl(targetUrl);
-                    // Simple HTML to markdown conversion (basic implementation)
-                    const markdown = htmlToMarkdown(content);
-                    // Process with prompt (in a real implementation, this would use a small LLM)
-                    // For now, we'll return the content with the prompt context
-                    return `Fetched content from ${targetUrl}
-
-Prompt: ${prompt}
-
-Content (first 5000 characters):
-${markdown.slice(0, 5000)}${markdown.length > 5000 ? '\n\n... (content truncated)' : ''}
-
-Summary: This is the content fetched from the URL. In a full implementation, this would be processed by a small LLM to answer the specific prompt.`;
-                }
-                catch (error) {
-                    return `Error fetching URL: ${error instanceof Error ? error.message : String(error)}`;
-                }
-            },
-        },
-        {
-            name: 'WebExtract',
-            description: `- Extracts clean, structured content from one or more URLs using Tavily Extract API
-- Superior to WebFetch for content extraction when TAVILY_API_KEY is available
-- Returns raw text content optimized for LLM consumption
-- Supports batch extraction of up to 20 URLs in a single call
-
-Usage notes:
-  - Requires TAVILY_API_KEY environment variable
-  - Best for extracting article content, documentation, blog posts
-  - Returns clean text without HTML artifacts
-  - More reliable than basic HTML parsing for complex pages
-  - Use for deep content extraction when you need full page text`,
-            parameters: {
-                type: 'object',
-                properties: {
-                    urls: {
-                        type: 'array',
-                        items: { type: 'string' },
-                        description: 'Array of URLs to extract content from (max 20)',
-                    },
-                },
-                required: ['urls'],
-            },
-            handler: async (args) => {
-                const urls = args['urls'];
-                if (!urls || !Array.isArray(urls) || urls.length === 0) {
-                    return 'Error: urls parameter is required and must be a non-empty array.';
-                }
-                if (urls.length > 20) {
-                    return 'Error: Maximum 20 URLs allowed per request.';
-                }
-                const tavilyKey = getSecretValue('TAVILY_API_KEY') || process.env['TAVILY_API_KEY']?.trim();
-                if (!tavilyKey) {
-                    return [
-                        'WebExtract requires TAVILY_API_KEY to be set.',
-                        'Get your API key at: https://tavily.com',
-                        'Use /secrets to configure this value.',
-                        '',
-                        'Falling back to WebFetch for basic extraction is available as an alternative.',
-                    ].join('\n');
-                }
-                try {
-                    const response = await fetch('https://api.tavily.com/extract', {
-                        method: 'POST',
-                        headers: {
-                            'Content-Type': 'application/json',
-                        },
-                        body: JSON.stringify({
-                            api_key: tavilyKey,
-                            urls: urls,
-                        }),
-                    });
-                    if (!response.ok) {
-                        const errorText = await response.text().catch(() => '');
-                        throw new Error(`Tavily Extract returned HTTP ${response.status}: ${errorText}`);
-                    }
-                    const payload = (await response.json());
-                    const results = payload.results || [];
-                    const failedUrls = payload.failed_results || [];
-                    if (results.length === 0 && failedUrls.length === urls.length) {
-                        return `Failed to extract content from all ${urls.length} URLs. This may be due to access restrictions or invalid URLs.`;
-                    }
-                    let output = `Extracted content from ${results.length}/${urls.length} URLs:\n\n`;
-                    for (const result of results) {
-                        output += `--- ${result.url} ---\n`;
-                        if (result.raw_content) {
-                            // Truncate very long content
-                            const content = result.raw_content.length > 10000
-                                ? result.raw_content.slice(0, 10000) + '\n\n... (content truncated)'
-                                : result.raw_content;
-                            output += `${content}\n\n`;
-                        }
-                        else {
-                            output += '(No content extracted)\n\n';
-                        }
-                    }
-                    if (failedUrls.length > 0) {
-                        output += `\nFailed URLs (${failedUrls.length}):\n`;
-                        for (const failed of failedUrls) {
-                            output += `- ${failed.url}: ${failed.error || 'Unknown error'}\n`;
-                        }
-                    }
-                    return output.trim();
-                }
-                catch (error) {
-                    return `Error extracting content: ${error instanceof Error ? error.message : String(error)}`;
-                }
-            },
-        },
-        {
-            name: 'WebSearch',
-            description: `- Allows Claude to search the web and use the results to inform responses
-- Provides up-to-date information for current events and recent data
-- Returns search result information formatted as search result blocks
-- Use this tool for accessing information beyond Claude's knowledge cutoff
-- Searches are performed automatically within a single API call
-
-Usage notes:
-  - Domain filtering is supported to include or block specific websites
-  - Web search is only available in the US
-  - Account for "Today's date" in <env>. For example, if <env> says "Today's date: 2025-07-01", and the user wants the latest docs, do not use 2024 in the search query. Use 2025.`,
-            parameters: {
-                type: 'object',
-                properties: {
-                    query: {
-                        type: 'string',
-                        description: 'The search query to use',
-                    },
-                    allowed_domains: {
-                        type: 'array',
-                        items: {
-                            type: 'string',
-                        },
-                        description: 'Only include search results from these domains',
-                    },
-                    blocked_domains: {
-                        type: 'array',
-                        items: {
-                            type: 'string',
-                        },
-                        description: 'Never include search results from these domains',
-                    },
-                },
-                required: ['query'],
-            },
-            handler: async (args) => {
-                const query = typeof args['query'] === 'string' ? args['query'].trim() : '';
-                const allowed = parseDomainList(args['allowed_domains']);
-                const blocked = parseDomainList(args['blocked_domains']);
-                if (!query) {
-                    return 'Error: query parameter is required.';
-                }
-                try {
-                    const provider = resolveSearchProvider();
-                    if (!provider) {
-                        return [
-                            'WebSearch requires TAVILY_API_KEY (recommended), BRAVE_SEARCH_API_KEY, or SERPAPI_API_KEY.',
-                            'Run /secrets (or set the environment variables directly) to configure an API key.',
-                            '',
-                            'Get your Tavily API key at: https://tavily.com (recommended)',
-                            'Get your Brave Search API key at: https://brave.com/search/api/',
-                            'Get your SerpAPI key at: https://serpapi.com/',
-                        ].join('\n');
-                    }
-                    const results = await provider.search({
-                        query,
-                        allowedDomains: allowed,
-                        blockedDomains: blocked,
-                        maxResults: 6,
-                    });
-                    if (!results.length) {
-                        return `No web results found for "${query}" ${formatFilterSummary(allowed, blocked)}.`;
-                    }
-                    return formatSearchResults(query, results, provider.label, allowed, blocked);
-                }
-                catch (error) {
-                    return `Error performing web search: ${error instanceof Error ? error.message : String(error)}`;
-                }
-            },
-        },
-    ];
-}
-function resolveSearchProvider() {
-    // Tavily is the preferred search provider - check secretStore then env
-    const tavilyKey = getSecretValue('TAVILY_API_KEY') || process.env['TAVILY_API_KEY']?.trim();
-    if (tavilyKey) {
-        return {
-            id: 'tavily',
-            label: 'Tavily Search',
-            search: (params) => performTavilySearch(params, tavilyKey),
-        };
-    }
-    const braveKey = getSecretValue('BRAVE_SEARCH_API_KEY') || process.env['BRAVE_SEARCH_API_KEY']?.trim();
-    if (braveKey) {
-        return {
-            id: 'brave',
-            label: 'Brave Search',
-            search: (params) => performBraveSearch(params, braveKey),
-        };
-    }
-    const serpKey = getSecretValue('SERPAPI_API_KEY') || process.env['SERPAPI_API_KEY']?.trim();
-    if (serpKey) {
-        return {
-            id: 'serpapi',
-            label: 'SerpAPI (Google)',
-            search: (params) => performSerpApiSearch(params, serpKey),
-        };
-    }
-    return null;
-}
-async function performBraveSearch(params, apiKey) {
-    const url = new URL('https://api.search.brave.com/res/v1/web/search');
-    url.searchParams.set('q', params.query);
-    url.searchParams.set('count', String(Math.min(params.maxResults * 2, 20)));
-    const response = await fetch(url, {
-        headers: {
-            Accept: 'application/json',
-            'X-Subscription-Token': apiKey,
-        },
-    });
-    if (!response.ok) {
-        throw new Error(`Brave Search returned HTTP ${response.status}`);
-    }
-    const payload = (await response.json());
-    const entries = Array.isArray(payload?.web?.results) ? payload.web.results : [];
-    const mapped = entries
-        .map((entry) => ({
-        title: entry.title || entry.url,
-        url: entry.url,
-        snippet: entry.description || entry.snippet || '',
-        source: entry.profile?.name || entry.source || safeHostname(entry.url) || undefined,
-        published: entry.publishedDate || entry.subtype,
-    }))
-        .filter((result) => Boolean(result.url));
-    return applyDomainFilters(mapped, params.allowedDomains, params.blockedDomains).slice(0, params.maxResults);
-}
-async function performSerpApiSearch(params, apiKey) {
-    const url = new URL('https://serpapi.com/search.json');
-    url.searchParams.set('engine', 'google');
-    url.searchParams.set('q', params.query);
-    url.searchParams.set('num', String(Math.min(params.maxResults * 2, 10)));
-    url.searchParams.set('api_key', apiKey);
-    const response = await fetch(url);
-    if (!response.ok) {
-        throw new Error(`SerpAPI returned HTTP ${response.status}`);
-    }
-    const payload = (await response.json());
-    const entries = Array.isArray(payload?.organic_results) ? payload.organic_results : [];
-    const mapped = entries
-        .map((entry) => ({
-        title: entry.title || entry.link,
-        url: entry.link,
-        snippet: entry.snippet || (Array.isArray(entry.snippet_highlighted_words) ? entry.snippet_highlighted_words.join(' ') : ''),
-        source: entry.source || entry.display_link || entry.displayed_link || safeHostname(entry.link) || undefined,
-        published: entry.date || entry.snippet_date,
-    }))
-        .filter((result) => Boolean(result.url));
-    return applyDomainFilters(mapped, params.allowedDomains, params.blockedDomains).slice(0, params.maxResults);
-}
-async function performTavilySearch(params, apiKey) {
-    const response = await fetch('https://api.tavily.com/search', {
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-            api_key: apiKey,
-            query: params.query,
-            search_depth: 'advanced',
-            include_answer: true,
-            include_raw_content: false,
-            max_results: Math.min(params.maxResults * 2, 10),
-            include_domains: params.allowedDomains.length ? params.allowedDomains : undefined,
-            exclude_domains: params.blockedDomains.length ? params.blockedDomains : undefined,
-        }),
-    });
-    if (!response.ok) {
-        const errorText = await response.text().catch(() => '');
-        throw new Error(`Tavily Search returned HTTP ${response.status}: ${errorText}`);
-    }
-    const payload = (await response.json());
-    const entries = Array.isArray(payload?.results) ? payload.results : [];
-    const mapped = entries
-        .map((entry) => ({
-        title: entry.title || entry.url,
-        url: entry.url,
-        snippet: entry.content || '',
-        source: safeHostname(entry.url) || undefined,
-        published: entry.published_date,
-        score: entry.score,
-    }))
-        .filter((result) => Boolean(result.url));
-    // Tavily already handles domain filtering via include_domains/exclude_domains
-    return mapped.slice(0, params.maxResults);
-}
-function parseDomainList(value) {
-    if (!Array.isArray(value)) {
-        return [];
-    }
-    return value
-        .map((entry) => (typeof entry === 'string' ? entry.trim().toLowerCase() : ''))
-        .filter(Boolean);
-}
-function applyDomainFilters(results, allowedDomains, blockedDomains) {
-    const normalizedAllowed = allowedDomains.map((domain) => domain.startsWith('.') ? domain.slice(1) : domain);
-    const normalizedBlocked = blockedDomains.map((domain) => domain.startsWith('.') ? domain.slice(1) : domain);
-    return results.filter((result) => {
-        const hostname = safeHostname(result.url);
-        if (!hostname) {
-            return false;
-        }
-        if (normalizedAllowed.length && !normalizedAllowed.some((domain) => hostname === domain || hostname.endsWith(`.${domain}`))) {
-            return false;
-        }
-        if (normalizedBlocked.some((domain) => hostname === domain || hostname.endsWith(`.${domain}`))) {
-            return false;
-        }
-        return true;
-    });
-}
-function formatSearchResults(query, results, providerLabel, allowed, blocked) {
-    const lines = [`Web Search Results for "${query}" (${providerLabel})`, ''];
-    results.forEach((result, index) => {
-        lines.push(`${index + 1}. ${result.title || result.url}`, result.url);
-        if (result.snippet) {
-            lines.push(`   ${result.snippet}`);
-        }
-        const meta = [];
-        if (result.source) {
-            meta.push(`source: ${result.source}`);
-        }
-        if (result.published) {
-            meta.push(`published: ${result.published}`);
-        }
-        if (meta.length) {
-            lines.push(`   (${meta.join(' · ')})`);
-        }
-        lines.push('');
-    });
-    lines.push(formatFilterSummary(allowed, blocked, providerLabel));
-    return lines.join('\n').trim();
-}
-function formatFilterSummary(allowed, blocked, providerLabel) {
-    const segments = [];
-    if (allowed.length) {
-        segments.push(`allowed: ${allowed.join(', ')}`);
-    }
-    if (blocked.length) {
-        segments.push(`blocked: ${blocked.join(', ')}`);
-    }
-    if (!segments.length) {
-        segments.push('none');
-    }
-    if (providerLabel) {
-        segments.push(`provider: ${providerLabel}`);
-    }
-    return `Filters: ${segments.join(' | ')}`;
-}
-function safeHostname(raw) {
-    if (!raw) {
-        return null;
-    }
-    try {
-        const url = new URL(raw);
-        return url.hostname.toLowerCase();
-    }
-    catch {
-        return null;
-    }
-}
-function fetchUrl(url) {
-    return new Promise((resolve, reject) => {
-        const client = url.startsWith('https://') ? https : http;
-        const request = client
-            .get(url, (res) => {
-            let data = '';
-            // Handle redirects
-            if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-                fetchUrl(res.headers.location)
-                    .then(resolve)
-                    .catch(reject);
-                return;
-            }
-            if (res.statusCode !== 200) {
-                reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
-                return;
-            }
-            res.on('data', (chunk) => {
-                data += chunk;
-            });
-            res.on('end', () => {
-                resolve(data);
-            });
-        })
-            .on('error', (err) => {
-            reject(err);
-        });
-        request.on('error', (err) => {
-            reject(err);
-        });
-    });
-}
-function htmlToMarkdown(html) {
-    // Very basic HTML to markdown conversion
-    let text = html;
-    // Remove script and style tags
-    text = text.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
-    text = text.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
-    // Convert common HTML tags
-    text = text.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
-    text = text.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
-    text = text.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
-    text = text.replace(/<p[^>]*>(.*?)<\/p>/gi, '$1\n\n');
-    text = text.replace(/<br\s*\/?>/gi, '\n');
-    text = text.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
-    text = text.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
-    text = text.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
-    text = text.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
-    text = text.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
-    text = text.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
-    // Remove remaining HTML tags
-    text = text.replace(/<[^>]+>/g, '');
-    // Decode HTML entities
-    text = text
-        .replace(/&nbsp;/g, ' ')
-        .replace(/&amp;/g, '&')
-        .replace(/&lt;/g, '<')
-        .replace(/&gt;/g, '>')
-        .replace(/&quot;/g, '"')
-        .replace(/&#39;/g, "'");
-    // Clean up excessive newlines
-    text = text.replace(/\n{3,}/g, '\n\n');
-    return text.trim();
-}

package/dist/bin/workspace.js

@@ -1,106 +0,0 @@
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { safeWorkspaceContext, validateWorkspaceOptions } from './workspace.validator.js';
-const PRIORITY_DOCS = ['README.md']; // Removed package.json to save context
-const IGNORED_DIRS = new Set([
-    '.git', 'node_modules', 'dist', '.erosolar', 'build', 'coverage', '.next', 'out',
-    '__pycache__', '.pytest_cache', '.mypy_cache', 'venv', '.venv', 'env',
-    'target', 'bin', 'obj', '.idea', '.vscode', '.DS_Store'
-]);
-const DEFAULT_TREE_DEPTH = 1; // Reduced from 2 to 1 for critical context savings
-const DEFAULT_MAX_ENTRIES = 30; // Further reduced from 50 to 30 - emergency reduction
-const DEFAULT_DOC_LIMIT = 200; // Further reduced from 300 to 200 - emergency reduction
-export function resolveWorkspaceCaptureOptions(env = process.env) {
-    return {
-        treeDepth: parsePositiveInt(env['EROSOLAR_CONTEXT_TREE_DEPTH']),
-        maxEntries: parsePositiveInt(env['EROSOLAR_CONTEXT_MAX_ENTRIES']),
-        docExcerptLimit: parsePositiveInt(env['EROSOLAR_CONTEXT_DOC_LIMIT']),
-    };
-}
-export function buildWorkspaceContext(root, options = {}) {
-    // CRITICAL: Validate options BEFORE building context
-    const optionsValidation = validateWorkspaceOptions(options);
-    if (!optionsValidation.valid) {
-        console.error('[Workspace Context] Invalid options:', optionsValidation.errors);
-        throw new Error(`Invalid workspace options: ${optionsValidation.errors.join(', ')}`);
-    }
-    // Log warnings if any
-    if (optionsValidation.warnings.length > 0) {
-        console.warn('[Workspace Context] Warnings:', optionsValidation.warnings);
-    }
-    const treeDepth = options.treeDepth ?? DEFAULT_TREE_DEPTH;
-    const maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
-    const docLimit = options.docExcerptLimit ?? DEFAULT_DOC_LIMIT;
-    try {
-        const treeLines = formatFileTree(root, treeDepth, maxEntries);
-        const docSnippets = capturePriorityDocs(root, docLimit);
-        // REMOVED: Rulebook duplication - rulebooks are already in system prompt
-        const sections = [`cwd: ${root}`, 'files:', ...treeLines];
-        if (docSnippets.length) {
-            sections.push(docSnippets.join('\n\n'));
-        }
-        // REMOVED: Rulebook duplication
-        // if (rulebooks.length) {
-        //   sections.push(rulebooks.join('\n\n'));
-        // }
-        const rawContent = sections.filter((section) => section.trim().length > 0).join('\n');
-        // CRITICAL: Validate and enforce limits on final output
-        const safe = safeWorkspaceContext(rawContent, {
-            truncate: true, // Auto-truncate if too large
-            throwOnError: false // Don't throw, just truncate
-        });
-        // Log stats for monitoring
-        if (process.env['DEBUG_CONTEXT']) {
-            console.log('[Workspace Context] Stats:', safe.stats);
-        }
-        return safe.content;
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        return `Workspace context unavailable (${message}).`;
-    }
-}
-function capturePriorityDocs(root, docLimit) {
-    return PRIORITY_DOCS.filter((name) => existsSync(join(root, name))).map((name) => {
-        try {
-            const content = readFileSync(join(root, name), 'utf8');
-            // Safety: Hard limit to prevent context explosion
-            const safeLimit = Math.min(docLimit, 300);
-            const snippet = content.length > safeLimit ? `${content.slice(0, safeLimit)}\n...` : content;
-            return `--- ${name} ---\n${snippet.trim()}`;
-        }
-        catch {
-            return `--- ${name} ---\n[Could not read file]`;
-        }
-    });
-}
-function formatFileTree(root, maxDepth, maxEntries) {
-    const lines = [];
-    const walk = (dir, depth, prefix) => {
-        if (depth > maxDepth || lines.length >= maxEntries) {
-            return;
-        }
-        const entries = readdirSync(dir, { withFileTypes: true })
-            .filter((entry) => !IGNORED_DIRS.has(entry.name))
-            .sort((a, b) => a.name.localeCompare(b.name));
-        for (const entry of entries) {
-            if (lines.length >= maxEntries) {
-                break;
-            }
-            const isDir = entry.isDirectory();
-            lines.push(`${prefix}${entry.name}${isDir ? '/' : ''}`);
-            if (isDir && depth < maxDepth) {
-                walk(join(dir, entry.name), depth + 1, `${prefix}  `);
-            }
-        }
-    };
-    walk(root, 0, '');
-    return lines;
-}
-function parsePositiveInt(raw) {
-    if (!raw) {
-        return undefined;
-    }
-    const value = Number.parseInt(raw, 10);
-    return Number.isFinite(value) && value > 0 ? value : undefined;
-}