codemini-cli 0.4.1 → 0.4.3

package/src/core/tools.js

@@ -15,7 +15,7 @@ import {
 import { evaluateCommandPolicy } from './command-policy.js';
 import { findEnclosingSymbol, queryAst, readAstNode, resolveAstTarget } from './ast.js';
 import { initializeProjectIndex, queryProjectIndex, refreshIndexedFile } from './project-index.js';
-import { checkReadDedup } from './agent-loop.js';
+import { checkReadDedup } from './tool-result-store.js';
 import { TOOL_SKIP_DIRS as SKIP_DIRS, TEXT_EXTENSIONS, CODE_WRITE_GUARD_EXTENSIONS, LANGUAGE_FILE_TYPES } from './constants.js';
 import { sha256Prefixed as sha256, sha256 as sha256Hash } from './crypto-utils.js';
 import { forgetMemory, listMemories, rememberMemory, searchMemories, captureToInbox } from './memory-store.js';
@@ -172,6 +172,55 @@ function collectPageLinks($, pageUrl, maxLinks = 20) {
   return links;
 }
 
+function extractPageContent(cheerio, html, pageUrl, { maxLinks, status = null, contentType = '', fetchMode = 'static' } = {}) {
+  const $ = cheerio.load(html);
+  $('script, style, noscript').remove();
+  const bodyText = $('body').text() || $.root().text();
+  const text = String(bodyText || '').replace(/[^\S\n]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
+  const title = trimPreview($('title').first().text(), 240);
+  const description = extractHtmlMeta($, 'description') || extractHtmlMeta($, 'og:description');
+  const links = collectPageLinks($, pageUrl, maxLinks);
+
+  return {
+    final_url: pageUrl,
+    title,
+    description,
+    text,
+    links,
+    metadata: {
+      status,
+      fetched_at: new Date().toISOString(),
+      content_type: contentType,
+      fetch_mode: fetchMode,
+      lang: String($('html').attr('lang') || '').trim()
+    }
+  };
+}
+
+function shouldTryBrowserRender(html, text) {
+  if (String(text || '').trim().length >= 120) return false;
+  return /<script\b/i.test(html) ||
+    /id=["']__(?:next|nuxt)["']/i.test(html) ||
+    /data-reactroot|ng-version|window\.__/i.test(html);
+}
+
+function playwrightInstallHint() {
+  return 'For JavaScript-rendered pages, install Playwright for richer web_fetch results: npm install -g playwright && playwright install chromium';
+}
+
+async function loadOptionalPlaywright() {
+  try {
+    return await import('playwright');
+  } catch (error) {
+    const code = String(error?.code || '');
+    const message = String(error?.message || '');
+    if (code === 'ERR_MODULE_NOT_FOUND' || /Cannot find package 'playwright'|Cannot find module 'playwright'/i.test(message)) {
+      return null;
+    }
+    throw error;
+  }
+}
+
 async function buildPlaywrightLaunchEnv() {
   const localLibDir = path.join(
     process.env.HOME || '',
@@ -204,44 +253,85 @@ async function webFetchPage(args = {}) {
     ? String(normalizedArgs.wait_until).trim()
     : 'domcontentloaded';
 
-  const [{ chromium }, cheerio] = await Promise.all([import('playwright'), import('cheerio')]);
+  const cheerio = await import('cheerio');
+  let staticResult = null;
+  let staticHtml = '';
+  let staticError = null;
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+    let response;
+    try {
+      response = await fetch(url, {
+        redirect: 'follow',
+        signal: controller.signal,
+        headers: {
+          'user-agent': 'CodeMiniCLI/0.4 web_fetch'
+        }
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+    staticHtml = await response.text();
+    staticResult = {
+      url,
+      ...extractPageContent(cheerio, staticHtml, response.url || url, {
+        maxLinks,
+        status: response.status,
+        contentType: response.headers.get('content-type') || '',
+        fetchMode: 'static'
+      })
+    };
+    if (!shouldTryBrowserRender(staticHtml, staticResult.text)) {
+      return staticResult;
+    }
+  } catch (error) {
+    staticError = error;
+  }
 
-  // Crawlee is intentionally disabled for now so single-page reads stay lightweight.
-  // If we later need multi-URL crawl orchestration, retries, or request queues, we can re-enable it here.
+  const playwright = await loadOptionalPlaywright();
+  if (!playwright) {
+    if (staticResult) {
+      return {
+        ...staticResult,
+        warnings: [playwrightInstallHint()]
+      };
+    }
+    throw new Error(`web_fetch failed and browser rendering is unavailable. ${playwrightInstallHint()}. Static fetch error: ${staticError?.message || staticError}`);
+  }
 
-  const browser = await chromium.launch({
-    headless: true,
-    env: await buildPlaywrightLaunchEnv()
-  });
+  let browser;
   try {
+    browser = await playwright.chromium.launch({
+      headless: true,
+      env: await buildPlaywrightLaunchEnv()
+    });
     const page = await browser.newPage();
     const response = await page.goto(url, { waitUntil, timeout: timeoutMs });
     const finalUrl = page.url();
     const html = await page.content();
-    const $ = cheerio.load(html);
-    const bodyText = $('body').text() || $.root().text();
-    const text = String(bodyText || '').replace(/[^\S\n]+/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
-    const title = trimPreview($('title').first().text() || (await page.title()), 240);
-    const description = extractHtmlMeta($, 'description') || extractHtmlMeta($, 'og:description');
-    const links = collectPageLinks($, finalUrl, maxLinks);
-
-    return {
+    const rendered = {
       url,
-      final_url: finalUrl,
-      title,
-      description,
-      text,
-      links,
-      metadata: {
+      ...extractPageContent(cheerio, html, finalUrl, {
+        maxLinks,
         status: response?.status?.() ?? null,
-        fetched_at: new Date().toISOString(),
-        content_type: response?.headers?.()['content-type'] || '',
-        wait_until: waitUntil,
-        lang: String($('html').attr('lang') || '').trim()
-      }
+        contentType: response?.headers?.()['content-type'] || '',
+        fetchMode: 'browser'
+      })
     };
+    rendered.metadata.wait_until = waitUntil;
+    rendered.title = rendered.title || trimPreview(await page.title(), 240);
+    return rendered;
+  } catch (error) {
+    if (staticResult) {
+      return {
+        ...staticResult,
+        warnings: [`Browser rendering fallback failed: ${error?.message || error}`]
+      };
+    }
+    throw error;
   } finally {
-    await browser.close();
+    if (browser) await browser.close();
   }
 }
 
@@ -255,30 +345,107 @@ async function webSearchQuery(config, args = {}) {
   if (!query) throw new Error('web_search requires query');
 
   const maxResults = clampNumber(normalizedArgs.max_results, 1, 20, 8);
-  const [{ search, SafeSearchType }] = await Promise.all([import('duck-duck-scrape')]);
-  const response = await search(query, {
-    safeSearch: SafeSearchType.MODERATE,
-    locale: String(normalizedArgs.locale || 'en-us').trim() || 'en-us',
-    region: String(normalizedArgs.region || 'wt-wt').trim() || 'wt-wt'
+  const locale = String(normalizedArgs.locale || config?.web?.search_locale || 'en-US').trim() || 'en-US';
+  const region = String(normalizedArgs.region || normalizedArgs.cc || config?.web?.search_region || (locale.toLowerCase().endsWith('-cn') ? 'CN' : 'US')).trim() || 'US';
+  const searchUrl = buildBingRssSearchUrl({
+    baseUrl: config?.web?.search_base_url,
+    query,
+    locale,
+    region
   });
+  const timeoutMs = clampNumber(normalizedArgs.timeout_ms || config?.web?.search_timeout_ms, 1_000, 60_000, 15_000);
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  let response;
+  try {
+    response = await fetch(searchUrl, {
+      redirect: 'follow',
+      signal: controller.signal,
+      headers: {
+        'user-agent': 'CodeMiniCLI/0.4 web_search',
+        accept: 'application/rss+xml, application/xml;q=0.9, text/xml;q=0.8, */*;q=0.5',
+        'accept-language': `${locale},en;q=0.8`
+      }
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+  if (!response.ok) {
+    throw new Error(`web_search Bing RSS request failed: HTTP ${response.status}`);
+  }
+
+  const xml = await response.text();
+  const cheerio = await import('cheerio');
+  const parsed = parseBingRssResults(cheerio, xml, maxResults);
 
   return {
     query,
-    no_results: response?.noResults === true,
-    results: Array.isArray(response?.results)
-      ? response.results.slice(0, maxResults).map((item) => ({
-          title: String(item?.title || '').trim(),
-          url: String(item?.url || '').trim(),
-          description: normalizeWhitespace(item?.description || item?.rawDescription || ''),
-          hostname: String(item?.hostname || '').trim()
-        }))
-      : [],
-    related: Array.isArray(response?.related)
-      ? response.related.slice(0, 8).map((item) => String(item?.text || item?.raw || '').trim()).filter(Boolean)
-      : []
+    engine: 'bing_rss',
+    source_url: response.url || searchUrl,
+    no_results: parsed.results.length === 0,
+    results: parsed.results,
+    related: []
   };
 }
 
+function buildBingRssSearchUrl({ baseUrl, query, locale, region }) {
+  const url = new URL(String(baseUrl || 'https://cn.bing.com/search'));
+  url.searchParams.set('q', query);
+  url.searchParams.set('mkt', locale);
+  url.searchParams.set('setlang', locale);
+  url.searchParams.set('cc', region);
+  url.searchParams.set('format', 'rss');
+  return url.toString();
+}
+
+function parseBingRssResults(cheerio, xml, maxResults) {
+  const $ = cheerio.load(xml, { xmlMode: true });
+  const results = [];
+  const seenUrls = new Set();
+  $('item').each((_, element) => {
+    if (results.length >= maxResults) return false;
+    const title = normalizeWhitespace($(element).find('title').first().text());
+    const url = normalizeSearchResultUrl($(element).find('link').first().text());
+    if (!title || !url || seenUrls.has(url)) return undefined;
+    seenUrls.add(url);
+    results.push({
+      title,
+      url,
+      description: normalizeRssDescription(cheerio, $(element).find('description').first().text()),
+      hostname: hostnameFromUrl(url),
+      published_at: normalizeWhitespace($(element).find('pubDate').first().text())
+    });
+    return undefined;
+  });
+  return { results };
+}
+
+function normalizeSearchResultUrl(value) {
+  const text = String(value || '').trim();
+  if (!text) return '';
+  try {
+    const parsed = new URL(text);
+    if (!['http:', 'https:'].includes(parsed.protocol)) return '';
+    return parsed.toString();
+  } catch {
+    return '';
+  }
+}
+
+function normalizeRssDescription(cheerio, value) {
+  const text = String(value || '').trim();
+  if (!text) return '';
+  return normalizeWhitespace(cheerio.load(text).text() || text);
+}
+
+function hostnameFromUrl(value) {
+  try {
+    return new URL(value).hostname;
+  } catch {
+    return '';
+  }
+}
+
 function findUniqueLineBlock(lines, blockContent) {
   const probeLines = splitLines(blockContent);
   if (probeLines.length === 0 || (probeLines.length === 1 && probeLines[0] === '')) return null;
@@ -877,7 +1044,7 @@ async function runCommand(root, config, args) {
     command,
     cwd: root,
     shell: config.shell.default,
-    timeoutMs: config.shell.timeout_ms
+    timeoutMs: Number(args?.timeout || args?.timeout_ms || args?.timeoutMs || config.shell.timeout_ms)
   });
   return { ...result, command };
 }
@@ -1752,24 +1919,6 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
         }
       }
     },
-    {
-      type: 'function',
-      function: {
-        name: 'glob',
-        description:
-          'Find files by glob pattern. Use this when you already know a filename pattern such as src/**/*.ts.',
-        parameters: {
-          type: 'object',
-          properties: {
-            pattern: { type: 'string', description: 'Glob pattern' },
-            path: { type: 'string', description: 'Directory to search' },
-            include_hidden: { type: 'boolean', description: 'Include dotfiles' },
-            max_results: { type: 'number', description: 'Max results' }
-          },
-          required: ['pattern']
-        }
-      }
-    },
     {
       type: 'function',
       function: {
@@ -1995,6 +2144,24 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
   ];
 
   const deferredDefinitions = {
+    glob: {
+      type: 'function',
+      function: {
+        name: 'glob',
+        description:
+          'Find files by glob pattern. Use this when you already know a filename pattern such as src/**/*.ts.',
+        parameters: {
+          type: 'object',
+          properties: {
+            pattern: { type: 'string', description: 'Glob pattern' },
+            path: { type: 'string', description: 'Directory to search' },
+            include_hidden: { type: 'boolean', description: 'Include dotfiles' },
+            max_results: { type: 'number', description: 'Max results' }
+          },
+          required: ['pattern']
+        }
+      }
+    },
     ast_query: {
       type: 'function',
       function: {
@@ -2036,7 +2203,7 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
       function: {
         name: 'web_fetch',
         description:
-          'Fetch and read a live web page. Uses Playwright to render the page, Cheerio to extract structured content, and Crawlee request handling to normalize the fetch flow. Use this for direct URL reads, not for keyword search.',
+          'Fetch and read a live web page. Uses a lightweight fetch + Cheerio reader by default, then falls back to optional Playwright browser rendering for JavaScript-heavy pages when Playwright is installed. Use this for direct URL reads, not for keyword search.',
         parameters: {
           type: 'object',
           properties: {
@@ -2055,15 +2222,15 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
       function: {
         name: 'web_search',
         description:
-          'Run a live web search through DuckDuckGo. Use this for keyword-based internet search. This tool respects config.web.search_enabled and will fail when network search is disabled.',
+          'Run a live web search by fetching Bing RSS results. Use this for keyword-based internet search. This tool respects config.web.search_enabled and will fail when network search is disabled.',
         parameters: {
           type: 'object',
           properties: {
             query: { type: 'string', description: 'Search query' },
             q: { type: 'string', description: 'Alias for query' },
             max_results: { type: 'number', description: 'Max results to return' },
-            locale: { type: 'string', description: 'DuckDuckGo locale such as en-us' },
-            region: { type: 'string', description: 'DuckDuckGo region such as wt-wt' }
+            locale: { type: 'string', description: 'Bing market and language such as en-US or zh-CN' },
+            region: { type: 'string', description: 'Bing country code such as US or CN' }
           },
           required: ['query']
         }
@@ -2140,7 +2307,7 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
       function: {
         name: 'dream_consolidate',
         description:
-          'Run a dream loop consolidation pass over inbox entries. Reads recent inbox items, deduplicates, evaluates lifecycle progression (observed → candidate → operational/longterm), promotes stable patterns into persistent memory, archives expired items, and writes an audit report. Use during off-hours or explicit maintenance.',
+          'Run a dream loop pass over inbox entries and existing memory buckets. Reads recent inbox items, deduplicates, evaluates lifecycle progression (observed → candidate → operational/longterm), promotes stable patterns into persistent memory, then uses LLM maintenance to merge/summarize/clean stale user/global/project memories when their bucket changed since the last maintenance marker. Writes an audit report. Use during off-hours or explicit maintenance.',
         parameters: {
           type: 'object',
           properties: {
@@ -2742,6 +2909,12 @@ export function getBuiltinTools({ workspaceRoot = process.cwd(), config, onSyste
       if (result.title) lines.push(`title: ${result.title}`);
       if (result.description) lines.push(`description: ${trimPreview(result.description, 200)}`);
       if (result.metadata?.status) lines.push(`status: ${result.metadata.status}`);
+      if (result.metadata?.fetch_mode) lines.push(`mode: ${result.metadata.fetch_mode}`);
+      if (Array.isArray(result.warnings)) {
+        for (const warning of result.warnings.slice(0, 3)) {
+          if (warning) lines.push(`warning: ${warning}`);
+        }
+      }
       if (Array.isArray(result.links) && result.links.length > 0) {
         lines.push(`links: ${result.links.slice(0, 5).map((item) => item.href).join(', ')}`);
       }