kiku-agent 1.0.0

package/server/agent/webSearch.js

@@ -0,0 +1,719 @@
+import { existsSync } from 'node:fs';
+import { execFile } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+let _cheerioLoad = null;
+async function loadCheerio() {
+  if (_cheerioLoad) return _cheerioLoad;
+  try {
+    const mod = await import('cheerio');
+    _cheerioLoad = mod.load;
+    return _cheerioLoad;
+  } catch {
+    return null;
+  }
+}
+
+function cleanHtml(html) {
+  if (!html) return '';
+  return html
+    .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
+    .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
+    .replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, ' ')
+    .replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, ' ')
+    .replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, ' ')
+    .replace(/<aside\b[^<]*(?:(?!<\/aside>)<[^<]*)*<\/aside>/gi, ' ')
+    .replace(/<iframe\b[^<]*(?:(?!<\/iframe>)<[^<]*)*<\/iframe>/gi, ' ')
+    .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&mdash;/g, '—')
+    .replace(/&ndash;/g, '–')
+    .replace(/&hellip;/g, '...')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function detectLanguage(query) {
+  if (/[\u0E00-\u0E7F]/.test(query)) return 'th';
+  if (/[\u4E00-\u9FA5]/.test(query)) return 'zh';
+  return 'en';
+}
+
+function expandQuery(query, lang) {
+  const expansions = [];
+  
+  if (lang === 'zh') {
+    if (query.includes('鞠婧祎')) {
+      expansions.push(query.replace('鞠婧祎', '鞠婧祎 最新'));
+      expansions.push(query.replace('鞠婧祎', '鞠婧祎 2026'));
+      expansions.push(query.replace('鞠婧祎', '鞠婧祎 官方'));
+    }
+  } else if (lang === 'th') {
+    if (query.includes('จวีจิ้งอี')) {
+      expansions.push(query.replace('จวีจิ้งอี', 'จวีจิ้งอี ล่าสุด'));
+      expansions.push(query.replace('จวีจิ้งอี', 'จวีจิ้งอี 2026'));
+      expansions.push(query.replace('จวีจิ้งอี', 'จวีจิ้งอี อย่างเป็นทางการ'));
+    }
+  } else {
+    if (query.includes('Ju Jingyi')) {
+      expansions.push(query.replace('Ju Jingyi', 'Ju Jingyi latest'));
+      expansions.push(query.replace('Ju Jingyi', 'Ju Jingyi 2026'));
+      expansions.push(query.replace('Ju Jingyi', 'Ju Jingyi official'));
+    }
+  }
+  
+  return expansions;
+}
+
+async function searchDuckDuckGo(query, maxResults) {
+  const results = [];
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 8000);
+
+    const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=${detectLanguage(query) === 'th' ? 'th-th' : detectLanguage(query) === 'zh' ? 'zh-cn' : 'en-us'}`;
+    const res = await fetch(searchUrl, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'th,en-US;q=0.9,en;q=0.8,zh-CN;q=0.7,zh;q=0.6'
+      },
+      signal: controller.signal
+    });
+
+    clearTimeout(timeoutId);
+
+    if (res.ok) {
+      const html = await res.text();
+      
+      const resultRegex = /<div[^>]*class="[^"]*result[^"]*"[^>]*>[\s\S]*?<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/gi;
+      let match;
+      while ((match = resultRegex.exec(html)) !== null && results.length < maxResults) {
+        let rawLink = match[1];
+        if (rawLink.includes('uddg=')) {
+          try {
+            const decoded = new URL('https://duckduckgo.com' + rawLink).searchParams.get('uddg');
+            if (decoded) rawLink = decoded;
+          } catch {}
+        }
+
+        const title = cleanHtml(match[2]);
+        const snippet = cleanHtml(match[3]);
+
+        if (title && snippet && !results.some(r => r.url === rawLink)) {
+          results.push({
+            title,
+            url: rawLink,
+            snippet,
+            source: 'DuckDuckGo'
+          });
+        }
+      }
+    }
+  } catch (err) {
+  }
+  
+  return results;
+}
+
+async function searchWikipedia(query, maxResults, lang) {
+  const results = [];
+  try {
+    const wikiUrl = `https://${lang}.wikipedia.org/w/api.php?action=query&list=search&srsearch=${encodeURIComponent(query)}&format=json&utf8=1&srlimit=${maxResults}&srenablerewrites=1`;
+    
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 6000);
+
+    const wikiRes = await fetch(wikiUrl, {
+      headers: { 'User-Agent': 'KikuAgent/1.0 (JuJingyi Fan Knowledge Bot)' },
+      signal: controller.signal
+    });
+
+    clearTimeout(timeoutId);
+
+    if (wikiRes.ok) {
+      const data = await wikiRes.json();
+      const wikiItems = data.query?.search || [];
+      for (const item of wikiItems) {
+        if (results.length >= maxResults) break;
+        results.push({
+          title: `[Wikipedia] ${item.title}`,
+          url: `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(item.title.replace(/ /g, '_'))}`,
+          snippet: cleanHtml(item.snippet.replace(/<span class="searchmatch">/g, '').replace(/<\/span>/g, '')),
+          source: 'Wikipedia'
+        });
+      }
+    }
+  } catch {
+  }
+  
+  return results;
+}
+
+async function searchBing(query, maxResults) {
+  const results = [];
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 8000);
+
+    const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&setlang=${detectLanguage(query) === 'th' ? 'th' : detectLanguage(query) === 'zh' ? 'zh-cn' : 'en-us'}`;
+    const res = await fetch(searchUrl, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'th,en-US;q=0.9,en;q=0.8,zh-CN;q=0.7,zh;q=0.6'
+      },
+      signal: controller.signal
+    });
+
+    clearTimeout(timeoutId);
+
+    if (res.ok) {
+      const html = await res.text();
+      
+      const linkRegex = /<h2><a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<p[^>]*class="b_lineclamp2 b_algoSlug"[^>]*>([\s\S]*?)<\/p>/gi;
+      let match;
+      while ((match = linkRegex.exec(html)) !== null && results.length < maxResults) {
+        const rawLink = match[1];
+        const title = cleanHtml(match[2]);
+        const snippet = cleanHtml(match[3]);
+
+        if (title && snippet && !results.some(r => r.url === rawLink)) {
+          results.push({
+            title,
+            url: rawLink,
+            snippet,
+            source: 'Bing'
+          });
+        }
+      }
+    }
+  } catch (err) {
+  }
+  
+  return results;
+}
+
+export async function searchWeb(query, maxResults = 8) {
+  if (!query || typeof query !== 'string') {
+    return { status: 'error', message: 'Query string is required for web search.' };
+  }
+
+  const lang = detectLanguage(query);
+  const results = [];
+  const sources = [];
+
+  try {
+    const ddgResults = await searchDuckDuckGo(query, maxResults);
+    ddgResults.forEach(r => {
+      if (!results.some(existing => existing.url === r.url)) {
+        results.push(r);
+        sources.push('DuckDuckGo');
+      }
+    });
+  } catch {}
+
+  if (results.length < maxResults) {
+    try {
+      const wikiResults = await searchWikipedia(query, maxResults - results.length, lang);
+      wikiResults.forEach(r => {
+        if (!results.some(existing => existing.url === r.url)) {
+          results.push(r);
+          sources.push('Wikipedia');
+        }
+      });
+    } catch {}
+  }
+
+  if (results.length < maxResults / 2) {
+    try {
+      const expandedQueries = expandQuery(query, lang);
+      for (const expandedQuery of expandedQueries) {
+        if (results.length >= maxResults) break;
+        const ddgResults = await searchDuckDuckGo(expandedQuery, Math.ceil(maxResults / 2));
+        ddgResults.forEach(r => {
+          if (!results.some(existing => existing.url === r.url)) {
+            results.push(r);
+            sources.push('DuckDuckGo (expanded)');
+          }
+        });
+      }
+    } catch {}
+  }
+
+  if (results.length < maxResults) {
+    try {
+      const bingResults = await searchBing(query, maxResults - results.length);
+      bingResults.forEach(r => {
+        if (!results.some(existing => existing.url === r.url)) {
+          results.push(r);
+          sources.push('Bing');
+        }
+      });
+    } catch {}
+  }
+
+  return {
+    status: results.length > 0 ? 'success' : 'no_results',
+    query,
+    language: lang,
+    count: results.length,
+    sources: [...new Set(sources)],
+    results: results.slice(0, maxResults)
+  };
+}
+
+export async function fetchWebpage(url, maxLength = 8000) {
+  if (!url || typeof url !== 'string' || !url.startsWith('http')) {
+    return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
+  }
+
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 15000);
+
+    const res = await fetch(url, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9,th;q=0.8,zh-CN;q=0.7'
+      },
+      signal: controller.signal,
+      redirect: 'follow'
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!res.ok) {
+      return { status: 'error', message: `HTTP ${res.status}: Failed to load webpage` };
+    }
+
+    const contentType = res.headers.get('content-type') || '';
+    const finalUrl = res.url || url;
+
+    if (contentType.includes('application/json')) {
+      const json = await res.json();
+      const text = JSON.stringify(json, null, 2);
+      return {
+        status: 'success',
+        url: finalUrl,
+        contentType: 'json',
+        contentLength: text.length,
+        content: text.length > maxLength ? text.slice(0, maxLength) + '... [truncated]' : text
+      };
+    }
+
+    const html = await res.text();
+    const cheerioLoad = await loadCheerio();
+
+    if (cheerioLoad) {
+      const $ = cheerioLoad(html);
+
+      $('script, style, noscript, iframe, svg, nav, footer, aside, form, button').remove();
+
+      const meta = {};
+      meta.title = $('title').first().text().trim() || '';
+      meta.description = $('meta[name="description"]').attr('content') || $('meta[property="og:description"]').attr('content') || '';
+      meta.author = $('meta[name="author"]').attr('content') || '';
+      meta.keywords = $('meta[name="keywords"]').attr('content') || '';
+      meta.ogTitle = $('meta[property="og:title"]').attr('content') || '';
+      meta.ogImage = $('meta[property="og:image"]').attr('content') || '';
+      meta.ogType = $('meta[property="og:type"]').attr('content') || '';
+      meta.canonical = $('link[rel="canonical"]').attr('href') || '';
+      meta.favicon = $('link[rel="icon"], link[rel="shortcut icon"]').attr('href') || '';
+      meta.lang = $('html').attr('lang') || '';
+
+      const headings = [];
+      $('h1, h2, h3').each((i, el) => {
+        if (i < 10) {
+          const txt = $(el).text().trim();
+          if (txt) headings.push({ level: el.tagName, text: txt });
+        }
+      });
+
+      const paragraphs = [];
+      $('article p, main p, .content p, .post-content p, .entry-content p, .article-body p, .wiki-content p').each((i, el) => {
+        if (i < 20) {
+          const txt = $(el).text().trim();
+          if (txt && txt.length > 20) paragraphs.push(txt);
+        }
+      });
+      if (paragraphs.length === 0) {
+        $('p').each((i, el) => {
+          if (i < 15) {
+            const txt = $(el).text().trim();
+            if (txt && txt.length > 30) paragraphs.push(txt);
+          }
+        });
+      }
+
+      const links = [];
+      $('article a, main a, .content a').each((i, el) => {
+        if (i < 15) {
+          const href = $(el).attr('href');
+          const text = $(el).text().trim();
+          if (href && text && text.length > 2) {
+            let absoluteUrl = href;
+            try { absoluteUrl = new URL(href, finalUrl).href; } catch {}
+            links.push({ text: text.slice(0, 80), url: absoluteUrl });
+          }
+        }
+      });
+
+      const images = [];
+      $('article img, main img, .content img').each((i, el) => {
+        if (i < 8) {
+          const src = $(el).attr('src');
+          const alt = $(el).attr('alt') || '';
+          if (src) {
+            let absoluteUrl = src;
+            try { absoluteUrl = new URL(src, finalUrl).href; } catch {}
+            images.push({ src: absoluteUrl, alt: alt.slice(0, 100) });
+          }
+        }
+      });
+
+      let bodyText = $('body').text().replace(/\s+/g, ' ').trim();
+      if (bodyText.length > maxLength) {
+        bodyText = bodyText.slice(0, maxLength) + '... [Content truncated for brevity]';
+      }
+
+      const result = {
+        status: 'success',
+        url: finalUrl,
+        contentType: 'html',
+        contentLength: bodyText.length,
+        content: bodyText || 'Unable to retrieve text content.',
+        meta: Object.fromEntries(Object.entries(meta).filter(([, v]) => v))
+      };
+
+      if (headings.length > 0) result.headings = headings;
+      if (paragraphs.length > 0) result.paragraphs = paragraphs;
+      if (links.length > 0) result.links = links;
+      if (images.length > 0) result.images = images;
+
+      return result;
+    }
+
+    let text = cleanHtml(html);
+    if (text.length > maxLength) {
+      text = text.slice(0, maxLength) + '... [Content truncated for brevity]';
+    }
+
+    return {
+      status: 'success',
+      url: finalUrl,
+      contentType: 'html',
+      contentLength: text.length,
+      content: text || 'Unable to retrieve text content from this webpage.'
+    };
+  } catch (err) {
+    return {
+      status: 'error',
+      url,
+      message: err.name === 'AbortError' ? 'Webpage fetch timed out (15s)' : err.message
+    };
+  }
+}
+
+export async function fetchJson(url, options = {}) {
+  if (!url || typeof url !== 'string' || !url.startsWith('http')) {
+    return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
+  }
+
+  const {
+    method = 'GET',
+    headers = {},
+    body = null,
+    timeoutMs = 12000,
+    maxResponseLength = 32000
+  } = options;
+
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+    const fetchOptions = {
+      method: method.toUpperCase(),
+      headers: {
+        'Accept': 'application/json, text/plain, */*',
+        'User-Agent': 'KikuAgent/2.0',
+        ...headers
+      },
+      signal: controller.signal
+    };
+
+    if (body && ['POST', 'PUT', 'PATCH'].includes(fetchOptions.method)) {
+      fetchOptions.body = typeof body === 'string' ? body : JSON.stringify(body);
+      fetchOptions.headers['Content-Type'] = 'application/json';
+    }
+
+    const res = await fetch(url, fetchOptions);
+    clearTimeout(timeoutId);
+
+    const responseInfo = {
+      status: res.status,
+      statusText: res.statusText,
+      contentType: res.headers.get('content-type') || '',
+      finalUrl: res.url || url
+    };
+
+    if (!res.ok) {
+      let errorBody = '';
+      try { errorBody = (await res.text()).slice(0, 1000); } catch {}
+      return {
+        status: 'error',
+        ...responseInfo,
+        message: `HTTP ${res.status} ${res.statusText}`,
+        errorBody
+      };
+    }
+
+    const rawText = await res.text();
+    let data;
+    try {
+      data = JSON.parse(rawText);
+    } catch {
+      return {
+        status: 'success',
+        ...responseInfo,
+        dataType: 'text',
+        contentLength: rawText.length,
+        data: rawText.length > maxResponseLength
+          ? rawText.slice(0, maxResponseLength) + '... [truncated]'
+          : rawText
+      };
+    }
+
+    const serialized = JSON.stringify(data, null, 2);
+    const truncated = serialized.length > maxResponseLength;
+
+    return {
+      status: 'success',
+      ...responseInfo,
+      dataType: 'json',
+      contentLength: serialized.length,
+      truncated,
+      data: truncated ? serialized.slice(0, maxResponseLength) + '... [truncated]' : data
+    };
+  } catch (err) {
+    return {
+      status: 'error',
+      url,
+      message: err.name === 'AbortError' ? `Request timed out (${timeoutMs}ms)` : err.message
+    };
+  }
+}
+
+function findChromePath() {
+  const p = process.platform;
+  if (p === 'win32') {
+    const candidates = [
+      (process.env.LOCALAPPDATA || '') + '/Google/Chrome/Application/chrome.exe',
+      (process.env['PROGRAMFILES'] || '') + '/Google/Chrome/Application/chrome.exe',
+      (process.env['PROGRAMFILES(X86)'] || '') + '/Google/Chrome/Application/chrome.exe',
+      'C:/Program Files/Google/Chrome/Application/chrome.exe',
+      'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
+    ];
+    for (const c of candidates) {
+      if (c && existsSync(c)) return c;
+    }
+  } else if (p === 'darwin') {
+    return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+  } else {
+    const candidates = ['/usr/bin/google-chrome', '/usr/bin/google-chrome-stable', '/usr/bin/chromium-browser', '/usr/bin/chromium'];
+    for (const c of candidates) {
+      if (existsSync(c)) return c;
+    }
+  }
+  return null;
+}
+
+export async function screenshotWebpage(url, options = {}) {
+  if (!url || typeof url !== 'string' || !url.startsWith('http')) {
+    return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
+  }
+
+  const { width = 1280, height = 800, fullPage = false, timeoutMs = 20000 } = options;
+  const chromePath = findChromePath();
+
+  if (!chromePath) {
+    return { status: 'error', message: 'Google Chrome not found. Install Chrome to enable screenshot feature.' };
+  }
+
+  const tmpDir = await mkdtemp(join(tmpdir(), 'kiku-screenshot-'));
+  const profileDir = join(tmpDir, 'chrome-profile');
+
+  try {
+    const screenshotFile = 'kiku_screenshot.png';
+    const screenshotPath = join(tmpDir, screenshotFile);
+    const args = [
+      '--headless=new',
+      '--no-sandbox',
+      '--disable-gpu',
+      '--disable-dev-shm-usage',
+      '--disable-software-rasterizer',
+      `--user-data-dir=${profileDir}`,
+      `--window-size=${width},${height}`,
+      `--screenshot=${screenshotPath}`,
+      '--hide-scrollbars',
+      '--virtual-time-budget=8000'
+    ];
+
+    args.push(url);
+
+    await new Promise((resolve, reject) => {
+      const child = execFile(chromePath, args, {
+        timeout: timeoutMs
+      }, (err, stdout, stderr) => {
+        if (err) reject(err);
+        else resolve({ stdout, stderr });
+      });
+      child.on('error', reject);
+    });
+
+    if (!existsSync(screenshotPath)) {
+      return { status: 'error', message: 'Screenshot was not generated. The page may have failed to load.' };
+    }
+
+    const imgBuffer = await readFile(screenshotPath);
+    const base64 = imgBuffer.toString('base64');
+    const sizeKB = Math.round(imgBuffer.length / 1024);
+
+    return {
+      status: 'success',
+      url,
+      width,
+      height,
+      format: 'png',
+      sizeKB,
+      base64Length: base64.length,
+      data: `data:image/png;base64,${base64}`
+    };
+  } catch (err) {
+    return {
+      status: 'error',
+      url,
+      message: `Screenshot failed: ${err.message}`
+    };
+  } finally {
+    try { await rm(tmpDir, { recursive: true, force: true }); } catch {}
+  }
+}
+
+export async function webExtract(query, opts = {}) {
+  if (!query || typeof query !== 'string') {
+    return { status: 'error', message: 'Query string is required.' };
+  }
+
+  const {
+    maxResults = 3,
+    fetchTimeoutMs = 12000,
+    maxContentLength = 4000,
+    allowedDomains = []
+  } = opts;
+
+  const cappedResults = Math.min(Math.max(maxResults, 1), 5);
+
+
+  const searchResult = await searchWeb(query, cappedResults + 2);
+  if (searchResult.status !== 'success' || !searchResult.results?.length) {
+    return {
+      status: 'no_results',
+      query,
+      message: 'No search results found for this query.',
+      searchResults: searchResult.results || []
+    };
+  }
+
+ 
+  let candidates = searchResult.results;
+  if (allowedDomains.length > 0) {
+    candidates = candidates.filter(r => {
+      try {
+        const host = new URL(r.url).hostname;
+        return allowedDomains.some(d => host.includes(d));
+      } catch { return false; }
+    });
+  }
+
+ 
+  const toFetch = candidates.slice(0, cappedResults);
+  const fetchPromises = toFetch.map(async (result) => {
+    try {
+      const page = await fetchWebpage(result.url, maxContentLength);
+      return {
+        title: result.title || page.meta?.title || '',
+        url: result.url,
+        snippet: result.snippet || '',
+        source: result.source || 'web',
+        ...(page.status === 'success' ? {
+          content: page.content || '',
+          meta: page.meta || {},
+          headings: page.headings || [],
+          paragraphs: (page.paragraphs || []).slice(0, 8),
+          links: (page.links || []).slice(0, 5),
+          images: (page.images || []).slice(0, 3),
+          fetchStatus: 'success'
+        } : {
+          content: result.snippet || '',
+          fetchStatus: 'failed',
+          fetchError: page.message || 'Failed to fetch page'
+        })
+      };
+    } catch (err) {
+      return {
+        title: result.title || '',
+        url: result.url,
+        snippet: result.snippet || '',
+        source: result.source || 'web',
+        content: result.snippet || '',
+        fetchStatus: 'error',
+        fetchError: err.message
+      };
+    }
+  });
+
+  const fetched = await Promise.all(fetchPromises);
+
+
+  const combinedContent = fetched
+    .filter(r => r.fetchStatus === 'success' && r.content)
+    .map(r => {
+      const metaLine = r.meta?.description ? `\nDescription: ${r.meta.description}` : '';
+      const headingsLine = r.headings?.length ? `\nHeadings: ${r.headings.map(h => h.text).join(' | ')}` : '';
+      return `### ${r.title}\nSource: ${r.url}${metaLine}${headingsLine}\n\n${r.content.slice(0, maxContentLength)}`;
+    })
+    .join('\n\n---\n\n');
+
+  return {
+    status: 'success',
+    query,
+    language: searchResult.language,
+    pagesFetched: fetched.length,
+    pagesSucceeded: fetched.filter(r => r.fetchStatus === 'success').length,
+    combinedContentLength: combinedContent.length,
+    pages: fetched.map(r => ({
+      title: r.title,
+      url: r.url,
+      snippet: r.snippet,
+      source: r.source,
+      fetchStatus: r.fetchStatus,
+      contentLength: r.content?.length || 0,
+      meta: r.meta || {},
+      headings: r.headings || [],
+      topParagraphs: (r.paragraphs || []).slice(0, 5),
+      topLinks: (r.links || []).slice(0, 3),
+      topImages: (r.images || []).slice(0, 2)
+    })),
+    combinedContent: combinedContent || 'No content could be extracted from search results.'
+  };
+}