kiku-agent 1.2.0 → 1.3.1

package/server/agent/websearch/adapters/duckduckgo.js

@@ -0,0 +1,97 @@
+import { createSearchResult } from '../core/types.js';
+import { safeFetchWithPinning } from '../security/ssrf.js';
+import { loadCheerio, decodeEntities, normalizeText } from '../extract.js';
+
+function cleanHtml(raw) {
+  if (!raw || typeof raw !== 'string') return '';
+  return normalizeText(decodeEntities(raw.replace(/<[^>]+>/g, ' ')));
+}
+
+async function parseResultsWithCheerio(html, maxResults) {
+  const cheerioLoad = await loadCheerio();
+  if (!cheerioLoad) return [];
+  const $ = cheerioLoad(html);
+  const results = [];
+  $('div.result, div.web-result').each((_, el) => {
+    if (results.length >= maxResults) return;
+    const $el = $(el);
+    const anchor = $el.find('a.result__a').first();
+    let link = anchor.attr('href') || '';
+    if (link.includes('uddg=')) {
+      try {
+        const decoded = new URL('https://duckduckgo.com' + link).searchParams.get('uddg');
+        if (decoded) link = decoded;
+      } catch {}
+    }
+    const title = normalizeText(decodeEntities(anchor.text()));
+    const snippet = normalizeText(decodeEntities($el.find('.result__snippet').first().text()));
+    if (link && title) results.push({ link, title, snippet });
+  });
+  return results;
+}
+
+function parseResultsWithRegex(html, maxResults) {
+  const results = [];
+  const resultRegex = /<div[^>]*class="[^"]*result[^"]*"[^>]*>[\s\S]*?<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/gi;
+  let match;
+  while ((match = resultRegex.exec(html)) !== null && results.length < maxResults) {
+    results.push({ link: match[1], title: cleanHtml(match[2]), snippet: cleanHtml(match[3]) });
+  }
+  return results;
+}
+
+export async function searchDuckDuckGo(query, options = {}) {
+  const maxResults = Number(options.limit) || 10;
+  const timeoutMs = Number(options.timeoutMs) || 8000;
+  const results = [];
+  const seenLinks = new Set();
+
+  try {
+    const lang = options.language || 'en';
+    const kl = lang === 'th' ? 'th-th' : (lang === 'zh' ? 'zh-cn' : 'en-us');
+    const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=${kl}`;
+
+    const res = await safeFetchWithPinning(searchUrl, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'th,en-US;q=0.9,en;q=0.8,zh-CN;q=0.7'
+      },
+      timeoutMs,
+      signal: options.signal
+    });
+
+    if (!res.ok) {
+      return [];
+    }
+
+    const html = await res.text();
+    const cheerioParsed = await parseResultsWithCheerio(html, maxResults);
+    const parsed = cheerioParsed.length > 0 ? cheerioParsed : parseResultsWithRegex(html, maxResults);
+
+    for (const item of parsed) {
+      if (results.length >= maxResults) break;
+      let rawLink = item.link;
+      if (rawLink.includes('uddg=')) {
+        try {
+          const decoded = new URL('https://duckduckgo.com' + rawLink).searchParams.get('uddg');
+          if (decoded) rawLink = decoded;
+        } catch {}
+      }
+
+      if (rawLink && item.title && !seenLinks.has(rawLink)) {
+        seenLinks.add(rawLink);
+        results.push(createSearchResult({
+          link: rawLink,
+          title: item.title,
+          snippet: item.snippet,
+          engine: 'duckduckgo',
+          score: 1.0 - (results.length * 0.05)
+        }));
+      }
+    }
+  } catch {
+  }
+
+  return results;
+}

package/server/agent/websearch/adapters/jina.js

@@ -0,0 +1,77 @@
+import { createSearchResult } from '../core/types.js';
+import { safeFetchWithPinning } from '../security/ssrf.js';
+
+
+export async function searchJina(query, options = {}) {
+  const maxResults = Number(options.limit) || 10;
+  const timeoutMs = Number(options.timeoutMs) || 10000;
+  const results = [];
+  const seenLinks = new Set();
+
+  try {
+    const searchUrl = `https://s.jina.ai/${encodeURIComponent(query)}`;
+    const res = await safeFetchWithPinning(searchUrl, {
+      headers: {
+        'Accept': 'application/json, text/plain, */*',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'X-No-Cache': 'false'
+      },
+      timeoutMs,
+      signal: options.signal
+    });
+
+    if (!res.ok) {
+      return [];
+    }
+
+    const contentType = res.headers.get('content-type') || '';
+    if (contentType.includes('application/json')) {
+      const json = await res.json();
+      const items = Array.isArray(json?.data) ? json.data : (Array.isArray(json?.results) ? json.results : []);
+
+      for (const item of items.slice(0, maxResults)) {
+        const link = item?.url || item?.link;
+        if (link && !seenLinks.has(link)) {
+          seenLinks.add(link);
+          results.push(createSearchResult({
+            link,
+            title: item.title || query,
+            snippet: item.description || item.content || '',
+            engine: 'jina',
+            score: 1.0 - (results.length * 0.05),
+            metadata: {
+              publishedTime: item.publishedTime || null
+            }
+          }));
+        }
+      }
+    } else {
+     
+      const text = await res.text();
+     
+      const blockRegex = /\[([^\]]+)\]\((https?:\/\/[^\s)]+)\)([\s\S]*?)(?=\n\[|$)/g;
+      let match;
+      while ((match = blockRegex.exec(text)) !== null && results.length < maxResults) {
+        const title = match[1].trim();
+        const link = match[2].trim();
+        const snippet = match[3].replace(/\s+/g, ' ').trim().slice(0, 300);
+
+        if (link && !seenLinks.has(link)) {
+          seenLinks.add(link);
+          results.push(createSearchResult({
+            link,
+            title,
+            snippet,
+            engine: 'jina',
+            score: 1.0 - (results.length * 0.05)
+          }));
+        }
+      }
+    }
+  } catch {
+  
+    return [];
+  }
+
+  return results;
+}

package/server/agent/websearch/adapters/searxng.js

@@ -0,0 +1,57 @@
+import { createSearchResult } from '../core/types.js';
+import { safeFetchWithPinning } from '../security/ssrf.js';
+
+
+export async function searchSearXNG(query, options = {}) {
+  const baseUrl = options.baseUrl || process.env.SEARXNG_URL;
+  if (!baseUrl) {
+   
+    return [];
+  }
+
+  const maxResults = Number(options.limit) || 10;
+  const timeoutMs = Number(options.timeoutMs) || 8000;
+  const results = [];
+
+  try {
+    const cleanBase = baseUrl.replace(/\/+$/, '');
+    const searchUrl = `${cleanBase}/search?q=${encodeURIComponent(query)}&format=json&language=${options.language || 'auto'}`;
+
+    const res = await safeFetchWithPinning(searchUrl, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'Accept': 'application/json, text/plain, */*'
+      },
+      timeoutMs,
+      signal: options.signal
+    });
+
+    if (!res.ok) {
+      return [];
+    }
+
+    const json = await res.json();
+    const rawResults = Array.isArray(json?.results) ? json.results : [];
+
+    for (const item of rawResults.slice(0, maxResults)) {
+      if (item?.url && item?.title) {
+        results.push(createSearchResult({
+          link: item.url,
+          title: item.title,
+          snippet: item.content || item.snippet || '',
+          engine: 'searxng',
+          score: Number(item.score) || (1.0 - results.length * 0.05),
+          metadata: {
+            engines: item.engines || [],
+            category: item.category || 'general'
+          }
+        }));
+      }
+    }
+  } catch {
+  
+    return [];
+  }
+
+  return results;
+}

package/server/agent/websearch/core/canonical.js

@@ -0,0 +1,89 @@
+const TRACKING_PARAMS = new Set([
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'utm_id',
+  'fbclid',
+  'gclid',
+  'dclid',
+  'gclsrc',
+  'ref',
+  'ref_src',
+  'spm',
+  'spm_id_from',
+  'from',
+  'source',
+  '_ga',
+  '_gl',
+  'mc_eid',
+  'yclid',
+  'trk',
+  'wt_mc',
+  'ocid'
+]);
+
+
+export function canonicalizeUrl(rawUrl) {
+  if (!rawUrl || typeof rawUrl !== 'string') return '';
+  const trimmed = rawUrl.trim();
+
+  let parsed;
+  try {
+    parsed = new URL(trimmed);
+  } catch {
+    return trimmed;
+  }
+
+ 
+  if (parsed.protocol === 'http:' && !parsed.hostname.includes('localhost') && !parsed.hostname.startsWith('127.')) {
+    parsed.protocol = 'https:';
+  }
+
+  
+  let hostname = parsed.hostname.toLowerCase();
+  if (hostname.startsWith('www.')) {
+    hostname = hostname.slice(4);
+  }
+  parsed.hostname = hostname;
+
+
+  if ((parsed.protocol === 'https:' && parsed.port === '443') || (parsed.protocol === 'http:' && parsed.port === '80')) {
+    parsed.port = '';
+  }
+
+ 
+  if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+    parsed.pathname = parsed.pathname.slice(0, -1);
+  }
+
+  
+  const searchParams = new URLSearchParams(parsed.search);
+  const remainingKeys = [];
+  for (const [key] of searchParams.entries()) {
+    const lowerKey = key.toLowerCase();
+    if (TRACKING_PARAMS.has(lowerKey) || lowerKey.startsWith('utm_')) {
+      searchParams.delete(key);
+    } else {
+      if (!remainingKeys.includes(key)) remainingKeys.push(key);
+    }
+  }
+
+  remainingKeys.sort();
+  const sortedParams = new URLSearchParams();
+  for (const key of remainingKeys) {
+    const vals = searchParams.getAll(key);
+    for (const val of vals) {
+      sortedParams.append(key, val);
+    }
+  }
+
+  const sortedQueryString = sortedParams.toString();
+  parsed.search = sortedQueryString ? `?${sortedQueryString}` : '';
+
+ 
+  parsed.hash = '';
+
+  return parsed.href;
+}

package/server/agent/websearch/core/fusion.js

@@ -0,0 +1,81 @@
+import { canonicalizeUrl } from './canonical.js';
+
+export const DEFAULT_RRF_K = 60;
+
+/**
+ * Merges and ranks search results across multiple engines using Reciprocal Rank Fusion (RRF).
+ * Formula: Score(d) = Sum_{e in E} ( 1 / (k + rank_e(d)) )
+ *
+ * @param {Array<Array<import('./types.js').SearchResult>>} engineResultLists - Array of result arrays from engines
+ * @param {Object} [options]
+ * @param {number} [options.k=60] - Smoothing constant
+ * @param {number} [options.limit=10] - Max fused results to return
+ * @returns {Array<import('./types.js').SearchResult>} Fused, deduplicated and ranked results
+ */
+export function fuseSearchResults(engineResultLists = [], options = {}) {
+  const k = Number.isFinite(options.k) ? Math.max(1, options.k) : DEFAULT_RRF_K;
+  const limit = Number.isFinite(options.limit) ? Math.max(1, options.limit) : 10;
+
+ 
+  const aggregated = new Map();
+
+  for (const list of engineResultLists) {
+    if (!Array.isArray(list)) continue;
+
+    for (let i = 0; i < list.length; i++) {
+      const item = list[i];
+      if (!item || !item.link) continue;
+
+      const canonicalLink = canonicalizeUrl(item.link);
+      if (!canonicalLink) continue;
+
+      const rank = i + 1; 
+      const rrfContribution = 1 / (k + rank);
+
+      let existing = aggregated.get(canonicalLink);
+      if (!existing) {
+        existing = {
+          link: canonicalLink,
+          title: item.title || '',
+          snippet: item.snippet || '',
+          engine: item.engine || 'meta',
+          score: 0,
+          metadata: {
+            engines: [item.engine].filter(Boolean),
+            ranks: { [item.engine]: rank },
+            originalLinks: [item.link],
+            snippets: item.snippet ? [item.snippet] : []
+          }
+        };
+        aggregated.set(canonicalLink, existing);
+      } else {
+        if (item.engine && !existing.metadata.engines.includes(item.engine)) {
+          existing.metadata.engines.push(item.engine);
+        }
+        if (item.engine) {
+          existing.metadata.ranks[item.engine] = rank;
+        }
+        if (item.link && !existing.metadata.originalLinks.includes(item.link)) {
+          existing.metadata.originalLinks.push(item.link);
+        }
+        if (item.title && (!existing.title || item.title.length > existing.title.length)) {
+          existing.title = item.title;
+        }
+        if (item.snippet && !existing.metadata.snippets.includes(item.snippet)) {
+          existing.metadata.snippets.push(item.snippet);
+        
+          if (item.snippet.length > existing.snippet.length) {
+            existing.snippet = item.snippet;
+          }
+        }
+      }
+
+      existing.score += rrfContribution;
+    }
+  }
+
+  const results = Array.from(aggregated.values());
+  results.sort((a, b) => b.score - a.score);
+
+  return results.slice(0, limit);
+}

package/server/agent/websearch/core/types.js

@@ -0,0 +1,29 @@
+/**
+ * Canonical Search Result Data Contract
+ *
+ * @typedef {Object} SearchResult
+ * @property {string} link - Canonical destination URL
+ * @property {string} title - Sanitized result title
+ * @property {string} snippet - Extracted text snippet or abstract
+ * @property {'duckduckgo' | 'searxng' | 'jina' | 'openserp' | 'yacy' | string} engine - Origin search engine
+ * @property {number} [score] - Optional relevance / RRF score
+ * @property {Record<string, unknown>} [metadata] - Optional engine-specific metadata
+ */
+
+export function createSearchResult({
+  link,
+  title = '',
+  snippet = '',
+  engine = 'duckduckgo',
+  score = 0,
+  metadata = {}
+}) {
+  return {
+    link: String(link || '').trim(),
+    title: String(title || '').trim(),
+    snippet: String(snippet || '').trim(),
+    engine: String(engine || 'duckduckgo').toLowerCase(),
+    score: Number.isFinite(Number(score)) ? Number(score) : 0,
+    metadata: metadata && typeof metadata === 'object' ? metadata : {}
+  };
+}