kiku-agent 1.3.0 → 1.3.1

package/server/agent/websearch/orchestrator.js

@@ -0,0 +1,266 @@
+import { searchDuckDuckGo } from './adapters/duckduckgo.js';
+import { searchSearXNG } from './adapters/searxng.js';
+import { searchJina } from './adapters/jina.js';
+import { fuseSearchResults } from './core/fusion.js';
+import { defaultLimiters } from './resilience/limiter.js';
+import { defaultBreakers } from './resilience/breaker.js';
+import { defaultSearchCache, computeSearchCacheKey, TTL_GENERAL_MS, TTL_NEWS_MS } from './resilience/cache.js';
+import { defaultDomainFilter } from './security/filter.js';
+import { safeFetchWithPinning } from './security/ssrf.js';
+import { defaultTelemetry } from './telemetry.js';
+
+function cleanHtml(raw) {
+  if (!raw || typeof raw !== 'string') return '';
+  return raw
+    .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
+    .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export class MetaSearchOrchestrator {
+  constructor(options = {}) {
+    this.limiters = options.limiters || defaultLimiters;
+    this.breakers = options.breakers || defaultBreakers;
+    this.cache = options.cache || defaultSearchCache;
+    this.filter = options.filter || defaultDomainFilter;
+    this.telemetry = options.telemetry || defaultTelemetry;
+    this.fallbackTimeoutMs = options.fallbackTimeoutMs || 2500;
+  }
+
+  
+  expandQuery(query) {
+    const raw = String(query || '').trim();
+    const expansions = [raw];
+
+    if (/[\u0E00-\u0E7F]/.test(raw)) {
+      
+      if (raw.includes('จวีจิ้งอี')) {
+        expansions.push(raw.replace('จวีจิ้งอี', 'Ju Jingyi'));
+        expansions.push(raw.replace('จวีจิ้งอี', '鞠婧祎'));
+      }
+    } else if (/[\u4E00-\u9FA5]/.test(raw)) {
+     
+      if (raw.includes('鞠婧祎')) {
+        expansions.push(raw.replace('鞠婧祎', 'Ju Jingyi'));
+      }
+    } else {
+    
+      if (/ju\s*jingyi/i.test(raw)) {
+        expansions.push(raw.replace(/ju\s*jingyi/gi, '鞠婧祎'));
+        expansions.push(raw.replace(/ju\s*jingyi/gi, 'จวีจิ้งอี'));
+      }
+    }
+
+    return [...new Set(expansions)].slice(0, 4);
+  }
+
+  
+  async _searchSingleEngine(engineName, query, options = {}) {
+    const breaker = this.breakers[engineName];
+    if (breaker && !breaker.canExecute()) {
+      return [];
+    }
+
+    const limiter = this.limiters[engineName];
+    if (limiter) {
+      try {
+        await limiter.acquire(1, 2000);
+      } catch (limiterErr) {
+        if (breaker) breaker.recordFailure(limiterErr);
+        return [];
+      }
+    }
+
+    const timeoutMs = options.timeoutMs || this.fallbackTimeoutMs;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+    try {
+      let results = [];
+      if (engineName === 'searxng') {
+        results = await searchSearXNG(query, { ...options, timeoutMs, signal: controller.signal });
+      } else if (engineName === 'duckduckgo') {
+        results = await searchDuckDuckGo(query, { ...options, timeoutMs, signal: controller.signal });
+      } else if (engineName === 'jina') {
+        results = await searchJina(query, { ...options, timeoutMs, signal: controller.signal });
+      }
+
+      clearTimeout(timer);
+      if (breaker) breaker.recordSuccess();
+      return results;
+    } catch (err) {
+      clearTimeout(timer);
+      if (breaker) {
+        breaker.recordFailure(err);
+        if (breaker.state === 'OPEN') {
+          this.telemetry.recordCircuitTrip(engineName);
+        }
+      }
+      return [];
+    }
+  }
+
+  
+  async searchWithFallback(query, options = {}) {
+    const startTime = Date.now();
+    const limit = options.limit || 10;
+    const isNews = options.isNews === true;
+
+   
+    const cacheKey = computeSearchCacheKey('meta', query, limit);
+    const cached = this.cache.get(cacheKey);
+    if (cached && !options.noCache) {
+      return { ...cached, fromCache: true };
+    }
+
+   
+    const engineChain = [];
+    if (process.env.SEARXNG_URL || options.searxngUrl) {
+      engineChain.push('searxng');
+    }
+    engineChain.push('duckduckgo');
+    engineChain.push('jina');
+
+    const collectedLists = [];
+    let fallbackTriggered = false;
+    let successfulEngine = null;
+
+    const engineSettled = await Promise.allSettled(
+      engineChain.map(engine => this._searchSingleEngine(engine, query, options))
+    );
+    const engineLists = engineChain.map((engine, index) => {
+      const settled = engineSettled[index];
+      return settled.status === 'fulfilled' && Array.isArray(settled.value) ? settled.value : [];
+    });
+    for (const list of engineLists) {
+      if (list.length > 0) {
+        collectedLists.push(list);
+      }
+    }
+
+    const firstEngineResults = engineLists[0] || [];
+    fallbackTriggered = engineChain.length > 1 && firstEngineResults.length < Math.ceil(limit / 2);
+    successfulEngine = engineChain.find((engine, index) => engineLists[index].length > 0) || null;
+
+    let fusedInputCount = collectedLists.reduce((sum, list) => sum + list.length, 0);
+    if (fusedInputCount < 3 && options.fanOut !== false) {
+      const expansions = this.expandQuery(query).slice(1);
+      const fanSettled = await Promise.allSettled(
+        expansions.map(expQuery => this._searchSingleEngine('duckduckgo', expQuery, options))
+      );
+      for (const settled of fanSettled) {
+        const list = settled.status === 'fulfilled' && Array.isArray(settled.value) ? settled.value : [];
+        if (list.length > 0) {
+          collectedLists.push(list);
+          fusedInputCount += list.length;
+        }
+      }
+    }
+
+   
+    const fused = fuseSearchResults(collectedLists, { limit, k: options.k || 60 });
+
+    
+    const filtered = fused.filter(item => {
+      try {
+        const host = new URL(item.link).hostname;
+        return this.filter.isDomainAllowed(host);
+      } catch {
+        return false;
+      }
+    });
+
+    const latencyMs = Date.now() - startTime;
+    const zeroResult = filtered.length === 0;
+
+    this.telemetry.recordSearch({
+      latencyMs,
+      fallbackOccurred: fallbackTriggered,
+      zeroResult,
+      engine: successfulEngine
+    });
+
+    const response = {
+      status: zeroResult ? 'no_results' : 'success',
+      query,
+      count: filtered.length,
+      engine: successfulEngine || 'hybrid',
+      latencyMs,
+      results: filtered
+    };
+
+    this.cache.set(cacheKey, response, zeroResult ? 30_000 : (isNews ? TTL_NEWS_MS : TTL_GENERAL_MS));
+
+    return response;
+  }
+
+
+  async executeSearch(query, options = {}) {
+    const searchResponse = await this.searchWithFallback(query, options);
+    if (searchResponse.status === 'no_results' || options.mode === 'bypass' || options.deepScrape !== true) {
+      return searchResponse;
+    }
+
+   
+    const topUrls = searchResponse.results.slice(0, 3);
+    const scrapePromises = topUrls.map(async (resItem) => {
+      try {
+        const pageRes = await safeFetchWithPinning(resItem.link, {
+          timeoutMs: 8000,
+          maxBytes: 256_000,
+          headers: {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+          }
+        });
+
+        if (!pageRes.ok) {
+          return {
+            title: resItem.title,
+            link: resItem.link,
+            snippet: resItem.snippet,
+            content: resItem.snippet,
+            scrapeStatus: 'failed'
+          };
+        }
+
+        const html = await pageRes.text();
+        const text = cleanHtml(html);
+
+        return {
+          title: resItem.title,
+          link: resItem.link,
+          snippet: resItem.snippet,
+          content: text.slice(0, 4000),
+          scrapeStatus: 'success'
+        };
+      } catch (err) {
+        return {
+          title: resItem.title,
+          link: resItem.link,
+          snippet: resItem.snippet,
+          content: resItem.snippet,
+          scrapeStatus: 'error',
+          error: err.message
+        };
+      }
+    });
+
+    const scrapedPages = await Promise.all(scrapePromises);
+    const combinedMarkdown = scrapedPages
+      .filter(p => p.content)
+      .map(p => `### ${p.title}\nSource: ${p.link}\n\n${p.content}`)
+      .join('\n\n---\n\n');
+
+    return {
+      ...searchResponse,
+      mode: 'deep_scrape',
+      scrapedPages,
+      combinedContent: combinedMarkdown || 'No deep content extracted.'
+    };
+  }
+}
+
+export const defaultMetaSearch = new MetaSearchOrchestrator();

package/server/agent/websearch/resilience/breaker.js

@@ -0,0 +1,97 @@
+export const CircuitState = Object.freeze({
+  CLOSED: 'CLOSED',
+  OPEN: 'OPEN',
+  HALF_OPEN: 'HALF_OPEN'
+});
+
+/**
+ * Circuit Breaker per Engine
+ * Tripping threshold: 3 consecutive failures
+ * Cool-down: 60s
+ */
+export class CircuitBreaker {
+  /**
+   * @param {Object} options
+   * @param {string} [options.name] - Breaker identifier
+   * @param {number} [options.failureThreshold=3] - Consecutive failures before opening
+   * @param {number} [options.cooldownMs=60000] - Duration to stay OPEN before HALF_OPEN
+   */
+  constructor(options = {}) {
+    this.name = options.name || 'default';
+    this.failureThreshold = options.failureThreshold || 3;
+    this.cooldownMs = options.cooldownMs || 60_000;
+
+    this.state = CircuitState.CLOSED;
+    this.consecutiveFailures = 0;
+    this.lastFailureTime = 0;
+    this.tripCount = 0;
+    this.probeInFlight = false;
+  }
+
+  canExecute() {
+    if (this.state === CircuitState.CLOSED) {
+      this.probeInFlight = false;
+      return true;
+    }
+
+    const now = Date.now();
+    if (this.state === CircuitState.OPEN) {
+      if (now - this.lastFailureTime >= this.cooldownMs) {
+        this.state = CircuitState.HALF_OPEN;
+        this.probeInFlight = true;
+        return true;
+      }
+      return false;
+    }
+
+    if (this.probeInFlight) {
+      return false;
+    }
+
+    this.probeInFlight = true;
+    return true;
+  }
+
+  recordSuccess() {
+    this.consecutiveFailures = 0;
+    this.state = CircuitState.CLOSED;
+    this.probeInFlight = false;
+  }
+
+  recordFailure(error) {
+    this.consecutiveFailures++;
+    this.lastFailureTime = Date.now();
+    this.probeInFlight = false;
+
+    if (this.state === CircuitState.HALF_OPEN || this.consecutiveFailures >= this.failureThreshold) {
+      if (this.state !== CircuitState.OPEN) {
+        this.tripCount++;
+      }
+      this.state = CircuitState.OPEN;
+    }
+  }
+
+  getState() {
+    this.canExecute(); 
+    return {
+      name: this.name,
+      state: this.state,
+      consecutiveFailures: this.consecutiveFailures,
+      tripCount: this.tripCount,
+      lastFailureTime: this.lastFailureTime
+    };
+  }
+
+  reset() {
+    this.state = CircuitState.CLOSED;
+    this.consecutiveFailures = 0;
+    this.lastFailureTime = 0;
+    this.probeInFlight = false;
+  }
+}
+
+export const defaultBreakers = {
+  duckduckgo: new CircuitBreaker({ name: 'duckduckgo', failureThreshold: 3, cooldownMs: 60_000 }),
+  searxng: new CircuitBreaker({ name: 'searxng', failureThreshold: 3, cooldownMs: 60_000 }),
+  jina: new CircuitBreaker({ name: 'jina', failureThreshold: 3, cooldownMs: 60_000 })
+};

package/server/agent/websearch/resilience/cache.js

@@ -0,0 +1,59 @@
+import crypto from 'node:crypto';
+
+export const TTL_GENERAL_MS = 10 * 60 * 1000;
+export const TTL_NEWS_MS = 2 * 60 * 1000;     
+
+export function computeSearchCacheKey(engine, query, count = 10) {
+  const normalized = `${String(engine || '').toLowerCase()}:${String(query || '').toLowerCase().trim()}:${count}`;
+  return crypto.createHash('sha256').update(normalized).digest('hex');
+}
+
+export class SearchResponseCache {
+  constructor(maxSize = 500) {
+    this.maxSize = Math.max(1, maxSize);
+    this.cache = new Map();
+  }
+
+  get(key) {
+    const entry = this.cache.get(key);
+    if (!entry) return undefined;
+    if (entry.expiresAt && Date.now() > entry.expiresAt) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    this.cache.delete(key);
+    this.cache.set(key, entry);
+    return entry.value;
+  }
+
+  set(key, value, ttlMs = TTL_GENERAL_MS) {
+    if (this.cache.has(key)) {
+      this.cache.delete(key);
+    } else if (this.cache.size >= this.maxSize) {
+      const oldestKey = this.cache.keys().next().value;
+      if (oldestKey !== undefined) this.cache.delete(oldestKey);
+    }
+    const expiresAt = ttlMs > 0 ? Date.now() + ttlMs : 0;
+    this.cache.set(key, { value, expiresAt });
+    return value;
+  }
+
+  has(key) {
+    return this.get(key) !== undefined;
+  }
+
+  delete(key) {
+    return this.cache.delete(key);
+  }
+
+  clear() {
+    this.cache.clear();
+  }
+
+  get size() {
+    return this.cache.size;
+  }
+}
+
+export const defaultSearchCache = new SearchResponseCache(500);

package/server/agent/websearch/resilience/limiter.js

@@ -0,0 +1,93 @@
+export class TokenBucketLimiter {
+  /**
+   * @param {Object} options
+   * @param {number} options.capacity - Maximum bucket burst capacity
+   * @param {number} options.refillRatePerSec - Number of tokens refilled per second
+   * @param {string} [options.name] - Identifier for logging
+   */
+  constructor(options = {}) {
+    this.name = options.name || 'default';
+    this.capacity = Math.max(1, options.capacity || 1);
+    this.refillRatePerSec = Math.max(0.001, options.refillRatePerSec || 1);
+    this.tokens = this.capacity;
+    this.lastRefill = Date.now();
+    this.queue = [];
+  }
+
+  _refill() {
+    const now = Date.now();
+    const elapsedSec = (now - this.lastRefill) / 1000;
+    this.tokens = Math.min(this.capacity, this.tokens + elapsedSec * this.refillRatePerSec);
+    this.lastRefill = now;
+  }
+
+  tryAcquire(cost = 1) {
+    this._refill();
+    if (this.tokens >= cost) {
+      this.tokens -= cost;
+      return true;
+    }
+    return false;
+  }
+
+
+  async acquire(cost = 1, timeoutMs = 15000) {
+    this._refill();
+    if (this.tokens >= cost) {
+      this.tokens -= cost;
+      return true;
+    }
+
+    return new Promise((resolve, reject) => {
+      let timer = null;
+      const entry = {
+        cost,
+        resolve: () => {
+          if (timer) clearTimeout(timer);
+          resolve(true);
+        },
+        reject: (err) => {
+          if (timer) clearTimeout(timer);
+          reject(err);
+        }
+      };
+
+      if (timeoutMs > 0) {
+        timer = setTimeout(() => {
+          const idx = this.queue.indexOf(entry);
+          if (idx !== -1) this.queue.splice(idx, 1);
+          entry.reject(new Error(`Rate limiter "${this.name}" timeout waiting for token`));
+        }, timeoutMs);
+      }
+
+      this.queue.push(entry);
+      this._scheduleProcess();
+    });
+  }
+
+  _scheduleProcess() {
+    if (this.queue.length === 0) return;
+    this._refill();
+
+    while (this.queue.length > 0) {
+      const next = this.queue[0];
+      if (this.tokens >= next.cost) {
+        this.tokens -= next.cost;
+        this.queue.shift();
+        next.resolve();
+      } else {
+      
+        const needed = next.cost - this.tokens;
+        const waitMs = Math.ceil((needed / this.refillRatePerSec) * 1000);
+        setTimeout(() => this._scheduleProcess(), Math.min(Math.max(waitMs, 50), 2000));
+        break;
+      }
+    }
+  }
+}
+
+export const defaultLimiters = {
+  duckduckgo: new TokenBucketLimiter({ name: 'duckduckgo', capacity: 2, refillRatePerSec: 1 }),
+  jina: new TokenBucketLimiter({ name: 'jina', capacity: 5, refillRatePerSec: 20 / 60 }),
+  searxng: new TokenBucketLimiter({ name: 'searxng', capacity: 5, refillRatePerSec: 5 })
+};

package/server/agent/websearch/security/filter.js

@@ -0,0 +1,36 @@
+const BLOCKED_DOMAINS = new Set([
+  'example.invalid',
+  'localhost',
+  'local',
+  'test',
+  'invalid'
+]);
+
+export class DomainFilter {
+  constructor(options = {}) {
+    this.blockedDomains = new Set([...BLOCKED_DOMAINS, ...(options.blockedDomains || [])]);
+    this.allowedDomains = options.allowedDomains ? new Set(options.allowedDomains) : null;
+  }
+
+  isDomainAllowed(hostname) {
+    if (!hostname || typeof hostname !== 'string') return false;
+    const clean = hostname.toLowerCase().trim().replace(/^www\./, '');
+
+    if (this.blockedDomains.has(clean)) return false;
+    for (const blocked of this.blockedDomains) {
+      if (clean.endsWith(`.${blocked}`)) return false;
+    }
+
+    if (this.allowedDomains && this.allowedDomains.size > 0) {
+      if (this.allowedDomains.has(clean)) return true;
+      for (const allowed of this.allowedDomains) {
+        if (clean.endsWith(`.${allowed}`)) return true;
+      }
+      return false;
+    }
+
+    return true;
+  }
+}
+
+export const defaultDomainFilter = new DomainFilter();