kiku-agent 1.2.0 → 1.3.1

package/server/agent/research.js

@@ -2,19 +2,219 @@ import { ResearchSubagent } from './subagent.js';
 import { executeTool } from './tools.js';
 import { eventBus, Events } from './eventBus.js';
 import { mapConcurrent } from './suggestions.js';
+import { checkUrlLiveness } from './webSearch.js';
 
-const CATALOG_TOOLS = Object.freeze([
+export const CATALOG_TOOLS = Object.freeze([
   'global_search', 'get_profile_and_fandom', 'search_works', 'get_work_detail',
   'get_character_roles', 'search_discography', 'get_music_videos',
   'get_media_links', 'get_brand_endorsements', 'get_career_timeline',
   'get_awards', 'get_social_feed',
 ]);
-const WEB_TOOLS = Object.freeze(['web_search', 'web_extract', 'fetch_webpage']);
-const VERIFY_TOOLS = Object.freeze([...CATALOG_TOOLS, ...WEB_TOOLS]);
+export const WEB_TOOLS = Object.freeze([
+  'web_search', 'web_extract', 'fetch_webpage',
+  'search_chinese_web', 'search_web_images', 'search_news', 'search_encyclopedia'
+]);
+export const VERIFY_TOOLS = Object.freeze([...CATALOG_TOOLS, ...WEB_TOOLS]);
+
+export function cleanUrl(rawUrl = '') {
+  let url = String(rawUrl || '').trim();
+  const hasTrailingEllipsis = /\.{3,}$|…$/.test(url);
+  let prev;
+  do {
+    prev = url;
+    url = url.replace(/^[`"'\(\[<|\s]+/, '');
+    url = url.replace(/[`"'\)\]>|\s]+$/, '');
+    url = url.replace(/(?:(?<!\.)\.(?!\.)|[;:!?,])+$/, '');
+  } while (url !== prev && url.length > 0);
+  if (hasTrailingEllipsis && !url.includes('...') && !url.includes('…')) {
+    url = `${url.replace(/\/+$/, '')}/...`;
+  }
+  return url;
+}
+
+export function extractAllRealUrls(toolResults = []) {
+  const urls = new Set();
+  const add = (candidate) => {
+    if (!candidate || typeof candidate !== 'string') return;
+    const cleaned = cleanUrl(candidate);
+    if (/^https?:\/\/[a-zA-Z0-9]/i.test(cleaned) && !cleaned.includes('...') && !cleaned.includes('…')) {
+      urls.add(cleaned);
+    }
+  };
+
+  const traverse = (obj, depth = 0) => {
+    if (!obj || depth > 5) return;
+    if (typeof obj === 'string') {
+      const found = obj.match(/https?:\/\/[^\s<>()"'|\[\]`]+/g) || [];
+      for (const f of found) add(f);
+    } else if (Array.isArray(obj)) {
+      for (const item of obj) traverse(item, depth + 1);
+    } else if (typeof obj === 'object') {
+      if (typeof obj.url === 'string') add(obj.url);
+      if (typeof obj.link === 'string') add(obj.link);
+      for (const key of Object.keys(obj)) {
+        traverse(obj[key], depth + 1);
+      }
+    }
+  };
+
+  for (const item of toolResults) {
+    traverse(item);
+  }
+  return [...urls];
+}
+
+export function resolveTruncatedUrl(truncatedUrl, knownUrls = []) {
+  const cleaned = cleanUrl(truncatedUrl);
+  const normalized = cleaned.replace(/…/g, '...');
+  if (!normalized.includes('...')) return normalized;
+
+  if (!knownUrls || knownUrls.length === 0) {
+    return normalized
+      .replace(/\/+\.\.\.[^\/]*$/, '')
+      .replace(/\/\.\.\.\//g, '/')
+      .replace(/\.{3,}/g, '');
+  }
+
+  try {
+    let domain = '';
+    try {
+      const parsed = new URL(normalized.replace(/\.\.\./g, 'placeholder'));
+      domain = parsed.hostname.replace(/^www\./, '').toLowerCase();
+    } catch {
+      const match = normalized.match(/https?:\/\/(?:www\.)?([^\/\s:]+)/i);
+      if (match) domain = match[1].toLowerCase();
+    }
+
+    const candidates = knownUrls.filter(known => {
+      if (!domain) return true;
+      try {
+        const kHost = new URL(known).hostname.replace(/^www\./, '').toLowerCase();
+        return kHost === domain || kHost.endsWith('.' + domain) || domain.endsWith('.' + kHost);
+      } catch {
+        return known.toLowerCase().includes(domain);
+      }
+    });
+
+    const pool = candidates.length > 0 ? candidates : knownUrls;
+
+    const rawTokens = decodeURIComponent(normalized)
+      .replace(/^https?:\/\/(?:www\.)?/i, '')
+      .split(/[\/\?#&]+/)
+      .flatMap(part => part.split(/\.\.\./))
+      .map(part => part.trim().replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, ''))
+      .filter(part => part.length >= 3 && !/^\.+$/.test(part));
+
+    const idTokens = rawTokens.filter(t => /^\d{4,}$/.test(t));
+
+    let bestMatch = null;
+    let highestScore = 0;
+
+    for (const known of pool) {
+      const decodedKnown = decodeURIComponent(known).toLowerCase();
+      let score = 0;
+
+      for (const id of idTokens) {
+        if (decodedKnown.includes(id.toLowerCase())) {
+          score += 50;
+        }
+      }
+
+      for (const token of rawTokens) {
+        const lowerToken = token.toLowerCase();
+        if (decodedKnown.includes(lowerToken)) {
+          score += 10;
+        }
+      }
+
+      if (score > highestScore) {
+        highestScore = score;
+        bestMatch = known;
+      }
+    }
+
+    if (bestMatch && (highestScore >= 20 || (idTokens.length > 0 && highestScore >= 50))) {
+      return bestMatch;
+    }
+  } catch {}
+
+  return normalized
+    .replace(/\/+\.\.\.[^\/]*$/, '')
+    .replace(/\/\.\.\.\//g, '/')
+    .replace(/\.{3,}/g, '');
+}
+
+export function expandTruncatedUrlsInText(text, knownUrls = []) {
+  if (!text) return text;
+  return text.replace(/https?:\/\/[^\s<>()"'\n`|\]]+(?:\.\.\.|…)[^\s<>()"'\n`|\]]*/g, (match) => {
+    let urlCandidate = match;
+    let trailingPunct = '';
+    const punctMatch = urlCandidate.match(/([,;:!?]+)$/);
+    if (punctMatch) {
+      trailingPunct = punctMatch[1];
+      urlCandidate = urlCandidate.slice(0, -trailingPunct.length);
+    }
+    const resolved = resolveTruncatedUrl(urlCandidate, knownUrls);
+    return resolved + trailingPunct;
+  });
+}
+
+export function uniqueUrls(text, knownUrls = []) {
+  const matches = String(text || '').match(/https?:\/\/[^\s<>()\]`"'|]+/g) || [];
+  const resolvedList = matches.map(url => resolveTruncatedUrl(cleanUrl(url), knownUrls));
+  const seen = new Set();
+  const results = [];
+
+  for (const u of resolvedList) {
+    if (!u || !/^https?:\/\/[a-zA-Z0-9]/i.test(u)) continue;
+    if (u.includes('...') || u.includes('…')) continue;
+    const cleanU = cleanUrl(u);
+    if (!seen.has(cleanU)) {
+      seen.add(cleanU);
+      results.push(cleanU);
+    }
+  }
+
+  for (const known of knownUrls) {
+    try {
+      const cleanKnown = cleanUrl(known);
+      if (cleanKnown.includes('...') || cleanKnown.includes('…')) continue;
+      const kObj = new URL(cleanKnown);
+      const host = kObj.hostname.replace(/^www\./, '');
+      if (host.length > 4 && text.toLowerCase().includes(host.toLowerCase()) && !seen.has(cleanKnown)) {
+        seen.add(cleanKnown);
+        results.push(cleanKnown);
+      }
+    } catch {}
+  }
+
+  return results;
+}
+
+export async function filterAliveSources(urls = [], realToolUrls = [], options = {}) {
+  if (!Array.isArray(urls) || urls.length === 0) return [];
+  if (options.checkLiveness === false || process.env.KIKU_SKIP_URL_LIVENESS === 'true') {
+    return urls.map(cleanUrl).filter(u => u && !u.includes('...') && !u.includes('…'));
+  }
+  const realSet = new Set(realToolUrls.map(u => cleanUrl(u).toLowerCase()));
+  const cleanedUrls = urls.map(cleanUrl);
+
+  const verified = [];
+  await mapConcurrent(cleanedUrls, 5, async (u) => {
+    if (!u || u.includes('...') || u.includes('…')) return;
+
+    const isDirectFromTool = realSet.has(u.toLowerCase());
 
-function uniqueUrls(text) {
-  const matches = String(text || '').match(/https?:\/\/[^\s<>()\]]+/g) || [];
-  return [...new Set(matches.map(url => url.replace(/[.,;:!?]+$/, '')))];
+    const check = await checkUrlLiveness(u, { timeoutMs: 2500 });
+    if (check.alive) {
+      verified.push(u);
+    } else if (isDirectFromTool && check.reason !== 'not_found' && check.reason !== 'dns_failed') {
+      verified.push(u);
+    }
+  });
+
+  const verifiedSet = new Set(verified);
+  return cleanedUrls.filter(u => verifiedSet.has(u));
 }
 
 function stableSignature(name, args) {
@@ -63,6 +263,7 @@ export class ResearchOrchestrator {
     this.maxIterations = Math.min(Math.max(options.maxIterations || 3, 1), 5);
     this.active = null;
     this.lastResult = null;
+    this._runLock = null;
   }
 
   getStatus() {
@@ -88,15 +289,31 @@ export class ResearchOrchestrator {
     if (!query) throw new Error('A research question is required.');
 
 
+    while (this._runLock) {
+      try { await this._runLock; } catch { }
+    }
+    let releaseLock;
+    this._runLock = new Promise(resolve => { releaseLock = resolve; });
+
+    try {
+      return await this._run(query, options);
+    } finally {
+      this._runLock = null;
+      releaseLock();
+    }
+  }
+
+  async _run(query, options = {}) {
     if (this.active && this.active.question === query && this.active.promise) {
       try {
         return await this.active.promise;
       } catch (err) {
-    
+
+        console.warn('[Kiku research] previous identical run failed, starting a new one:', err?.message || err);
       }
     }
 
-  
+
     if (this.active) {
       try {
         this.active.controller.abort();
@@ -140,6 +357,11 @@ export class ResearchOrchestrator {
     );
     const concurrency = isLocalProvider ? 1 : Math.min(this.maxWorkers, plan.length);
 
+    const overallTimeoutMs = Number(options.timeout || process.env.KIKU_RESEARCH_TIMEOUT_MS) || (isLocalProvider ? 240000 : 120000);
+    const overallTimer = setTimeout(() => {
+      controller.abort(new Error(`Deep research timed out after ${overallTimeoutMs / 1000}s`));
+    }, overallTimeoutMs);
+
     const executeJob = async () => {
       try {
         const settled = await mapConcurrent(plan, concurrency, async task => {
@@ -153,29 +375,50 @@ export class ResearchOrchestrator {
             label: task.label,
           });
 
-          const worker = new ResearchSubagent({
-            client: this.client,
-            executeTool: cachedExecute,
-            allowedTools: task.tools,
-            maxIterations: this.maxIterations,
-          });
-          const result = await worker.run(task.objective, {
-            signal: controller.signal,
-            skillContext: options.skillContext,
-            onProgress: event => {
-              if (event.type === 'tool') status.toolCalls += 1;
-              options.onProgress?.({ ...event, workerId: task.id, label: task.label });
-            },
-          });
-          status.state = 'complete';
-          options.onProgress?.({ type: 'worker-complete', id: task.id, label: task.label });
-          eventBus.emit(Events.AGENT_PLAN_STEP_UPDATE, {
-            planId: job.id,
-            stepId: task.id,
-            state: 'complete',
-            label: task.label,
-          });
-          return { id: task.id, label: task.label, ...result };
+          const workerTimeoutMs = Number(options.workerTimeoutMs || process.env.KIKU_RESEARCH_WORKER_TIMEOUT_MS) || (isLocalProvider ? 90000 : 45000);
+          const workerAbort = new AbortController();
+          const timer = setTimeout(() => {
+            workerAbort.abort(new Error(`Worker ${task.label} timed out after ${workerTimeoutMs / 1000}s`));
+          }, workerTimeoutMs);
+          const onParentAbort = () => workerAbort.abort(controller.signal.reason);
+          controller.signal.addEventListener('abort', onParentAbort, { once: true });
+
+          try {
+            const worker = new ResearchSubagent({
+              client: this.client,
+              executeTool: cachedExecute,
+              allowedTools: task.tools,
+              maxIterations: this.maxIterations,
+            });
+            const result = await worker.run(task.objective, {
+              signal: workerAbort.signal,
+              skillContext: options.skillContext,
+              onProgress: event => {
+                if (event.type === 'tool') status.toolCalls += 1;
+                options.onProgress?.({ ...event, workerId: task.id, label: task.label });
+              },
+            });
+            clearTimeout(timer);
+            status.state = 'complete';
+            options.onProgress?.({ type: 'worker-complete', id: task.id, label: task.label });
+            eventBus.emit(Events.AGENT_PLAN_STEP_UPDATE, {
+              planId: job.id,
+              stepId: task.id,
+              state: 'complete',
+              label: task.label,
+            });
+            return { id: task.id, label: task.label, ...result };
+          } catch (workerErr) {
+            clearTimeout(timer);
+            if (controller.signal.aborted) throw workerErr;
+            console.warn(`[Kiku Research] Worker ${task.label} failed or timed out:`, workerErr?.message || workerErr);
+            status.state = 'failed';
+            status.error = workerErr?.message || 'Worker timed out';
+            return { id: task.id, label: task.label, answer: '', error: status.error, toolCalls: [] };
+          } finally {
+            clearTimeout(timer);
+            controller.signal.removeEventListener('abort', onParentAbort);
+          }
         });
 
         const reports = settled.map((entry, index) => {
@@ -188,19 +431,49 @@ export class ResearchOrchestrator {
         });
         if (controller.signal.aborted) throw controller.signal.reason || new DOMException('Aborted', 'AbortError');
 
+        // Extract all real URLs discovered across all tool calls
+        const realToolUrls = extractAllRealUrls(reports.flatMap(r => (r.toolCalls || []).map(tc => tc.result)));
+        const realToolUrlSet = new Set(realToolUrls);
+        const recordRealUrls = (found) => {
+          for (const u of found) {
+            if (!realToolUrlSet.has(u)) {
+              realToolUrlSet.add(u);
+              realToolUrls.push(u);
+            }
+          }
+        };
+        for (const promise of sharedToolCache.values()) {
+          try {
+            const res = await promise;
+            recordRealUrls(extractAllRealUrls([res]));
+          } catch {}
+        }
+
         const usable = reports.filter(report => report.answer);
         if (usable.length === 0) {
           const { webExtract } = await import('./webSearch.js');
           const fallbackRes = await webExtract(query);
           if (fallbackRes?.text) {
             usable.push({ label: 'Live Web Intelligence', answer: fallbackRes.text, sources: fallbackRes.sources || [] });
+            recordRealUrls(extractAllRealUrls([fallbackRes]));
           } else {
             throw new Error('All research workers failed and no web intelligence was returned.');
           }
         }
+
+        usable.forEach(report => {
+          if (report.answer) {
+            report.answer = expandTruncatedUrlsInText(report.answer, realToolUrls);
+          }
+        });
+
         const synthesisPrompt = options.mode === 'verify'
-          ? 'Return a clear verdict: VERIFIED, UNVERIFIED, CONFLICTING, or FALSE. Explain briefly and cite only URLs present in the reports.'
-          : 'Synthesize a comprehensive, detailed, and polished answer with verified facts. Cite only URLs present in the reports.';
+          ? 'Return a clear verdict: VERIFIED, UNVERIFIED, CONFLICTING, or FALSE. Explain briefly and cite only valid source URLs. Never truncate URLs with "..." or elide them.'
+          : `Synthesize a comprehensive, detailed, and polished answer with verified facts.
+CRITICAL FORMATTING DIRECTIVE FOR SOURCES AND TABLES:
+- In any markdown tables, lists, or citations, ALWAYS output the FULL, COMPLETE, UNTRUNCATED URL.
+- NEVER shorten, elide, or truncate URLs with "..." (e.g. NEVER write "https://domain.com/play/..." or "https://...").
+- Fans and users MUST be able to copy, click, or browse each source URL directly. Every URL in tables and lists must be 100% complete and fully formed.`;
         const synthesis = await this.client.chat({
           messages: [
             { role: 'system', content: `You are the Kiku research editor. ${synthesisPrompt} Never invent a source, date, title, or URL. Match the language of the question. Do not reveal hidden reasoning.` },
@@ -210,8 +483,13 @@ export class ResearchOrchestrator {
           stream: false,
           signal: controller.signal,
         });
-        const answer = synthesis?.content || usable.map(report => `### ${report.label}\n${report.answer}`).join('\n\n');
-        const sources = uniqueUrls(usable.map(report => report.answer).join('\n'));
+
+        const rawAnswer = synthesis?.content || usable.map(report => `### ${report.label}\n${report.answer}`).join('\n\n');
+        const answer = expandTruncatedUrlsInText(rawAnswer, realToolUrls);
+        const combinedText = usable.map(report => report.answer).join('\n') + '\n' + answer;
+        const rawSources = uniqueUrls(combinedText, realToolUrls);
+        const sources = await filterAliveSources(rawSources, realToolUrls, options);
+
         const result = {
           id: job.id,
           mode: options.mode || 'research',
@@ -231,6 +509,7 @@ export class ResearchOrchestrator {
         });
         return result;
       } finally {
+        clearTimeout(overallTimer);
         options.signal?.removeEventListener('abort', abortParent);
         if (this.active?.id === job.id) {
           this.active = null;

package/server/agent/routingInterceptor.js

@@ -0,0 +1,77 @@
+export class RoutingInterceptor {
+  constructor() {
+    this.historyCallSignatures = new Set();
+  }
+
+  interceptToolCall(toolCall, turnIndex = 0) {
+    if (!toolCall) return { valid: false, reason: 'Empty tool call' };
+
+    const name = toolCall.name || toolCall.function?.name || '';
+    const rawParams = toolCall.params || toolCall.function?.arguments || {};
+    let params = rawParams;
+    if (typeof params === 'string') {
+      try { params = JSON.parse(params); } catch { params = {}; }
+    }
+
+    const signature = `${name}:${JSON.stringify(params)}`;
+
+  
+    if (name === 'global_search' && turnIndex === 0) {
+      const query = (params.query || '').toLowerCase();
+
+      if (/仙剑|花间令|剧|drama|role|costume|character|เรื่อง|ซีรีส์|ละคร|ตัวละคร|บทบาท|in blossom|sword and fairy/i.test(query)) {
+        return {
+          rerouted: true,
+          replacement: { name: 'search_works', params: { query: params.query } },
+          reason: 'Auto-rerouted from illegal first-turn global_search to search_works'
+        };
+      }
+
+    
+      if (/view from the wall|เพลง|song|single|ost|title|壁上观|如纱|不苦|find the way/i.test(query)) {
+        return {
+          rerouted: true,
+          replacement: { name: 'search_discography', params: { query: params.query } },
+          reason: 'Auto-rerouted from illegal first-turn global_search to search_discography'
+        };
+      }
+
+    
+      if (/award|honor|รางวัล|เกียรติยศ|东方风云榜|风云榜|weibo night/i.test(query)) {
+        return {
+          rerouted: true,
+          replacement: { name: 'get_awards', params: { query: params.query } },
+          reason: 'Auto-rerouted from illegal first-turn global_search to get_awards'
+        };
+      }
+    }
+
+    if (name === 'get_brand_endorsements') {
+      const brandParam = String(params.brand || params.query || '').toLowerCase();
+      if (/by my poi|be my poi|\bpoi\b|เพลง|mv|video|single|song|ost|stage/i.test(brandParam)) {
+        return {
+          rerouted: true,
+          replacement: { name: 'get_music_videos', params: { query: params.brand || params.query } },
+          reason: 'Auto-rerouted from misrouted get_brand_endorsements to get_music_videos for media/song query'
+        };
+      }
+    }
+
+   
+    if (this.historyCallSignatures.has(signature)) {
+      return {
+        aborted: true,
+        reason: 'Duplicate call signature detected. Forcing synthesis.'
+      };
+    }
+
+    this.historyCallSignatures.add(signature);
+    return { valid: true, toolCall };
+  }
+
+  reset() {
+    this.historyCallSignatures.clear();
+  }
+}
+
+export const routingInterceptor = new RoutingInterceptor();

package/server/agent/rumoredProjects.js

@@ -0,0 +1,15 @@
+export const RUMORED_OR_EXTERNAL_PROJECTS = [
+  {
+    id: 'qi-xing-cai',
+    keywords: ['七星彩', 'qi xiang cai', 'qi xing cai', 'ชีซิงไฉ่', 'seven star color'],
+    titleCn: '七星彩',
+    titleEn: 'Qi Xing Cai (Seven Star Color)',
+    status: 'rumored_casting',
+    producer: 'Youku (优酷) x Stellar Media (恒星引力传媒)',
+    originalNovel: '《七星彩》 by 明月珰 (Mingyue Dang)',
+    leads: '纪澄 (Ji Cheng) & 沈彻 (Shen Che)',
+    details: 'โปรเจกต์ซีรีส์จีนโบราณฟอร์มยักษ์ของ Youku ร่วมกับค่ายเหิงซิงอิ่นลี่ (ผู้สร้างของรักของข้า Love Between Fairy and Devil) ดัดแปลงจากนิยายของหมิงเยว่ตัง เล่าเรื่องราวของจี้เฉิง บุตรสาวตระกูลพ่อค้าที่ใช้ปฏิภาณไหวพริบเอาตัวรอดจนได้พบกับแม่ทัพเสิ่นเช่อ',
+    buzzReason: 'เหตุผลที่ชาวเน็ตให้ความสนใจอย่างมากคือ กระแสข่าวลือแคสติ้งนักแสดงนำ (Casting Rumors / 选角瓜) โดยมีการลือชื่อนักแสดงหญิงแถวหน้าหลายคนรวมถึง จวีจิ้งอี, หลูอวี้เสี่ยว ฯลฯ แต่ปัจจุบันยังไม่มีการประกาศยืนยันนักแสดงทางการจากทางค่ายหรือสตูดิโอ',
+    disambiguation: 'คนละผลงานกับซีรีส์ 《千香》 (The Eternal Fragrance / พันกลิ่นบุปผา) ที่จวีจิ้งอีนำแสดงร่วมกับ ซ่งเวยหลง'
+  }
+];