kiku-agent 1.2.0 → 1.3.0

package/server/agent/contextCompactor.js

@@ -1,15 +1,46 @@
+import { dynamicToolRouter } from './toolRouter.js';
+
+export function estimateTextTokens(text) {
+  if (!text) return 0;
+  const str = String(text).trim();
+  if (!str) return 0;
+  const dense = (str.match(/[\u0E00-\u0E7F\u3040-\u30FF\u3400-\u9FFF\uF900-\uFAFF\uAC00-\uD7AF]/g) || []).length;
+  const ascii = str.length - dense;
+  const denseTokens = Math.ceil(dense * 1.25);
+  if (ascii === 0) return denseTokens;
+  const remainder = str.replace(/[\u0E00-\u0E7F\u3040-\u30FF\u3400-\u9FFF\uF900-\uFAFF\uAC00-\uD7AF]/g, ' ');
+  const words = remainder.split(/\s+/).filter(Boolean).length;
+  return denseTokens + Math.max(Math.ceil(words * 1.3), Math.ceil(ascii / 3.5));
+}
+
+const messageTokenCache = new WeakMap();
+
+export function estimateMessageTokens(msg) {
+  if (!msg || typeof msg !== 'object') return 0;
+  const contentKey = typeof msg.content === 'string'
+    ? msg.content
+    : (msg.content != null ? JSON.stringify(msg.content) : '');
+  const cached = messageTokenCache.get(msg);
+  if (cached && cached.contentKey === contentKey && cached.toolCalls === msg.tool_calls) {
+    return cached.tokens;
+  }
+  const tokens = (contentKey ? estimateTextTokens(contentKey) : 0)
+    + (msg.tool_calls ? estimateTextTokens(JSON.stringify(msg.tool_calls)) : 0);
+  messageTokenCache.set(msg, { contentKey, toolCalls: msg.tool_calls, tokens });
+  return tokens;
+}
+
 export function estimateTokenCount(textOrMessages) {
   if (!textOrMessages) return 0;
   if (typeof textOrMessages === 'string') {
-    return Math.ceil(textOrMessages.length / 3.5);
+    return estimateTextTokens(textOrMessages);
   }
   if (Array.isArray(textOrMessages)) {
-    let chars = 0;
+    let tokens = 0;
     for (const msg of textOrMessages) {
-      if (typeof msg.content === 'string') chars += msg.content.length;
-      if (msg.tool_calls) chars += JSON.stringify(msg.tool_calls).length;
+      tokens += estimateMessageTokens(msg);
     }
-    return Math.ceil(chars / 3.5);
+    return tokens;
   }
   return 0;
 }
@@ -38,42 +69,225 @@ export class ContextCompactor {
     return points.slice(-8).join(' | ');
   }
 
-  compact(conversationHistory = [], options = {}) {
+
+  _splitWindow(conversationHistory, keepRecent) {
+    const systemMessage = conversationHistory[0]?.role === 'system' ? conversationHistory[0] : null;
+    const nonSystemMessages = systemMessage ? conversationHistory.slice(1) : conversationHistory;
+
+    let start = Math.max(0, nonSystemMessages.length - keepRecent);
+    while (start > 0 && nonSystemMessages[start]?.role === 'tool') start--;
+  
+    while (start < nonSystemMessages.length && nonSystemMessages[start]?.role === 'tool') start++;
+
+    return {
+      systemMessage,
+      olderMessages: nonSystemMessages.slice(0, start),
+      recentMessages: nonSystemMessages.slice(start),
+    };
+  }
+
+  _assemble(systemMessage, olderMessages, recentMessages, summaryText) {
+    const text = summaryText ?? this.summarizeHistorySync(olderMessages);
+    const compacted = [];
+    if (systemMessage) compacted.push(systemMessage);
+    if (text) compacted.push({ role: 'system', content: `[CONVERSATION CONTEXT SUMMARY OF EARLIER MESSAGES: ${text}]` });
+    compacted.push(...recentMessages);
+    return compacted;
+  }
+
+  _shouldCompact(conversationHistory, options) {
     if (!Array.isArray(conversationHistory) || conversationHistory.length <= this.keepRecentMessages + 1) {
-      return conversationHistory;
+      return false;
     }
-
     const currentTokens = estimateTokenCount(conversationHistory);
     const maxTokens = options.maxTokens || this.maxTokens;
     const keepRecent = options.keepRecentMessages || this.keepRecentMessages;
-
     if (currentTokens <= maxTokens && conversationHistory.length <= keepRecent + 2) {
+      return false;
+    }
+    return true;
+  }
+
+  compact(conversationHistory = [], options = {}) {
+    if (!this._shouldCompact(conversationHistory, options)) {
       return conversationHistory;
     }
+    const keepRecent = options.keepRecentMessages || this.keepRecentMessages;
+    const { systemMessage, olderMessages, recentMessages } = this._splitWindow(conversationHistory, keepRecent);
+    if (olderMessages.length === 0) return conversationHistory;
+    return this._assemble(systemMessage, olderMessages, recentMessages);
+  }
 
-    const systemMessage = conversationHistory[0]?.role === 'system' ? conversationHistory[0] : null;
-    const nonSystemMessages = systemMessage ? conversationHistory.slice(1) : conversationHistory;
 
-    if (nonSystemMessages.length <= keepRecent) {
+  async compactAsync(conversationHistory = [], options = {}) {
+    if (!this._shouldCompact(conversationHistory, options)) {
       return conversationHistory;
     }
+    const keepRecent = options.keepRecentMessages || this.keepRecentMessages;
+    const { systemMessage, olderMessages, recentMessages } = this._splitWindow(conversationHistory, keepRecent);
+    if (olderMessages.length === 0) return conversationHistory;
 
-    const olderMessages = nonSystemMessages.slice(0, -keepRecent);
-    const recentMessages = nonSystemMessages.slice(-keepRecent);
+    let llmSummary = null;
+    if (typeof options.summarizer === 'function') {
+      try {
+        llmSummary = String(await options.summarizer(olderMessages) || '').trim() || null;
+      } catch { }
+    }
+    return this._assemble(systemMessage, olderMessages, recentMessages, llmSummary ?? undefined);
+  }
+}
 
-    const summaryText = this.summarizeHistorySync(olderMessages);
-    const summaryContextMessage = {
-      role: 'system',
-      content: `[CONVERSATION CONTEXT SUMMARY OF EARLIER MESSAGES: ${summaryText}]`
-    };
+export const contextCompactor = new ContextCompactor();
 
-    const compacted = [];
-    if (systemMessage) compacted.push(systemMessage);
-    if (summaryText) compacted.push(summaryContextMessage);
-    compacted.push(...recentMessages);
+export function compactToolsForPrompt(query = '', options = {}) {
+  const rawQuery = String(query || '');
+  const router = options.router || dynamicToolRouter;
 
-    return compacted;
+  let routed = router.routeToolsForQuery(rawQuery, options) || [];
+
+
+  if (/photoshoot|magazine|editorial|port|大片|九月刊|写真|นิตยสาร|แฟชั่นเซ็ต/i.test(rawQuery)) {
+ 
+    if (!/contract|ambassador|commercial|代言|พรีเซนเตอร์|ทูต/i.test(rawQuery)) {
+      routed = routed.filter(t => (t.function?.name || t.schema?.name || t.name) !== 'get_brand_endorsements');
+    }
+   
+    const hasGallery = routed.some(t => (t.function?.name || t.schema?.name || t.name) === 'get_gallery_photos');
+    if (!hasGallery) {
+      const galleryTool = router.allTools?.find(t => (t.function?.name || t.schema?.name) === 'get_gallery_photos');
+      if (galleryTool) {
+        routed.unshift(galleryTool);
+      }
+    }
   }
+
+  return routed.map(t => {
+    if (!t.schema && t.function) return { ...t, schema: t.function };
+    if (!t.function && t.schema) return { ...t, function: t.schema };
+    return t;
+  });
+}
+
+const SEARCH_URL_REGEX = /(SearchUrl|SearchQuery|TrailerSearch|commercialSearch|youtubeCommercialSearch|bilibiliSearchUrl)/i;
+const ESSENTIAL_WORK_KEYS = new Set([
+  'id', 'title', 'titleTh', 'titleCn', 'titleEn', 'year', 'role', 'roleCn', 'duration', 'category', 'synopsis'
+]);
+const ESSENTIAL_BRAND_KEYS = new Set([
+  'brand', 'brandNameEn', 'brandNameCn', 'year', 'title', 'category', 'isGlobal', 'highlight', 'tier', 'salesHighlights', 'stat', 'revenue', 'detail'
+]);
+const ESSENTIAL_SALES_KEYS = new Set([
+  'brand', 'brandId', 'stat', 'revenue', 'detail'
+]);
+const ESSENTIAL_TIMELINE_KEYS = new Set([
+  'year', 'period', 'title', 'titleCn', 'era', 'description', 'highlight', 'details'
+]);
+const ESSENTIAL_AWARD_KEYS = new Set([
+  'year', 'award', 'title', 'ceremony', 'category', 'work', 'result'
+]);
+const ESSENTIAL_VIDEO_KEYS = new Set([
+  'id', 'title', 'titleTh', 'titleCn', 'titleEn', 'year', 'releaseYear', 'category', 'duration', 'youtubeId', 'work', 'dramaName'
+]);
+const ESSENTIAL_SONG_KEYS = new Set([
+  'id', 'title', 'titleTh', 'titleCn', 'titleEn', 'year', 'category', 'type', 'work', 'role', 'duration', 'youtubeId', 'youtubeUrl', 'isDirectLink'
+]);
+const ESSENTIAL_ROLE_KEYS = new Set([
+  'workTitle', 'year', 'role', 'roleCn'
+]);
+
+function stripSearchUrls(obj) {
+  if (Array.isArray(obj)) {
+    return obj.map(stripSearchUrls);
+  }
+  if (obj && typeof obj === 'object') {
+    const clean = {};
+    for (const [k, v] of Object.entries(obj)) {
+      if (SEARCH_URL_REGEX.test(k)) continue;
+      clean[k] = stripSearchUrls(v);
+    }
+    return clean;
+  }
+  return obj;
+}
+
+export function compactToolResultForContext(result, options = {}) {
+  if (!result || typeof result !== 'object') return result;
+  const maxTokens = options.maxTokens || 800;
+
+  let stripped = stripSearchUrls(result);
+
+  const projectList = (arr, keys) => {
+    if (!Array.isArray(arr)) return arr;
+    return arr.map(item => {
+      if (!item || typeof item !== 'object') return item;
+      const p = {};
+      for (const k of keys) {
+        if (item[k] !== undefined) p[k] = item[k];
+      }
+      return p;
+    });
+  };
+
+  if (Array.isArray(stripped.works)) stripped.works = projectList(stripped.works, ESSENTIAL_WORK_KEYS);
+  if (Array.isArray(stripped.discography)) stripped.discography = projectList(stripped.discography, ESSENTIAL_SONG_KEYS);
+  if (Array.isArray(stripped.songs)) stripped.songs = projectList(stripped.songs, ESSENTIAL_SONG_KEYS);
+  if (Array.isArray(stripped.brands)) stripped.brands = projectList(stripped.brands, ESSENTIAL_BRAND_KEYS);
+  if (Array.isArray(stripped.salesHighlights)) stripped.salesHighlights = projectList(stripped.salesHighlights, ESSENTIAL_SALES_KEYS);
+  if (Array.isArray(stripped.timeline)) stripped.timeline = projectList(stripped.timeline, ESSENTIAL_TIMELINE_KEYS);
+  if (Array.isArray(stripped.milestones)) stripped.milestones = projectList(stripped.milestones, ESSENTIAL_TIMELINE_KEYS);
+  if (Array.isArray(stripped.eras)) stripped.eras = projectList(stripped.eras, ESSENTIAL_TIMELINE_KEYS);
+  if (Array.isArray(stripped.awards)) stripped.awards = projectList(stripped.awards, ESSENTIAL_AWARD_KEYS);
+  if (Array.isArray(stripped.videos)) stripped.videos = projectList(stripped.videos, ESSENTIAL_VIDEO_KEYS);
+  if (Array.isArray(stripped.musicVideos)) stripped.musicVideos = projectList(stripped.musicVideos, ESSENTIAL_VIDEO_KEYS);
+  if (Array.isArray(stripped.roles)) stripped.roles = projectList(stripped.roles, ESSENTIAL_ROLE_KEYS);
+
+  
+  let serialized = JSON.stringify(stripped);
+  let tokenCount = estimateTextTokens(serialized);
+  if (tokenCount <= maxTokens) return stripped;
+
+  const truncateNode = (node) => {
+    if (Array.isArray(node)) {
+      if (node.length > 5) {
+        return node.slice(0, 5).map(truncateNode);
+      }
+      return node.map(truncateNode);
+    }
+    if (node && typeof node === 'object') {
+      const out = {};
+      for (const [k, v] of Object.entries(node)) {
+        out[k] = truncateNode(v);
+      }
+      return out;
+    }
+    if (typeof node === 'string' && node.length > 240) {
+      return `${node.slice(0, 240)}…`;
+    }
+    return node;
+  };
+
+  stripped = truncateNode(stripped);
+  serialized = JSON.stringify(stripped);
+  tokenCount = estimateTextTokens(serialized);
+
+  if (tokenCount > maxTokens) {
+    const aggressive = (node) => {
+      if (Array.isArray(node)) return node.slice(0, 3).map(aggressive);
+      if (node && typeof node === 'object') {
+        const out = {};
+        for (const [k, v] of Object.entries(node)) {
+          out[k] = aggressive(v);
+        }
+        return out;
+      }
+      if (typeof node === 'string' && node.length > 120) {
+        return `${node.slice(0, 120)}…`;
+      }
+      return node;
+    };
+    stripped = aggressive(stripped);
+  }
+
+  stripped._compactedForContext = true;
+  return stripped;
 }
 
-export const contextCompactor = new ContextCompactor();

package/server/agent/dynamicCatalog.js

@@ -0,0 +1,192 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { KIKU_DATA_DIR } from './appPaths.js';
+
+export const DEFAULT_DYNAMIC_CATALOG_PATH = path.join(KIKU_DATA_DIR, 'dynamic-catalog.json');
+
+const VALID_CATEGORIES = new Set(['brands', 'timeline', 'gallery', 'works', 'events']);
+
+export function createEmptyCatalog() {
+  return {
+    version: 1,
+    updatedAt: new Date().toISOString(),
+    brands: [],
+    timeline: [],
+    gallery: [],
+    works: [],
+    events: []
+  };
+}
+
+export function loadDynamicCatalog(catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH) {
+  try {
+    if (fs.existsSync(catalogPath)) {
+      const raw = fs.readFileSync(catalogPath, 'utf8');
+      const data = JSON.parse(raw);
+      if (data && typeof data === 'object') {
+        return {
+          version: data.version || 1,
+          updatedAt: data.updatedAt || new Date().toISOString(),
+          brands: Array.isArray(data.brands) ? data.brands : [],
+          timeline: Array.isArray(data.timeline) ? data.timeline : [],
+          gallery: Array.isArray(data.gallery) ? data.gallery : [],
+          works: Array.isArray(data.works) ? data.works : [],
+          events: Array.isArray(data.events) ? data.events : []
+        };
+      }
+    }
+  } catch (err) {
+  
+  }
+  return createEmptyCatalog();
+}
+
+export function saveDynamicCatalog(catalog, catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH) {
+  try {
+    const dir = path.dirname(catalogPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    const safeCatalog = {
+      version: 1,
+      updatedAt: new Date().toISOString(),
+      brands: Array.isArray(catalog?.brands) ? catalog.brands : [],
+      timeline: Array.isArray(catalog?.timeline) ? catalog.timeline : [],
+      gallery: Array.isArray(catalog?.gallery) ? catalog.gallery : [],
+      works: Array.isArray(catalog?.works) ? catalog.works : [],
+      events: Array.isArray(catalog?.events) ? catalog.events : []
+    };
+    fs.writeFileSync(catalogPath, JSON.stringify(safeCatalog, null, 2), 'utf8');
+    return true;
+  } catch (err) {
+    return false;
+  }
+}
+
+export function validateCatalogEntry(category, entry) {
+  if (!VALID_CATEGORIES.has(category)) {
+    return { valid: false, reason: `Invalid category "${category}". Allowed: ${Array.from(VALID_CATEGORIES).join(', ')}` };
+  }
+  if (!entry || typeof entry !== 'object') {
+    return { valid: false, reason: 'Entry must be a non-null object' };
+  }
+
+
+  const hasTitle = Boolean(
+    entry.title || entry.titleCn || entry.titleEn || entry.titleTh ||
+    entry.brand || entry.brandNameEn || entry.brandNameCn || entry.name
+  );
+  if (!hasTitle) {
+    return { valid: false, reason: 'Entry must specify at least one title or brand name' };
+  }
+
+  return { valid: true };
+}
+
+export function addDynamicEntry(category, rawEntry, { catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH, trustTier = 1, source = 'manual' } = {}) {
+  const validation = validateCatalogEntry(category, rawEntry);
+  if (!validation.valid) {
+    throw new Error(validation.reason);
+  }
+
+  const catalog = loadDynamicCatalog(catalogPath);
+  const now = new Date().toISOString();
+  const currentYear = new Date().getFullYear();
+
+  const idPrefix = `dyn-${category.slice(0, 3)}-`;
+  const cleanId = rawEntry.id
+    ? String(rawEntry.id).trim()
+    : `${idPrefix}${Date.now().toString(36)}-${crypto.randomBytes(3).toString('hex')}`;
+
+  const entry = {
+    ...rawEntry,
+    id: cleanId,
+    year: Number(rawEntry.year) || currentYear,
+    trustTier: Number(trustTier) || 1,
+    source: String(source || 'manual'),
+    createdAt: rawEntry.createdAt || now,
+    updatedAt: now
+  };
+
+ 
+  const list = catalog[category];
+  const existingIdx = list.findIndex(item => item.id === cleanId);
+  if (existingIdx >= 0) {
+    list[existingIdx] = entry;
+  } else {
+    list.unshift(entry);
+  }
+
+  saveDynamicCatalog(catalog, catalogPath);
+  return entry;
+}
+
+export function getDynamicEntries(category, filter = null, catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH) {
+  if (!VALID_CATEGORIES.has(category)) return [];
+  const catalog = loadDynamicCatalog(catalogPath);
+  let entries = catalog[category] || [];
+
+  if (typeof filter === 'function') {
+    entries = entries.filter(filter);
+  } else if (filter && typeof filter === 'object') {
+    entries = entries.filter(item => {
+      for (const [k, v] of Object.entries(filter)) {
+        if (v !== undefined && item[k] !== v) return false;
+      }
+      return true;
+    });
+  }
+
+  return entries;
+}
+
+export function clearDynamicCatalog(catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH) {
+  const empty = createEmptyCatalog();
+  saveDynamicCatalog(empty, catalogPath);
+  return empty;
+}
+
+export function getDynamicCatalogStats(catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH) {
+  const catalog = loadDynamicCatalog(catalogPath);
+  return {
+    totalEntries: (catalog.brands.length + catalog.timeline.length + catalog.gallery.length + catalog.works.length + catalog.events.length),
+    brandsCount: catalog.brands.length,
+    timelineCount: catalog.timeline.length,
+    galleryCount: catalog.gallery.length,
+    worksCount: catalog.works.length,
+    eventsCount: catalog.events.length,
+    updatedAt: catalog.updatedAt
+  };
+}
+
+export function pruneExpiredEntries(maxDaysOrOptions = 90, maybeCatalogPath = null) {
+  let maxDays = 90;
+  let catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH;
+
+  if (typeof maxDaysOrOptions === 'object' && maxDaysOrOptions !== null) {
+    maxDays = maxDaysOrOptions.maxDays ?? maxDaysOrOptions.ttlDays ?? 90;
+    catalogPath = maxDaysOrOptions.catalogPath || DEFAULT_DYNAMIC_CATALOG_PATH;
+  } else {
+    maxDays = Number(maxDaysOrOptions) || 90;
+    if (maybeCatalogPath) catalogPath = maybeCatalogPath;
+  }
+
+  const catalog = loadDynamicCatalog(catalogPath);
+  const cutoff = Date.now() - (maxDays * 86400000);
+  let prunedCount = 0;
+
+  for (const cat of VALID_CATEGORIES) {
+    const before = catalog[cat].length;
+    catalog[cat] = catalog[cat].filter(item => {
+      const created = item.createdAt ? new Date(item.createdAt).getTime() : 0;
+      return !created || created >= cutoff;
+    });
+    prunedCount += (before - catalog[cat].length);
+  }
+
+  if (prunedCount > 0) {
+    saveDynamicCatalog(catalog, catalogPath);
+  }
+  return { prunedCount };
+}

package/server/agent/envBootstrap.js

@@ -0,0 +1,46 @@
+/**
+ * envBootstrap — one-shot .env loading for the kiku-agent process graph.
+ *
+ * This module MUST be imported before any module that snapshots the
+ * environment at evaluation time (config.js provider defaults, etc.).
+ * cli.js imports it first; youtubeAuth.js imports it defensively because
+ * it is also reached through other entry graphs (mcp stdio, web chat).
+ *
+ * Loading rules (unchanged):
+ *   1. package-root/.env      — development checkouts
+ *   2. <KIKU_DATA_DIR>/.env   — user-owned location (global installs)
+ * Both files are read; existing process.env values always win.
+ */
+
+import fs from 'node:fs';
+import { PACKAGE_ENV_PATH, USER_ENV_PATH } from './envManager.js';
+
+let applied = false;
+let appliedFiles = [];
+
+export function bootstrapEnv({ packageEnvPath = PACKAGE_ENV_PATH, userEnvPath = USER_ENV_PATH, force = false } = {}) {
+  if (applied && !force) {
+    return { applied: true, files: [...appliedFiles] };
+  }
+  applied = true;
+  appliedFiles = [];
+
+  try {
+    for (const envFile of [packageEnvPath, userEnvPath]) {
+      if (!fs.existsSync(envFile)) continue;
+      appliedFiles.push(envFile);
+      const text = fs.readFileSync(envFile, 'utf8');
+      for (const line of text.split('\n')) {
+        const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
+        if (!m || line.trim().startsWith('#')) continue;
+        const key = m[1].trim();
+        const val = m[2].trim().replace(/^["']|["']$/g, '');
+        if (process.env[key] === undefined) process.env[key] = val;
+      }
+    }
+  } catch {}
+
+  return { applied: true, files: [...appliedFiles] };
+}
+
+bootstrapEnv();