kiku-agent 1.0.11 → 1.2.0

package/server/agent/clarificationAnalytics.js

@@ -0,0 +1,477 @@
+
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+
+const DEFAULT_CONFIG = {
+  enabled: true,
+  confidenceThreshold: 0.65,    
+  maxRounds: 2,                 
+  timeoutMs: 30000,             
+  preferLLMQuestions: false,     
+  trackedCategories: [          
+    'timeframe', 'scope', 'depth', 'comparison', 'media_type',
+    'language', 'specific_work', 'aspect_focus', 'date_range', 'content_format'
+  ],
+};
+
+const ANALYTICS_RETENTION_DAYS = 30;
+
+export class ClarificationAnalytics {
+  constructor(options = {}) {
+    this.storagePath = options.storagePath || join(process.cwd(), '.kiku_sessions', 'clarification_analytics.json');
+    this.stats = {
+      totalQueries: 0,
+      totalClarifications: 0,
+      totalSkipped: 0,
+      totalTimedOut: 0,
+      roundsHistogram: {},       
+      categoryStats: {},         
+      confidenceBuckets: {},     
+      recentSessions: [],        
+      dailyStats: {},            
+    };
+    this._maxRecentSessions = options.maxRecentSessions || 100;
+    this._load();
+  }
+
+  _load() {
+    try {
+      if (existsSync(this.storagePath)) {
+        const data = JSON.parse(readFileSync(this.storagePath, 'utf-8'));
+        this.stats = { ...this.stats, ...data };
+      }
+    } catch {
+      
+    }
+  }
+
+  _save() {
+    try {
+      const dir = dirname(this.storagePath);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+      }
+      writeFileSync(this.storagePath, JSON.stringify(this.stats, null, 2), { mode: 0o600 });
+    } catch {
+      
+    }
+  }
+
+  
+  recordClarificationStart(queryId, questions, confidenceScore) {
+    const today = new Date().toISOString().split('T')[0];
+    
+    const session = {
+      queryId,
+      startedAt: Date.now(),
+      confidenceScore,
+      rounds: [{ round: 1, questions: questions.map(q => q.category) }],
+      totalQuestions: questions.length,
+      answered: false,
+      skipped: false,
+      timedOut: false,
+      finalRound: 1,
+      answerSummary: null,
+    };
+
+    
+    this.stats.totalClarifications++;
+    
+    
+    if (!this.stats.dailyStats[today]) {
+      this.stats.dailyStats[today] = { asked: 0, answered: 0, skipped: 0 };
+    }
+    this.stats.dailyStats[today].asked++;
+
+    
+    for (const q of questions) {
+      if (!this.stats.categoryStats[q.category]) {
+        this.stats.categoryStats[q.category] = { asked: 0, answered: 0, totalResponseTimeMs: 0 };
+      }
+      this.stats.categoryStats[q.category].asked++;
+    }
+
+    
+    const bucket = this._getConfidenceBucket(confidenceScore);
+    this.stats.confidenceBuckets[bucket] = (this.stats.confidenceBuckets[bucket] || 0) + 1;
+
+    
+    this.stats.recentSessions.unshift(session);
+    if (this.stats.recentSessions.length > this._maxRecentSessions) {
+      this.stats.recentSessions = this.stats.recentSessions.slice(0, this._maxRecentSessions);
+    }
+
+    this._save();
+    return session;
+  }
+
+  
+  recordClarificationAnswer(queryId, answers, round, responseTimeMs) {
+    const session = this.stats.recentSessions.find(s => s.queryId === queryId);
+    if (!session) return;
+
+    session.answered = true;
+    session.answerSummary = answers;
+    session.finalRound = round;
+    
+    
+    if (round > 1) {
+      session.rounds.push({ round, questions: Object.keys(answers) });
+      session.totalQuestions += Object.keys(answers).length;
+    }
+
+    
+    for (const [category] of Object.entries(answers)) {
+      if (this.stats.categoryStats[category]) {
+        this.stats.categoryStats[category].answered++;
+        this.stats.categoryStats[category].totalResponseTimeMs += responseTimeMs;
+      }
+    }
+
+    
+    const today = new Date().toISOString().split('T')[0];
+    if (this.stats.dailyStats[today]) {
+      this.stats.dailyStats[today].answered++;
+    }
+
+    
+    this.stats.roundsHistogram[round] = (this.stats.roundsHistogram[round] || 0) + 1;
+
+    this._save();
+  }
+
+  
+  recordClarificationSkipped(queryId) {
+    const session = this.stats.recentSessions.find(s => s.queryId === queryId);
+    if (session) {
+      session.skipped = true;
+      session.answered = false;
+    }
+
+    this.stats.totalSkipped++;
+    
+    const today = new Date().toISOString().split('T')[0];
+    if (this.stats.dailyStats[today]) {
+      this.stats.dailyStats[today].skipped++;
+    }
+
+    this._save();
+  }
+
+  
+  recordClarificationTimeout(queryId) {
+    const session = this.stats.recentSessions.find(s => s.queryId === queryId);
+    if (session) {
+      session.timedOut = true;
+    }
+
+    this.stats.totalTimedOut++;
+    this._save();
+  }
+
+  
+  getAnalytics(options = {}) {
+    const { days = 7 } = options;
+    const cutoffDate = new Date();
+    cutoffDate.setDate(cutoffDate.getDate() - days);
+    const cutoffStr = cutoffDate.toISOString().split('T')[0];
+
+    
+    const periodStats = {};
+    for (const [date, stats] of Object.entries(this.stats.dailyStats)) {
+      if (date >= cutoffStr) {
+        periodStats[date] = stats;
+      }
+    }
+
+    
+    let periodAsked = 0, periodAnswered = 0, periodSkipped = 0;
+    for (const stats of Object.values(periodStats)) {
+      periodAsked += stats.asked;
+      periodAnswered += stats.answered;
+      periodSkipped += stats.skipped;
+    }
+
+    
+    const categoryEffectiveness = {};
+    for (const [category, stats] of Object.entries(this.stats.categoryStats)) {
+      const avgResponseTime = stats.answered > 0
+        ? Math.round(stats.totalResponseTimeMs / stats.answered)
+        : 0;
+      categoryEffectiveness[category] = {
+        asked: stats.asked,
+        answered: stats.answered,
+        skipRate: stats.asked > 0
+          ? ((stats.asked - stats.answered) / stats.asked * 100).toFixed(1) + '%'
+          : '0%',
+        avgResponseTimeMs: avgResponseTime,
+      };
+    }
+
+    
+    const sessionsWithRounds = this.stats.recentSessions.filter(s => s.answered);
+    const avgRounds = sessionsWithRounds.length > 0
+      ? (sessionsWithRounds.reduce((sum, s) => sum + s.finalRound, 0) / sessionsWithRounds.length).toFixed(2)
+      : 0;
+
+    return {
+      summary: {
+        totalClarifications: this.stats.totalClarifications,
+        totalAnswered: this.stats.totalClarifications - this.stats.totalSkipped - this.stats.totalTimedOut,
+        totalSkipped: this.stats.totalSkipped,
+        totalTimedOut: this.stats.totalTimedOut,
+        overallSkipRate: this.stats.totalClarifications > 0
+          ? ((this.stats.totalSkipped / this.stats.totalClarifications) * 100).toFixed(1) + '%'
+          : '0%',
+        avgRoundsPerSession: parseFloat(avgRounds),
+      },
+      period: {
+        days,
+        asked: periodAsked,
+        answered: periodAnswered,
+        skipped: periodSkipped,
+        dailyBreakdown: periodStats,
+      },
+      categories: categoryEffectiveness,
+      roundsDistribution: this.stats.roundsHistogram,
+      confidenceBuckets: this.stats.confidenceBuckets,
+      recentSessions: this.stats.recentSessions.slice(0, 10),
+    };
+  }
+
+  
+  getCategoryEffectiveness(category) {
+    const stats = this.stats.categoryStats[category];
+    if (!stats || stats.asked === 0) return 50; 
+
+    const answerRate = stats.answered / stats.asked;
+    const avgResponseTime = stats.answered > 0
+      ? stats.totalResponseTimeMs / stats.answered
+      : 5000;
+    
+    
+    const responseScore = Math.max(0, 1 - (avgResponseTime / 10000));
+    
+    
+    return Math.round((answerRate * 0.7 + responseScore * 0.3) * 100);
+  }
+
+  
+  getTopCategories(n = 5) {
+    const categories = Object.keys(this.stats.categoryStats);
+    const scored = categories.map(cat => ({
+      category: cat,
+      effectiveness: this.getCategoryEffectiveness(cat),
+      asked: this.stats.categoryStats[cat].asked,
+    }));
+    return scored.sort((a, b) => b.effectiveness - a.effectiveness).slice(0, n);
+  }
+
+  
+  cleanup(olderThanDays = ANALYTICS_RETENTION_DAYS) {
+    const cutoffDate = new Date();
+    cutoffDate.setDate(cutoffDate.getDate() - olderThanDays);
+    const cutoffStr = cutoffDate.toISOString().split('T')[0];
+
+    
+    for (const date of Object.keys(this.stats.dailyStats)) {
+      if (date < cutoffStr) {
+        delete this.stats.dailyStats[date];
+      }
+    }
+
+    
+    this.stats.recentSessions = this.stats.recentSessions.filter(
+      s => s.startedAt > cutoffDate.getTime()
+    );
+
+    this._save();
+  }
+
+  _getConfidenceBucket(score) {
+    if (score < 0.2) return '0-20';
+    if (score < 0.4) return '20-40';
+    if (score < 0.6) return '40-60';
+    if (score < 0.8) return '60-80';
+    return '80-100';
+  }
+}
+
+export class ClarificationConfig {
+  constructor(options = {}) {
+    this.storagePath = options.storagePath || join(process.cwd(), '.kiku_sessions', 'clarification_config.json');
+    this.config = { ...DEFAULT_CONFIG };
+    this._load();
+  }
+
+  _load() {
+    try {
+      if (existsSync(this.storagePath)) {
+        const data = JSON.parse(readFileSync(this.storagePath, 'utf-8'));
+        this.config = { ...DEFAULT_CONFIG, ...data };
+      }
+    } catch {
+      
+    }
+  }
+
+  _save() {
+    try {
+      const dir = dirname(this.storagePath);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+      }
+      writeFileSync(this.storagePath, JSON.stringify(this.config, null, 2), { mode: 0o600 });
+    } catch {
+      
+    }
+  }
+
+  
+  getConfig() {
+    return { ...this.config };
+  }
+
+  
+  updateConfig(updates) {
+    
+    if (updates.confidenceThreshold !== undefined) {
+      const val = Number(updates.confidenceThreshold);
+      if (isNaN(val) || val < 0 || val > 1) {
+        throw new Error('confidenceThreshold must be between 0 and 1');
+      }
+      this.config.confidenceThreshold = val;
+    }
+
+    if (updates.maxRounds !== undefined) {
+      const val = Number(updates.maxRounds);
+      if (isNaN(val) || val < 1 || val > 5) {
+        throw new Error('maxRounds must be between 1 and 5');
+      }
+      this.config.maxRounds = val;
+    }
+
+    if (updates.timeoutMs !== undefined) {
+      const val = Number(updates.timeoutMs);
+      if (isNaN(val) || val < 5000 || val > 60000) {
+        throw new Error('timeoutMs must be between 5000 and 60000');
+      }
+      this.config.timeoutMs = val;
+    }
+
+    if (updates.enabled !== undefined) {
+      this.config.enabled = Boolean(updates.enabled);
+    }
+
+    if (updates.preferLLMQuestions !== undefined) {
+      this.config.preferLLMQuestions = Boolean(updates.preferLLMQuestions);
+    }
+
+    this._save();
+    return this.getConfig();
+  }
+
+  
+  reset() {
+    this.config = { ...DEFAULT_CONFIG };
+    this._save();
+    return this.getConfig();
+  }
+
+  
+  get(key) {
+    return this.config[key];
+  }
+
+  
+  set(key, value) {
+    this.config[key] = value;
+    this._save();
+  }
+}
+
+export function formatAnalyticsForCLI(analytics, colors = {}) {
+  const C = colors;
+  const lines = [];
+
+  lines.push(`${C.gold || ''}┌─ Clarification Analytics ──────────────────────────┐${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+
+  
+  const s = analytics.summary;
+  lines.push(`${C.gold || ''}│${C.reset || ''} ${C.bold || ''}Summary${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Total Clarifications: ${C.white || ''}${s.totalClarifications}${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Answered: ${C.green || ''}${s.totalAnswered}${C.reset || ''}  Skipped: ${C.yellow || ''}${s.totalSkipped}${C.reset || ''}  Timeout: ${C.red || ''}${s.totalTimedOut}${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Skip Rate: ${C.dim || ''}${s.overallSkipRate}${C.reset || ''}  Avg Rounds: ${C.dim || ''}${s.avgRoundsPerSession}${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+
+  
+  const topCats = Object.entries(analytics.categories)
+    .sort(([,a], [,b]) => b.asked - a.asked)
+    .slice(0, 5);
+
+  if (topCats.length > 0) {
+    lines.push(`${C.gold || ''}│${C.reset || ''} ${C.bold || ''}Top Categories${C.reset || ''}`);
+    for (const [cat, stats] of topCats) {
+      lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}${cat.padEnd(16)}${C.reset || ''} Asked: ${stats.asked}  Answered: ${stats.answered}  Skip: ${stats.skipRate}`);
+    }
+    lines.push(`${C.gold || ''}│${C.reset || ''}`);
+  }
+
+  
+  const p = analytics.period;
+  lines.push(`${C.gold || ''}│${C.reset || ''} ${C.bold || ''}Last ${p.days} Days${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Asked: ${p.asked}  Answered: ${p.answered}  Skipped: ${p.skipped}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+
+  lines.push(`${C.gold || ''}└────────────────────────────────────────────────────┘${C.reset || ''}`);
+
+  return lines.join('\n');
+}
+
+export function formatConfigForCLI(config, colors = {}) {
+  const C = colors;
+  const lines = [];
+
+  lines.push(`${C.gold || ''}┌─ Clarification Settings ───────────────────────────┐${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''} ${C.bold || ''}Current Configuration${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Enabled:            ${config.enabled ? `${C.green || ''}✓ ON${C.reset || ''}` : `${C.red || ''}✗ OFF${C.reset || ''}`}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Confidence Threshold: ${C.white || ''}${(config.confidenceThreshold * 100).toFixed(0)}%${C.reset || ''} ${C.dim || ''}(ask questions when confidence < this)${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Max Rounds:         ${C.white || ''}${config.maxRounds}${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   Timeout:            ${C.white || ''}${(config.timeoutMs / 1000).toFixed(1)}s${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   LLM Questions:      ${config.preferLLMQuestions ? `${C.green || ''}✓ ON${C.reset || ''}` : `${C.yellow || ''}✗ OFF${C.reset || ''}`}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''} ${C.dim || ''}Commands:${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config threshold <0-100>${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config rounds <1-5>${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config timeout <5-60>${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config on|off${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config llm on|off${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify config reset${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}   ${C.dim || ''}/clarify analytics${C.reset || ''}`);
+  lines.push(`${C.gold || ''}│${C.reset || ''}`);
+
+  lines.push(`${C.gold || ''}└────────────────────────────────────────────────────┘${C.reset || ''}`);
+
+  return lines.join('\n');
+}
+
+export function formatAnalyticsForAPI(analytics) {
+  return {
+    type: 'clarification_analytics',
+    data: analytics,
+    timestamp: Date.now(),
+  };
+}
+
+export default {
+  ClarificationAnalytics,
+  ClarificationConfig,
+  formatAnalyticsForCLI,
+  formatConfigForCLI,
+  formatAnalyticsForAPI,
+  DEFAULT_CONFIG,
+};