@cloudstreamsoftware/claude-tools 1.0.0 → 1.1.0

package/scripts/lib/skill-effectiveness.js

@@ -0,0 +1,486 @@
+/**
+ * Skill Effectiveness Tracker
+ *
+ * Tracks which skills lead to success vs. corrections/failures.
+ * Used to:
+ * 1. Prioritize skill suggestions (show most effective first)
+ * 2. Identify skills that need improvement
+ * 3. Deprecate skills with low success rates
+ * 4. Calculate ROI of skill library
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { ensureDir, getDateTimeString, getClaudeDir } = require('./utils');
+
+// Storage paths
+const KNOWLEDGE_DIR = path.join(getClaudeDir(), 'knowledge');
+const EFFECTIVENESS_FILE = path.join(KNOWLEDGE_DIR, 'skill-effectiveness.json');
+
+// Thresholds
+const THRESHOLDS = {
+  MIN_USES_FOR_CONFIDENCE: 5, // Minimum uses before trusting effectiveness
+  LOW_EFFECTIVENESS: 0.6, // Below this is concerning
+  HIGH_EFFECTIVENESS: 0.85, // Above this is excellent
+  DEPRECATION_THRESHOLD: 0.4, // Below this after 10+ uses = deprecate candidate
+  PROMOTION_THRESHOLD: 0.9, // Above this = promote to shared skills
+};
+
+/**
+ * Default effectiveness data structure
+ */
+function createDefaultData() {
+  return {
+    version: '1.0.0',
+    lastUpdated: getDateTimeString(),
+    skills: {},
+    summary: {
+      totalApplications: 0,
+      totalSuccesses: 0,
+      totalCorrections: 0,
+      averageEffectiveness: 0,
+    },
+  };
+}
+
+/**
+ * Load effectiveness data
+ */
+function loadEffectivenessData() {
+  ensureDir(KNOWLEDGE_DIR);
+
+  try {
+    if (fs.existsSync(EFFECTIVENESS_FILE)) {
+      const content = fs.readFileSync(EFFECTIVENESS_FILE, 'utf8');
+      return JSON.parse(content);
+    }
+  } catch (err) {
+    console.error(`[SkillEffectiveness] Error loading data: ${err.message}`);
+  }
+
+  return createDefaultData();
+}
+
+/**
+ * Save effectiveness data
+ */
+function saveEffectivenessData(data) {
+  ensureDir(KNOWLEDGE_DIR);
+  data.lastUpdated = getDateTimeString();
+
+  try {
+    fs.writeFileSync(EFFECTIVENESS_FILE, JSON.stringify(data, null, 2), 'utf8');
+    return true;
+  } catch (err) {
+    console.error(`[SkillEffectiveness] Error saving data: ${err.message}`);
+    return false;
+  }
+}
+
+/**
+ * Record a skill application
+ * @param {string} skillName - Name of the skill applied
+ * @param {string} skillType - 'builtin' or 'learned'
+ * @param {object} context - { filePath, technologies, complianceMode }
+ */
+function recordApplication(skillName, skillType = 'builtin', context = {}) {
+  const data = loadEffectivenessData();
+
+  if (!data.skills[skillName]) {
+    data.skills[skillName] = {
+      type: skillType,
+      applications: 0,
+      successes: 0,
+      corrections: 0,
+      errors: 0,
+      firstUsed: getDateTimeString(),
+      lastUsed: null,
+      contexts: [],
+      effectiveness: 0,
+      trend: 'new',
+    };
+  }
+
+  const skill = data.skills[skillName];
+  skill.applications++;
+  skill.lastUsed = getDateTimeString();
+
+  // Track context (keep last 10)
+  skill.contexts.push({
+    timestamp: getDateTimeString(),
+    filePath: context.filePath || null,
+    technologies: context.technologies || [],
+    complianceMode: context.complianceMode || null,
+  });
+  if (skill.contexts.length > 10) {
+    skill.contexts.shift();
+  }
+
+  // Update summary
+  data.summary.totalApplications++;
+
+  saveEffectivenessData(data);
+
+  return {
+    skillName,
+    applicationNumber: skill.applications,
+  };
+}
+
+/**
+ * Record a success after skill application
+ * @param {string} skillName - Name of the skill
+ * @param {object} details - Optional details about the success
+ */
+function recordSuccess(skillName, details = {}) {
+  const data = loadEffectivenessData();
+
+  if (!data.skills[skillName]) {
+    // Skill not tracked, ignore
+    return null;
+  }
+
+  const skill = data.skills[skillName];
+  skill.successes++;
+
+  // Recalculate effectiveness
+  skill.effectiveness = calculateEffectiveness(skill);
+
+  // Update trend
+  skill.trend = determineTrend(skill);
+
+  // Update summary
+  data.summary.totalSuccesses++;
+  data.summary.averageEffectiveness = calculateAverageEffectiveness(data.skills);
+
+  saveEffectivenessData(data);
+
+  return {
+    skillName,
+    newEffectiveness: skill.effectiveness,
+    trend: skill.trend,
+  };
+}
+
+/**
+ * Record a correction after skill application
+ * @param {string} skillName - Name of the skill
+ * @param {object} correction - { type, description }
+ */
+function recordCorrection(skillName, correction = {}) {
+  const data = loadEffectivenessData();
+
+  if (!data.skills[skillName]) {
+    return null;
+  }
+
+  const skill = data.skills[skillName];
+  skill.corrections++;
+
+  // Recalculate effectiveness
+  skill.effectiveness = calculateEffectiveness(skill);
+
+  // Update trend
+  skill.trend = determineTrend(skill);
+
+  // Update summary
+  data.summary.totalCorrections++;
+  data.summary.averageEffectiveness = calculateAverageEffectiveness(data.skills);
+
+  saveEffectivenessData(data);
+
+  // Check if skill should be flagged
+  const analysis = analyzeSkill(skill, skillName);
+
+  return {
+    skillName,
+    newEffectiveness: skill.effectiveness,
+    trend: skill.trend,
+    analysis,
+  };
+}
+
+/**
+ * Record an error after skill application
+ * @param {string} skillName - Name of the skill
+ * @param {string} errorMessage - Error description
+ */
+function recordError(skillName, errorMessage = '') {
+  const data = loadEffectivenessData();
+
+  if (!data.skills[skillName]) {
+    return null;
+  }
+
+  const skill = data.skills[skillName];
+  skill.errors++;
+
+  // Errors count against effectiveness
+  skill.effectiveness = calculateEffectiveness(skill);
+  skill.trend = determineTrend(skill);
+
+  saveEffectivenessData(data);
+
+  return {
+    skillName,
+    newEffectiveness: skill.effectiveness,
+    errorCount: skill.errors,
+  };
+}
+
+/**
+ * Calculate effectiveness score for a skill
+ */
+function calculateEffectiveness(skill) {
+  const total = skill.applications;
+  if (total === 0) return 0;
+
+  // Successes are positive, corrections and errors are negative
+  const positive = skill.successes;
+  const negative = skill.corrections + skill.errors;
+
+  // Effectiveness = successes / (successes + negatives)
+  // If no outcomes recorded yet, use applications as denominator
+  const outcomes = positive + negative;
+  if (outcomes === 0) return 0.5; // Neutral if no outcomes
+
+  return positive / outcomes;
+}
+
+/**
+ * Determine trend based on recent performance
+ */
+function determineTrend(skill) {
+  if (skill.applications < THRESHOLDS.MIN_USES_FOR_CONFIDENCE) {
+    return 'new';
+  }
+
+  if (skill.effectiveness >= THRESHOLDS.HIGH_EFFECTIVENESS) {
+    return 'excellent';
+  }
+
+  if (skill.effectiveness >= 0.7) {
+    return 'good';
+  }
+
+  if (skill.effectiveness >= THRESHOLDS.LOW_EFFECTIVENESS) {
+    return 'moderate';
+  }
+
+  if (skill.effectiveness >= THRESHOLDS.DEPRECATION_THRESHOLD) {
+    return 'concerning';
+  }
+
+  return 'poor';
+}
+
+/**
+ * Calculate average effectiveness across all skills
+ */
+function calculateAverageEffectiveness(skills) {
+  const skillsWithData = Object.values(skills).filter(
+    (s) => s.applications >= THRESHOLDS.MIN_USES_FOR_CONFIDENCE
+  );
+
+  if (skillsWithData.length === 0) return 0;
+
+  const sum = skillsWithData.reduce((acc, s) => acc + s.effectiveness, 0);
+  return sum / skillsWithData.length;
+}
+
+/**
+ * Analyze a skill and return recommendations
+ */
+function analyzeSkill(skill, skillName) {
+  const analysis = {
+    status: 'ok',
+    recommendations: [],
+  };
+
+  // Not enough data
+  if (skill.applications < THRESHOLDS.MIN_USES_FOR_CONFIDENCE) {
+    analysis.status = 'insufficient_data';
+    analysis.recommendations.push('Need more usage data for reliable analysis');
+    return analysis;
+  }
+
+  // Deprecation candidate
+  if (skill.effectiveness < THRESHOLDS.DEPRECATION_THRESHOLD) {
+    analysis.status = 'deprecation_candidate';
+    analysis.recommendations.push(`Low effectiveness (${Math.round(skill.effectiveness * 100)}%)`);
+    analysis.recommendations.push('Consider reviewing and updating this skill');
+    analysis.recommendations.push('May need deprecation if not improved');
+  }
+
+  // Low effectiveness
+  else if (skill.effectiveness < THRESHOLDS.LOW_EFFECTIVENESS) {
+    analysis.status = 'needs_improvement';
+    analysis.recommendations.push(
+      `Below target effectiveness (${Math.round(skill.effectiveness * 100)}%)`
+    );
+    analysis.recommendations.push('Review recent corrections for patterns');
+  }
+
+  // Promotion candidate
+  else if (skill.effectiveness >= THRESHOLDS.PROMOTION_THRESHOLD && skill.type === 'learned') {
+    analysis.status = 'promotion_candidate';
+    analysis.recommendations.push(`High effectiveness (${Math.round(skill.effectiveness * 100)}%)`);
+    analysis.recommendations.push('Consider sharing with team via /skill-submit');
+  }
+
+  // Excellent performance
+  else if (skill.effectiveness >= THRESHOLDS.HIGH_EFFECTIVENESS) {
+    analysis.status = 'excellent';
+  }
+
+  return analysis;
+}
+
+/**
+ * Get effectiveness report for all skills
+ */
+function getEffectivenessReport() {
+  const data = loadEffectivenessData();
+
+  const skills = Object.entries(data.skills).map(([name, skill]) => ({
+    name,
+    ...skill,
+    analysis: analyzeSkill(skill, name),
+  }));
+
+  // Sort by effectiveness (highest first)
+  skills.sort((a, b) => b.effectiveness - a.effectiveness);
+
+  // Categorize
+  const excellent = skills.filter((s) => s.trend === 'excellent');
+  const good = skills.filter((s) => s.trend === 'good');
+  const moderate = skills.filter((s) => s.trend === 'moderate');
+  const concerning = skills.filter((s) => s.trend === 'concerning' || s.trend === 'poor');
+  const newSkills = skills.filter((s) => s.trend === 'new');
+
+  return {
+    summary: data.summary,
+    categories: {
+      excellent,
+      good,
+      moderate,
+      concerning,
+      new: newSkills,
+    },
+    promotionCandidates: skills.filter((s) => s.analysis.status === 'promotion_candidate'),
+    deprecationCandidates: skills.filter((s) => s.analysis.status === 'deprecation_candidate'),
+    needsImprovement: skills.filter((s) => s.analysis.status === 'needs_improvement'),
+  };
+}
+
+/**
+ * Get top effective skills for a given context
+ * Used by skill-injector to prioritize suggestions
+ */
+function getTopEffectiveSkills(matchedSkills, limit = 3) {
+  const data = loadEffectivenessData();
+
+  // Score matched skills by effectiveness
+  const scored = matchedSkills.map((skill) => {
+    const effectivenessData = data.skills[skill.skill] || null;
+
+    let effectivenessScore = 0.5; // Default neutral
+    if (effectivenessData && effectivenessData.applications >= THRESHOLDS.MIN_USES_FOR_CONFIDENCE) {
+      effectivenessScore = effectivenessData.effectiveness;
+    }
+
+    return {
+      ...skill,
+      effectivenessScore,
+      effectivenessData,
+      combinedScore: skill.score * 0.6 + effectivenessScore * 0.4, // 60% match, 40% effectiveness
+    };
+  });
+
+  // Sort by combined score
+  scored.sort((a, b) => b.combinedScore - a.combinedScore);
+
+  return scored.slice(0, limit);
+}
+
+/**
+ * Generate markdown report
+ */
+function generateMarkdownReport() {
+  const report = getEffectivenessReport();
+
+  let md = `# Skill Effectiveness Report\n\n`;
+  md += `Generated: ${getDateTimeString()}\n\n`;
+
+  // Summary
+  md += `## Summary\n\n`;
+  md += `- Total Applications: ${report.summary.totalApplications}\n`;
+  md += `- Total Successes: ${report.summary.totalSuccesses}\n`;
+  md += `- Total Corrections: ${report.summary.totalCorrections}\n`;
+  md += `- Average Effectiveness: ${Math.round(report.summary.averageEffectiveness * 100)}%\n\n`;
+
+  // Excellent skills
+  if (report.categories.excellent.length > 0) {
+    md += `## Excellent Skills (85%+)\n\n`;
+    for (const skill of report.categories.excellent) {
+      md += `- **${skill.name}**: ${Math.round(skill.effectiveness * 100)}% (${skill.applications} uses)\n`;
+    }
+    md += `\n`;
+  }
+
+  // Promotion candidates
+  if (report.promotionCandidates.length > 0) {
+    md += `## Promotion Candidates (Share with Team)\n\n`;
+    for (const skill of report.promotionCandidates) {
+      md += `- **${skill.name}**: ${Math.round(skill.effectiveness * 100)}% effectiveness\n`;
+      md += `  - Run: \`/skill-submit ${skill.name}\`\n`;
+    }
+    md += `\n`;
+  }
+
+  // Needs improvement
+  if (report.needsImprovement.length > 0) {
+    md += `## Skills Needing Improvement\n\n`;
+    for (const skill of report.needsImprovement) {
+      md += `- **${skill.name}**: ${Math.round(skill.effectiveness * 100)}%\n`;
+      for (const rec of skill.analysis.recommendations) {
+        md += `  - ${rec}\n`;
+      }
+    }
+    md += `\n`;
+  }
+
+  // Deprecation candidates
+  if (report.deprecationCandidates.length > 0) {
+    md += `## Deprecation Candidates\n\n`;
+    for (const skill of report.deprecationCandidates) {
+      md += `- **${skill.name}**: ${Math.round(skill.effectiveness * 100)}% (${skill.applications} uses)\n`;
+      md += `  - Consider deprecating or major revision\n`;
+    }
+    md += `\n`;
+  }
+
+  return md;
+}
+
+module.exports = {
+  // Recording functions
+  recordApplication,
+  recordSuccess,
+  recordCorrection,
+  recordError,
+
+  // Query functions
+  getEffectivenessReport,
+  getTopEffectiveSkills,
+  analyzeSkill,
+
+  // Report generation
+  generateMarkdownReport,
+
+  // Data access
+  loadEffectivenessData,
+
+  // Thresholds (for external access)
+  THRESHOLDS,
+};