gsd-pi 2.33.1 → 2.34.0-dev.6769913

package/dist/resources/extensions/gsd/skill-health.js

@@ -0,0 +1,324 @@
+/**
+ * GSD Skill Health — Dashboard, Staleness, and Heal-Skill Integration (#599)
+ *
+ * Aggregates skill telemetry from metrics.json to surface:
+ *   - Per-skill pass/fail rates, token usage, and trends
+ *   - Staleness warnings for unused skills
+ *   - Declining performance flags
+ *   - Heal-skill suggestions (inspired by glittercowboy's heal-skill command)
+ *
+ * The heal-skill concept: when an agent deviates from what a skill recommends
+ * during execution, detect the drift and propose specific fixes with user
+ * approval before applying. This closes the feedback loop that SkillsBench
+ * research identified as critical for skill quality.
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { getAgentDir } from "@gsd/pi-coding-agent";
+import { formatCost, formatTokenCount, loadLedgerFromDisk } from "./metrics.js";
+import { detectStaleSkills } from "./skill-telemetry.js";
+// ─── Constants ────────────────────────────────────────────────────────────────
+/** Default staleness threshold in days */
+const DEFAULT_STALE_DAYS = 60;
+/** Success rate below this triggers a flag */
+const SUCCESS_RATE_THRESHOLD = 0.70;
+/** Token increase percentage that triggers a "rising" flag */
+const TOKEN_RISE_THRESHOLD = 0.20;
+/** Minimum uses before trend analysis kicks in */
+const MIN_USES_FOR_TREND = 5;
+/** Window size for trend comparison (compare last N to previous N) */
+const TREND_WINDOW = 5;
+// ─── Public API ───────────────────────────────────────────────────────────────
+/**
+ * Generate a full skill health report from metrics data.
+ */
+export function generateSkillHealthReport(basePath, staleDays) {
+    const ledger = loadLedgerFromDisk(basePath);
+    const unitsWithSkills = (ledger?.units ?? []).filter(u => u.skills && u.skills.length > 0);
+    const threshold = staleDays ?? DEFAULT_STALE_DAYS;
+    const skillMap = aggregateBySkill(unitsWithSkills);
+    const skills = Array.from(skillMap.values()).sort((a, b) => b.totalUses - a.totalUses);
+    const staleSkills = detectStaleSkills(unitsWithSkills, threshold);
+    const decliningSkills = skills.filter(s => s.flagged).map(s => s.name);
+    const suggestions = generateSuggestions(skills, staleSkills);
+    return {
+        generatedAt: new Date().toISOString(),
+        totalUnitsWithSkills: unitsWithSkills.length,
+        skills,
+        staleSkills,
+        decliningSkills,
+        suggestions,
+    };
+}
+/**
+ * Format a skill health report for terminal display.
+ */
+export function formatSkillHealthReport(report) {
+    const lines = [];
+    lines.push("Skill Health Report");
+    lines.push("═".repeat(60));
+    lines.push(`Generated: ${report.generatedAt}`);
+    lines.push(`Units with skill data: ${report.totalUnitsWithSkills}`);
+    lines.push("");
+    if (report.skills.length === 0) {
+        lines.push("No skill telemetry data yet. Run auto-mode to start collecting.");
+        lines.push("Skill usage is recorded per-unit in metrics.json.");
+        return lines.join("\n");
+    }
+    // Main table
+    lines.push("Skill                    Uses  Success%  Avg Tokens  Trend     Last Used");
+    lines.push("─".repeat(80));
+    for (const s of report.skills) {
+        const name = s.name.padEnd(24).slice(0, 24);
+        const uses = String(s.totalUses).padStart(5);
+        const success = `${Math.round(s.successRate * 100)}%`.padStart(8);
+        const tokens = formatTokenCount(s.avgTokens).padStart(11);
+        const trend = s.tokenTrend.padEnd(10);
+        const lastUsed = s.staleDays === 0 ? "today" :
+            s.staleDays === 1 ? "1 day ago" :
+                `${s.staleDays} days ago`;
+        const flag = s.flagged ? " ⚠" : "";
+        lines.push(`${name}${uses}${success}${tokens}  ${trend}${lastUsed}${flag}`);
+    }
+    // Stale skills
+    if (report.staleSkills.length > 0) {
+        lines.push("");
+        lines.push("Stale Skills (unused for 60+ days):");
+        for (const name of report.staleSkills) {
+            lines.push(`  ⏸  ${name}`);
+        }
+    }
+    // Declining skills
+    if (report.decliningSkills.length > 0) {
+        lines.push("");
+        lines.push("Declining Skills (flagged for review):");
+        for (const name of report.decliningSkills) {
+            const entry = report.skills.find(s => s.name === name);
+            if (entry?.flagReason) {
+                lines.push(`  ⚠  ${name}: ${entry.flagReason}`);
+            }
+        }
+    }
+    // Suggestions
+    if (report.suggestions.length > 0) {
+        lines.push("");
+        lines.push("Heal Suggestions:");
+        for (const sug of report.suggestions) {
+            const icon = sug.severity === "critical" ? "🔴" : sug.severity === "warning" ? "🟡" : "🔵";
+            lines.push(`  ${icon} ${sug.skillName}: ${sug.message}`);
+        }
+    }
+    return lines.join("\n");
+}
+/**
+ * Format a detailed health view for a single skill.
+ */
+export function formatSkillDetail(basePath, skillName) {
+    const ledger = loadLedgerFromDisk(basePath);
+    const units = (ledger?.units ?? []).filter(u => u.skills?.includes(skillName));
+    const lines = [];
+    lines.push(`Skill Detail: ${skillName}`);
+    lines.push("═".repeat(50));
+    if (units.length === 0) {
+        lines.push("No usage data recorded for this skill.");
+        return lines.join("\n");
+    }
+    const totalTokens = units.reduce((s, u) => s + u.tokens.total, 0);
+    const totalCost = units.reduce((s, u) => s + u.cost, 0);
+    const avgTokens = Math.round(totalTokens / units.length);
+    const avgCost = totalCost / units.length;
+    lines.push(`Total uses: ${units.length}`);
+    lines.push(`Total tokens: ${formatTokenCount(totalTokens)}`);
+    lines.push(`Total cost: ${formatCost(totalCost)}`);
+    lines.push(`Avg tokens/use: ${formatTokenCount(avgTokens)}`);
+    lines.push(`Avg cost/use: ${formatCost(avgCost)}`);
+    lines.push("");
+    // Recent uses
+    lines.push("Recent uses:");
+    const recent = units.slice(-10).reverse();
+    for (const u of recent) {
+        const date = new Date(u.finishedAt).toISOString().slice(0, 10);
+        lines.push(`  ${date}  ${u.id.padEnd(20)}  ${formatTokenCount(u.tokens.total).padStart(8)} tokens  ${formatCost(u.cost)}`);
+    }
+    // Check for SKILL.md existence
+    const skillPath = join(getAgentDir(), "skills", skillName, "SKILL.md");
+    if (existsSync(skillPath)) {
+        const stat = require("node:fs").statSync(skillPath);
+        lines.push("");
+        lines.push(`SKILL.md: ${skillPath}`);
+        lines.push(`Last modified: ${stat.mtime.toISOString().slice(0, 10)}`);
+    }
+    return lines.join("\n");
+}
+/**
+ * Build the heal-skill prompt for a post-unit hook.
+ * This is the GSD-integrated version of glittercowboy's heal-skill concept.
+ *
+ * The prompt instructs the agent to:
+ * 1. Detect which skill was loaded during the completed unit
+ * 2. Analyze whether the agent deviated from the skill's instructions
+ * 3. If deviations found, propose specific fixes (not auto-apply)
+ * 4. Write suggestions to a review queue for human approval
+ */
+export function buildHealSkillPrompt(unitId) {
+    return `## Skill Heal Analysis
+
+Analyze the just-completed unit (${unitId}) for skill drift.
+
+### Steps
+
+1. **Identify loaded skill**: Check which SKILL.md file was read during this unit.
+   If no skill was loaded, write "No skill loaded — skipping heal analysis" and stop.
+
+2. **Read the skill**: Load the SKILL.md that was used.
+
+3. **Compare execution to skill guidance**: Review what the agent actually did vs what
+   the skill recommended. Look for:
+   - API patterns the skill recommended that the agent did differently
+   - Error handling approaches the skill specified but the agent bypassed
+   - Conventions the skill documented that the agent ignored
+   - Outdated instructions in the skill that caused errors or retries
+
+4. **Assess drift severity**:
+   - **None**: Agent followed skill correctly → write "No drift detected" to the summary and stop
+   - **Minor**: Agent found a better approach but skill isn't wrong → note in KNOWLEDGE.md
+   - **Significant**: Skill has outdated or incorrect guidance → propose fix
+
+5. **If significant drift found**, write a heal suggestion to \`.gsd/skill-review-queue.md\`:
+
+\`\`\`markdown
+### {skill-name} (flagged {date})
+- **Unit:** ${unitId}
+- **Issue:** {1-2 sentence description}
+- **Root cause:** {outdated API / incorrect pattern / missing context}
+- **Proposed fix:**
+  - File: SKILL.md
+  - Section: {section name}
+  - Current: {quote the incorrect text}
+  - Suggested: {the corrected text}
+- **Action:** [ ] Reviewed [ ] Updated [ ] Dismissed
+\`\`\`
+
+**Important:** Do NOT modify the skill directly. Write the suggestion to the review queue.
+The SkillsBench research shows that human-curated skills outperform auto-generated ones by +16.2pp.
+The human review step is what makes this valuable.`;
+}
+/**
+ * Compute stale skills that should be added to avoid_skills.
+ * Returns only skills not already in the avoid list.
+ */
+export function computeStaleAvoidList(basePath, currentAvoidList, staleDays) {
+    const ledger = loadLedgerFromDisk(basePath);
+    const units = (ledger?.units ?? []).filter(u => u.skills && u.skills.length > 0);
+    const stale = detectStaleSkills(units, staleDays ?? DEFAULT_STALE_DAYS);
+    const avoidSet = new Set(currentAvoidList);
+    return stale.filter(s => !avoidSet.has(s));
+}
+// ─── Internals ────────────────────────────────────────────────────────────────
+function aggregateBySkill(units) {
+    const map = new Map();
+    for (const u of units) {
+        if (!u.skills)
+            continue;
+        for (const skill of u.skills) {
+            let entry = map.get(skill);
+            if (!entry) {
+                entry = { uses: [] };
+                map.set(skill, entry);
+            }
+            entry.uses.push(u);
+        }
+    }
+    const result = new Map();
+    const now = Date.now();
+    for (const [name, { uses }] of map) {
+        const totalTokens = uses.reduce((s, u) => s + u.tokens.total, 0);
+        const totalCost = uses.reduce((s, u) => s + u.cost, 0);
+        const avgTokens = Math.round(totalTokens / uses.length);
+        const avgCost = totalCost / uses.length;
+        // Success rate: units that didn't have excessive retries (proxy: low tool call count relative to messages)
+        // Without direct retry tracking, use a heuristic: success if toolCalls < assistantMessages * 20
+        const successCount = uses.filter(u => u.toolCalls < u.assistantMessages * 20).length;
+        const successRate = uses.length > 0 ? successCount / uses.length : 1;
+        // Token trend
+        const tokenTrend = computeTokenTrend(uses);
+        // Last used
+        const lastUsed = Math.max(...uses.map(u => u.finishedAt));
+        const staleDays = Math.floor((now - lastUsed) / (24 * 60 * 60 * 1000));
+        // Flag conditions
+        let flagged = false;
+        let flagReason;
+        if (uses.length >= MIN_USES_FOR_TREND) {
+            if (successRate < SUCCESS_RATE_THRESHOLD) {
+                flagged = true;
+                flagReason = `Success rate ${Math.round(successRate * 100)}% (below ${Math.round(SUCCESS_RATE_THRESHOLD * 100)}% threshold)`;
+            }
+            else if (tokenTrend === "rising") {
+                flagged = true;
+                flagReason = `Token usage trending upward (${Math.round(TOKEN_RISE_THRESHOLD * 100)}%+ increase)`;
+            }
+        }
+        result.set(name, {
+            name,
+            totalUses: uses.length,
+            successRate,
+            avgTokens,
+            tokenTrend,
+            lastUsed,
+            staleDays,
+            avgCost,
+            flagged,
+            flagReason,
+        });
+    }
+    return result;
+}
+function computeTokenTrend(uses) {
+    if (uses.length < MIN_USES_FOR_TREND * 2)
+        return "stable";
+    // Sort by start time
+    const sorted = [...uses].sort((a, b) => a.startedAt - b.startedAt);
+    const window = Math.min(TREND_WINDOW, Math.floor(sorted.length / 2));
+    const recent = sorted.slice(-window);
+    const previous = sorted.slice(-window * 2, -window);
+    const recentAvg = recent.reduce((s, u) => s + u.tokens.total, 0) / recent.length;
+    const previousAvg = previous.reduce((s, u) => s + u.tokens.total, 0) / previous.length;
+    if (previousAvg === 0)
+        return "stable";
+    const change = (recentAvg - previousAvg) / previousAvg;
+    if (change > TOKEN_RISE_THRESHOLD)
+        return "rising";
+    if (change < -TOKEN_RISE_THRESHOLD)
+        return "declining";
+    return "stable";
+}
+function generateSuggestions(skills, staleSkills) {
+    const suggestions = [];
+    for (const skill of skills) {
+        if (skill.totalUses >= MIN_USES_FOR_TREND && skill.successRate < SUCCESS_RATE_THRESHOLD) {
+            suggestions.push({
+                skillName: skill.name,
+                trigger: "declining_success",
+                message: `Success rate dropped to ${Math.round(skill.successRate * 100)}% over ${skill.totalUses} uses. Review SKILL.md for outdated patterns.`,
+                severity: skill.successRate < 0.5 ? "critical" : "warning",
+            });
+        }
+        if (skill.tokenTrend === "rising" && skill.totalUses >= MIN_USES_FOR_TREND * 2) {
+            suggestions.push({
+                skillName: skill.name,
+                trigger: "rising_tokens",
+                message: `Token usage trending upward. Skill may be causing inefficient execution patterns.`,
+                severity: "info",
+            });
+        }
+    }
+    for (const name of staleSkills) {
+        suggestions.push({
+            skillName: name,
+            trigger: "stale",
+            message: `Not used in ${DEFAULT_STALE_DAYS}+ days. Consider archiving or updating.`,
+            severity: "info",
+        });
+    }
+    return suggestions;
+}

package/dist/resources/extensions/gsd/{skill-telemetry.ts → skill-telemetry.js}

@@ -10,118 +10,102 @@
  *   2. During execution, recordSkillRead() tracks explicit SKILL.md reads
  *   3. At unit completion, getAndClearSkills() returns the loaded list for metrics
  */
-
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync } from "node:fs";
 import { join } from "node:path";
 import { getAgentDir } from "@gsd/pi-coding-agent";
-
 // ─── In-memory state ──────────────────────────────────────────────────────────
-
 /** Skills available in the system prompt for the current unit */
-let availableSkills: string[] = [];
-
+let availableSkills = [];
 /** Skills explicitly read (SKILL.md loaded) during the current unit */
-const activelyLoadedSkills = new Set<string>();
-
+const activelyLoadedSkills = new Set();
 // ─── Public API ───────────────────────────────────────────────────────────────
-
 /**
  * Capture the list of available skill names at dispatch time.
  * Called before each unit starts.
  */
-export function captureAvailableSkills(): void {
-  const skillsDir = join(getAgentDir(), "skills");
-  availableSkills = listSkillNames(skillsDir);
-  activelyLoadedSkills.clear();
+export function captureAvailableSkills() {
+    const skillsDir = join(getAgentDir(), "skills");
+    availableSkills = listSkillNames(skillsDir);
+    activelyLoadedSkills.clear();
 }
-
 /**
  * Record that a skill was actively loaded (its SKILL.md was read).
  * Call this when the agent reads a SKILL.md file.
  */
-export function recordSkillRead(skillName: string): void {
-  activelyLoadedSkills.add(skillName);
+export function recordSkillRead(skillName) {
+    activelyLoadedSkills.add(skillName);
 }
-
 /**
  * Get the skill names for the current unit and clear state.
  * Returns actively loaded skills if any, otherwise available skills.
  * This gives the most useful signal: if the agent read specific skills,
  * report those; otherwise report what was available.
  */
-export function getAndClearSkills(): string[] {
-  const result = activelyLoadedSkills.size > 0
-    ? Array.from(activelyLoadedSkills)
-    : [...availableSkills];
-  availableSkills = [];
-  activelyLoadedSkills.clear();
-  return result;
+export function getAndClearSkills() {
+    const result = activelyLoadedSkills.size > 0
+        ? Array.from(activelyLoadedSkills)
+        : [...availableSkills];
+    availableSkills = [];
+    activelyLoadedSkills.clear();
+    return result;
 }
-
 /**
  * Reset all telemetry state. Called when auto-mode stops.
  */
-export function resetSkillTelemetry(): void {
-  availableSkills = [];
-  activelyLoadedSkills.clear();
+export function resetSkillTelemetry() {
+    availableSkills = [];
+    activelyLoadedSkills.clear();
 }
-
 /**
  * Get last-used timestamps for all skills from metrics data.
  * Returns a Map from skill name to most recent ms timestamp.
  */
-export function getSkillLastUsed(units: Array<{ finishedAt: number; skills?: string[] }>): Map<string, number> {
-  const lastUsed = new Map<string, number>();
-  for (const u of units) {
-    if (!u.skills) continue;
-    for (const skill of u.skills) {
-      const existing = lastUsed.get(skill) ?? 0;
-      if (u.finishedAt > existing) {
-        lastUsed.set(skill, u.finishedAt);
-      }
+export function getSkillLastUsed(units) {
+    const lastUsed = new Map();
+    for (const u of units) {
+        if (!u.skills)
+            continue;
+        for (const skill of u.skills) {
+            const existing = lastUsed.get(skill) ?? 0;
+            if (u.finishedAt > existing) {
+                lastUsed.set(skill, u.finishedAt);
+            }
+        }
     }
-  }
-  return lastUsed;
+    return lastUsed;
 }
-
 /**
  * Detect stale skills — those not used within the given threshold (in days).
  * Returns skill names that should be deprioritized.
  */
-export function detectStaleSkills(
-  units: Array<{ finishedAt: number; skills?: string[] }>,
-  thresholdDays: number,
-): string[] {
-  if (thresholdDays <= 0) return [];
-
-  const lastUsed = getSkillLastUsed(units);
-  const cutoff = Date.now() - (thresholdDays * 24 * 60 * 60 * 1000);
-  const stale: string[] = [];
-
-  // Check all installed skills, not just those with usage data
-  const skillsDir = join(getAgentDir(), "skills");
-  const installed = listSkillNames(skillsDir);
-
-  for (const skill of installed) {
-    const lastTs = lastUsed.get(skill);
-    if (lastTs === undefined || lastTs < cutoff) {
-      stale.push(skill);
+export function detectStaleSkills(units, thresholdDays) {
+    if (thresholdDays <= 0)
+        return [];
+    const lastUsed = getSkillLastUsed(units);
+    const cutoff = Date.now() - (thresholdDays * 24 * 60 * 60 * 1000);
+    const stale = [];
+    // Check all installed skills, not just those with usage data
+    const skillsDir = join(getAgentDir(), "skills");
+    const installed = listSkillNames(skillsDir);
+    for (const skill of installed) {
+        const lastTs = lastUsed.get(skill);
+        if (lastTs === undefined || lastTs < cutoff) {
+            stale.push(skill);
+        }
     }
-  }
-
-  return stale;
+    return stale;
 }
-
 // ─── Internals ────────────────────────────────────────────────────────────────
-
-function listSkillNames(skillsDir: string): string[] {
-  if (!existsSync(skillsDir)) return [];
-  try {
-    return readdirSync(skillsDir, { withFileTypes: true })
-      .filter(d => d.isDirectory() && !d.name.startsWith("."))
-      .filter(d => existsSync(join(skillsDir, d.name, "SKILL.md")))
-      .map(d => d.name);
-  } catch {
-    return [];
-  }
+function listSkillNames(skillsDir) {
+    if (!existsSync(skillsDir))
+        return [];
+    try {
+        return readdirSync(skillsDir, { withFileTypes: true })
+            .filter(d => d.isDirectory() && !d.name.startsWith("."))
+            .filter(d => existsSync(join(skillsDir, d.name, "SKILL.md")))
+            .map(d => d.name);
+    }
+    catch {
+        return [];
+    }
 }