gsd-pi 2.18.0 → 2.20.0

package/src/resources/extensions/gsd/skill-health.ts

@@ -0,0 +1,417 @@
+/**
+ * GSD Skill Health — Dashboard, Staleness, and Heal-Skill Integration (#599)
+ *
+ * Aggregates skill telemetry from metrics.json to surface:
+ *   - Per-skill pass/fail rates, token usage, and trends
+ *   - Staleness warnings for unused skills
+ *   - Declining performance flags
+ *   - Heal-skill suggestions (inspired by glittercowboy's heal-skill command)
+ *
+ * The heal-skill concept: when an agent deviates from what a skill recommends
+ * during execution, detect the drift and propose specific fixes with user
+ * approval before applying. This closes the feedback loop that SkillsBench
+ * research identified as critical for skill quality.
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { getAgentDir } from "@gsd/pi-coding-agent";
+import type { UnitMetrics, MetricsLedger } from "./metrics.js";
+import { formatCost, formatTokenCount, loadLedgerFromDisk } from "./metrics.js";
+import { getSkillLastUsed, detectStaleSkills } from "./skill-telemetry.js";
+
+// ─── Types ────────────────────────────────────────────────────────────────────
+
+export interface SkillHealthEntry {
+  name: string;
+  totalUses: number;
+  /** Success rate: units with this skill that completed without retry */
+  successRate: number;
+  /** Average tokens per unit when this skill is loaded */
+  avgTokens: number;
+  /** Token trend over recent uses */
+  tokenTrend: "stable" | "rising" | "declining";
+  /** Timestamp of most recent use */
+  lastUsed: number;
+  /** Days since last use */
+  staleDays: number;
+  /** Average cost per unit when this skill is loaded */
+  avgCost: number;
+  /** Whether this skill is flagged for review */
+  flagged: boolean;
+  /** Reason for flag, if any */
+  flagReason?: string;
+}
+
+export interface SkillHealthReport {
+  generatedAt: string;
+  totalUnitsWithSkills: number;
+  skills: SkillHealthEntry[];
+  staleSkills: string[];
+  decliningSkills: string[];
+  suggestions: SkillHealSuggestion[];
+}
+
+export interface SkillHealSuggestion {
+  skillName: string;
+  trigger: "declining_success" | "rising_tokens" | "high_retry_rate" | "stale";
+  message: string;
+  severity: "info" | "warning" | "critical";
+}
+
+// ─── Constants ────────────────────────────────────────────────────────────────
+
+/** Default staleness threshold in days */
+const DEFAULT_STALE_DAYS = 60;
+
+/** Success rate below this triggers a flag */
+const SUCCESS_RATE_THRESHOLD = 0.70;
+
+/** Token increase percentage that triggers a "rising" flag */
+const TOKEN_RISE_THRESHOLD = 0.20;
+
+/** Minimum uses before trend analysis kicks in */
+const MIN_USES_FOR_TREND = 5;
+
+/** Window size for trend comparison (compare last N to previous N) */
+const TREND_WINDOW = 5;
+
+// ─── Public API ───────────────────────────────────────────────────────────────
+
+/**
+ * Generate a full skill health report from metrics data.
+ */
+export function generateSkillHealthReport(basePath: string, staleDays?: number): SkillHealthReport {
+  const ledger = loadLedgerFromDisk(basePath);
+  const unitsWithSkills = (ledger?.units ?? []).filter(u => u.skills && u.skills.length > 0);
+  const threshold = staleDays ?? DEFAULT_STALE_DAYS;
+
+  const skillMap = aggregateBySkill(unitsWithSkills);
+  const skills = Array.from(skillMap.values()).sort((a, b) => b.totalUses - a.totalUses);
+  const staleSkills = detectStaleSkills(unitsWithSkills, threshold);
+  const decliningSkills = skills.filter(s => s.flagged).map(s => s.name);
+  const suggestions = generateSuggestions(skills, staleSkills);
+
+  return {
+    generatedAt: new Date().toISOString(),
+    totalUnitsWithSkills: unitsWithSkills.length,
+    skills,
+    staleSkills,
+    decliningSkills,
+    suggestions,
+  };
+}
+
+/**
+ * Format a skill health report for terminal display.
+ */
+export function formatSkillHealthReport(report: SkillHealthReport): string {
+  const lines: string[] = [];
+
+  lines.push("Skill Health Report");
+  lines.push("═".repeat(60));
+  lines.push(`Generated: ${report.generatedAt}`);
+  lines.push(`Units with skill data: ${report.totalUnitsWithSkills}`);
+  lines.push("");
+
+  if (report.skills.length === 0) {
+    lines.push("No skill telemetry data yet. Run auto-mode to start collecting.");
+    lines.push("Skill usage is recorded per-unit in metrics.json.");
+    return lines.join("\n");
+  }
+
+  // Main table
+  lines.push("Skill                    Uses  Success%  Avg Tokens  Trend     Last Used");
+  lines.push("─".repeat(80));
+
+  for (const s of report.skills) {
+    const name = s.name.padEnd(24).slice(0, 24);
+    const uses = String(s.totalUses).padStart(5);
+    const success = `${Math.round(s.successRate * 100)}%`.padStart(8);
+    const tokens = formatTokenCount(s.avgTokens).padStart(11);
+    const trend = s.tokenTrend.padEnd(10);
+    const lastUsed = s.staleDays === 0 ? "today" :
+      s.staleDays === 1 ? "1 day ago" :
+      `${s.staleDays} days ago`;
+    const flag = s.flagged ? " ⚠" : "";
+    lines.push(`${name}${uses}${success}${tokens}  ${trend}${lastUsed}${flag}`);
+  }
+
+  // Stale skills
+  if (report.staleSkills.length > 0) {
+    lines.push("");
+    lines.push("Stale Skills (unused for 60+ days):");
+    for (const name of report.staleSkills) {
+      lines.push(`  ⏸  ${name}`);
+    }
+  }
+
+  // Declining skills
+  if (report.decliningSkills.length > 0) {
+    lines.push("");
+    lines.push("Declining Skills (flagged for review):");
+    for (const name of report.decliningSkills) {
+      const entry = report.skills.find(s => s.name === name);
+      if (entry?.flagReason) {
+        lines.push(`  ⚠  ${name}: ${entry.flagReason}`);
+      }
+    }
+  }
+
+  // Suggestions
+  if (report.suggestions.length > 0) {
+    lines.push("");
+    lines.push("Heal Suggestions:");
+    for (const sug of report.suggestions) {
+      const icon = sug.severity === "critical" ? "🔴" : sug.severity === "warning" ? "🟡" : "🔵";
+      lines.push(`  ${icon} ${sug.skillName}: ${sug.message}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+/**
+ * Format a detailed health view for a single skill.
+ */
+export function formatSkillDetail(basePath: string, skillName: string): string {
+  const ledger = loadLedgerFromDisk(basePath);
+  const units = (ledger?.units ?? []).filter(u => u.skills?.includes(skillName));
+  const lines: string[] = [];
+
+  lines.push(`Skill Detail: ${skillName}`);
+  lines.push("═".repeat(50));
+
+  if (units.length === 0) {
+    lines.push("No usage data recorded for this skill.");
+    return lines.join("\n");
+  }
+
+  const totalTokens = units.reduce((s, u) => s + u.tokens.total, 0);
+  const totalCost = units.reduce((s, u) => s + u.cost, 0);
+  const avgTokens = Math.round(totalTokens / units.length);
+  const avgCost = totalCost / units.length;
+
+  lines.push(`Total uses: ${units.length}`);
+  lines.push(`Total tokens: ${formatTokenCount(totalTokens)}`);
+  lines.push(`Total cost: ${formatCost(totalCost)}`);
+  lines.push(`Avg tokens/use: ${formatTokenCount(avgTokens)}`);
+  lines.push(`Avg cost/use: ${formatCost(avgCost)}`);
+  lines.push("");
+
+  // Recent uses
+  lines.push("Recent uses:");
+  const recent = units.slice(-10).reverse();
+  for (const u of recent) {
+    const date = new Date(u.finishedAt).toISOString().slice(0, 10);
+    lines.push(`  ${date}  ${u.id.padEnd(20)}  ${formatTokenCount(u.tokens.total).padStart(8)} tokens  ${formatCost(u.cost)}`);
+  }
+
+  // Check for SKILL.md existence
+  const skillPath = join(getAgentDir(), "skills", skillName, "SKILL.md");
+  if (existsSync(skillPath)) {
+    const stat = require("node:fs").statSync(skillPath);
+    lines.push("");
+    lines.push(`SKILL.md: ${skillPath}`);
+    lines.push(`Last modified: ${stat.mtime.toISOString().slice(0, 10)}`);
+  }
+
+  return lines.join("\n");
+}
+
+/**
+ * Build the heal-skill prompt for a post-unit hook.
+ * This is the GSD-integrated version of glittercowboy's heal-skill concept.
+ *
+ * The prompt instructs the agent to:
+ * 1. Detect which skill was loaded during the completed unit
+ * 2. Analyze whether the agent deviated from the skill's instructions
+ * 3. If deviations found, propose specific fixes (not auto-apply)
+ * 4. Write suggestions to a review queue for human approval
+ */
+export function buildHealSkillPrompt(unitId: string): string {
+  return `## Skill Heal Analysis
+
+Analyze the just-completed unit (${unitId}) for skill drift.
+
+### Steps
+
+1. **Identify loaded skill**: Check which SKILL.md file was read during this unit.
+   If no skill was loaded, write "No skill loaded — skipping heal analysis" and stop.
+
+2. **Read the skill**: Load the SKILL.md that was used.
+
+3. **Compare execution to skill guidance**: Review what the agent actually did vs what
+   the skill recommended. Look for:
+   - API patterns the skill recommended that the agent did differently
+   - Error handling approaches the skill specified but the agent bypassed
+   - Conventions the skill documented that the agent ignored
+   - Outdated instructions in the skill that caused errors or retries
+
+4. **Assess drift severity**:
+   - **None**: Agent followed skill correctly → write "No drift detected" to the summary and stop
+   - **Minor**: Agent found a better approach but skill isn't wrong → note in KNOWLEDGE.md
+   - **Significant**: Skill has outdated or incorrect guidance → propose fix
+
+5. **If significant drift found**, write a heal suggestion to \`.gsd/skill-review-queue.md\`:
+
+\`\`\`markdown
+### {skill-name} (flagged {date})
+- **Unit:** ${unitId}
+- **Issue:** {1-2 sentence description}
+- **Root cause:** {outdated API / incorrect pattern / missing context}
+- **Proposed fix:**
+  - File: SKILL.md
+  - Section: {section name}
+  - Current: {quote the incorrect text}
+  - Suggested: {the corrected text}
+- **Action:** [ ] Reviewed [ ] Updated [ ] Dismissed
+\`\`\`
+
+**Important:** Do NOT modify the skill directly. Write the suggestion to the review queue.
+The SkillsBench research shows that human-curated skills outperform auto-generated ones by +16.2pp.
+The human review step is what makes this valuable.`;
+}
+
+/**
+ * Compute stale skills that should be added to avoid_skills.
+ * Returns only skills not already in the avoid list.
+ */
+export function computeStaleAvoidList(
+  basePath: string,
+  currentAvoidList: string[],
+  staleDays?: number,
+): string[] {
+  const ledger = loadLedgerFromDisk(basePath);
+  const units = (ledger?.units ?? []).filter(u => u.skills && u.skills.length > 0);
+  const stale = detectStaleSkills(units, staleDays ?? DEFAULT_STALE_DAYS);
+  const avoidSet = new Set(currentAvoidList);
+
+  return stale.filter(s => !avoidSet.has(s));
+}
+
+// ─── Internals ────────────────────────────────────────────────────────────────
+
+function aggregateBySkill(units: UnitMetrics[]): Map<string, SkillHealthEntry> {
+  const map = new Map<string, { uses: UnitMetrics[] }>();
+
+  for (const u of units) {
+    if (!u.skills) continue;
+    for (const skill of u.skills) {
+      let entry = map.get(skill);
+      if (!entry) {
+        entry = { uses: [] };
+        map.set(skill, entry);
+      }
+      entry.uses.push(u);
+    }
+  }
+
+  const result = new Map<string, SkillHealthEntry>();
+  const now = Date.now();
+
+  for (const [name, { uses }] of map) {
+    const totalTokens = uses.reduce((s, u) => s + u.tokens.total, 0);
+    const totalCost = uses.reduce((s, u) => s + u.cost, 0);
+    const avgTokens = Math.round(totalTokens / uses.length);
+    const avgCost = totalCost / uses.length;
+
+    // Success rate: units that didn't have excessive retries (proxy: low tool call count relative to messages)
+    // Without direct retry tracking, use a heuristic: success if toolCalls < assistantMessages * 20
+    const successCount = uses.filter(u => u.toolCalls < u.assistantMessages * 20).length;
+    const successRate = uses.length > 0 ? successCount / uses.length : 1;
+
+    // Token trend
+    const tokenTrend = computeTokenTrend(uses);
+
+    // Last used
+    const lastUsed = Math.max(...uses.map(u => u.finishedAt));
+    const staleDays = Math.floor((now - lastUsed) / (24 * 60 * 60 * 1000));
+
+    // Flag conditions
+    let flagged = false;
+    let flagReason: string | undefined;
+
+    if (uses.length >= MIN_USES_FOR_TREND) {
+      if (successRate < SUCCESS_RATE_THRESHOLD) {
+        flagged = true;
+        flagReason = `Success rate ${Math.round(successRate * 100)}% (below ${Math.round(SUCCESS_RATE_THRESHOLD * 100)}% threshold)`;
+      } else if (tokenTrend === "rising") {
+        flagged = true;
+        flagReason = `Token usage trending upward (${Math.round(TOKEN_RISE_THRESHOLD * 100)}%+ increase)`;
+      }
+    }
+
+    result.set(name, {
+      name,
+      totalUses: uses.length,
+      successRate,
+      avgTokens,
+      tokenTrend,
+      lastUsed,
+      staleDays,
+      avgCost,
+      flagged,
+      flagReason,
+    });
+  }
+
+  return result;
+}
+
+function computeTokenTrend(uses: UnitMetrics[]): "stable" | "rising" | "declining" {
+  if (uses.length < MIN_USES_FOR_TREND * 2) return "stable";
+
+  // Sort by start time
+  const sorted = [...uses].sort((a, b) => a.startedAt - b.startedAt);
+  const window = Math.min(TREND_WINDOW, Math.floor(sorted.length / 2));
+
+  const recent = sorted.slice(-window);
+  const previous = sorted.slice(-window * 2, -window);
+
+  const recentAvg = recent.reduce((s, u) => s + u.tokens.total, 0) / recent.length;
+  const previousAvg = previous.reduce((s, u) => s + u.tokens.total, 0) / previous.length;
+
+  if (previousAvg === 0) return "stable";
+
+  const change = (recentAvg - previousAvg) / previousAvg;
+
+  if (change > TOKEN_RISE_THRESHOLD) return "rising";
+  if (change < -TOKEN_RISE_THRESHOLD) return "declining";
+  return "stable";
+}
+
+function generateSuggestions(skills: SkillHealthEntry[], staleSkills: string[]): SkillHealSuggestion[] {
+  const suggestions: SkillHealSuggestion[] = [];
+
+  for (const skill of skills) {
+    if (skill.totalUses >= MIN_USES_FOR_TREND && skill.successRate < SUCCESS_RATE_THRESHOLD) {
+      suggestions.push({
+        skillName: skill.name,
+        trigger: "declining_success",
+        message: `Success rate dropped to ${Math.round(skill.successRate * 100)}% over ${skill.totalUses} uses. Review SKILL.md for outdated patterns.`,
+        severity: skill.successRate < 0.5 ? "critical" : "warning",
+      });
+    }
+
+    if (skill.tokenTrend === "rising" && skill.totalUses >= MIN_USES_FOR_TREND * 2) {
+      suggestions.push({
+        skillName: skill.name,
+        trigger: "rising_tokens",
+        message: `Token usage trending upward. Skill may be causing inefficient execution patterns.`,
+        severity: "info",
+      });
+    }
+  }
+
+  for (const name of staleSkills) {
+    suggestions.push({
+      skillName: name,
+      trigger: "stale",
+      message: `Not used in ${DEFAULT_STALE_DAYS}+ days. Consider archiving or updating.`,
+      severity: "info",
+    });
+  }
+
+  return suggestions;
+}

package/src/resources/extensions/gsd/skill-telemetry.ts

@@ -0,0 +1,127 @@
+/**
+ * GSD Skill Telemetry — Track which skills are loaded per unit (#599)
+ *
+ * Captures skill names at dispatch time for inclusion in UnitMetrics.
+ * Distinguishes between "available" skills (in system prompt) and
+ * "actively loaded" skills (read via tool calls during execution).
+ *
+ * Data flow:
+ *   1. At dispatch, captureAvailableSkills() records skills from the system prompt
+ *   2. During execution, recordSkillRead() tracks explicit SKILL.md reads
+ *   3. At unit completion, getAndClearSkills() returns the loaded list for metrics
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { getAgentDir } from "@gsd/pi-coding-agent";
+
+// ─── In-memory state ──────────────────────────────────────────────────────────
+
+/** Skills available in the system prompt for the current unit */
+let availableSkills: string[] = [];
+
+/** Skills explicitly read (SKILL.md loaded) during the current unit */
+const activelyLoadedSkills = new Set<string>();
+
+// ─── Public API ───────────────────────────────────────────────────────────────
+
+/**
+ * Capture the list of available skill names at dispatch time.
+ * Called before each unit starts.
+ */
+export function captureAvailableSkills(): void {
+  const skillsDir = join(getAgentDir(), "skills");
+  availableSkills = listSkillNames(skillsDir);
+  activelyLoadedSkills.clear();
+}
+
+/**
+ * Record that a skill was actively loaded (its SKILL.md was read).
+ * Call this when the agent reads a SKILL.md file.
+ */
+export function recordSkillRead(skillName: string): void {
+  activelyLoadedSkills.add(skillName);
+}
+
+/**
+ * Get the skill names for the current unit and clear state.
+ * Returns actively loaded skills if any, otherwise available skills.
+ * This gives the most useful signal: if the agent read specific skills,
+ * report those; otherwise report what was available.
+ */
+export function getAndClearSkills(): string[] {
+  const result = activelyLoadedSkills.size > 0
+    ? Array.from(activelyLoadedSkills)
+    : [...availableSkills];
+  availableSkills = [];
+  activelyLoadedSkills.clear();
+  return result;
+}
+
+/**
+ * Reset all telemetry state. Called when auto-mode stops.
+ */
+export function resetSkillTelemetry(): void {
+  availableSkills = [];
+  activelyLoadedSkills.clear();
+}
+
+/**
+ * Get last-used timestamps for all skills from metrics data.
+ * Returns a Map from skill name to most recent ms timestamp.
+ */
+export function getSkillLastUsed(units: Array<{ finishedAt: number; skills?: string[] }>): Map<string, number> {
+  const lastUsed = new Map<string, number>();
+  for (const u of units) {
+    if (!u.skills) continue;
+    for (const skill of u.skills) {
+      const existing = lastUsed.get(skill) ?? 0;
+      if (u.finishedAt > existing) {
+        lastUsed.set(skill, u.finishedAt);
+      }
+    }
+  }
+  return lastUsed;
+}
+
+/**
+ * Detect stale skills — those not used within the given threshold (in days).
+ * Returns skill names that should be deprioritized.
+ */
+export function detectStaleSkills(
+  units: Array<{ finishedAt: number; skills?: string[] }>,
+  thresholdDays: number,
+): string[] {
+  if (thresholdDays <= 0) return [];
+
+  const lastUsed = getSkillLastUsed(units);
+  const cutoff = Date.now() - (thresholdDays * 24 * 60 * 60 * 1000);
+  const stale: string[] = [];
+
+  // Check all installed skills, not just those with usage data
+  const skillsDir = join(getAgentDir(), "skills");
+  const installed = listSkillNames(skillsDir);
+
+  for (const skill of installed) {
+    const lastTs = lastUsed.get(skill);
+    if (lastTs === undefined || lastTs < cutoff) {
+      stale.push(skill);
+    }
+  }
+
+  return stale;
+}
+
+// ─── Internals ────────────────────────────────────────────────────────────────
+
+function listSkillNames(skillsDir: string): string[] {
+  if (!existsSync(skillsDir)) return [];
+  try {
+    return readdirSync(skillsDir, { withFileTypes: true })
+      .filter(d => d.isDirectory() && !d.name.startsWith("."))
+      .filter(d => existsSync(join(skillsDir, d.name, "SKILL.md")))
+      .map(d => d.name);
+  } catch {
+    return [];
+  }
+}

package/src/resources/extensions/gsd/state.ts

@@ -32,8 +32,10 @@ import {
 
 import { milestoneIdSort, findMilestoneIds } from './guided-flow.js';
 import { nativeBatchParseGsdFiles, type BatchParsedFile } from './native-parser-bridge.js';
+import { isDbAvailable, _getAdapter } from './gsd-db.js';
 
 import { join, resolve } from 'path';
+import { debugCount, debugTime } from './debug-logger.js';
 
 // ─── Query Functions ───────────────────────────────────────────────────────
 
@@ -116,7 +118,10 @@ export async function deriveState(basePath: string): Promise<GSDState> {
     return _stateCache.result;
   }
 
+  const stopTimer = debugTime("derive-state-impl");
   const result = await _deriveStateImpl(basePath);
+  stopTimer({ phase: result.phase, milestone: result.activeMilestone?.id });
+  debugCount("deriveStateCalls");
   _stateCache = { basePath, result, timestamp: Date.now() };
   return result;
 }
@@ -131,6 +136,30 @@ async function _deriveStateImpl(basePath: string): Promise<GSDState> {
   const fileContentCache = new Map<string, string>();
   const gsdDir = gsdRoot(basePath);
 
+  // ── DB-first content loading ──
+  // When the DB is available, load artifact content from the artifacts table
+  // (indexed SELECT instead of O(N) file I/O). Falls back to native Rust batch
+  // parser, which in turn falls back to sequential JS reads via cachedLoadFile.
+  let dbContentLoaded = false;
+  if (isDbAvailable()) {
+    const adapter = _getAdapter();
+    if (adapter) {
+      try {
+        const rows = adapter.prepare('SELECT path, full_content FROM artifacts').all();
+        for (const row of rows) {
+          const relPath = (row as Record<string, unknown>)['path'] as string;
+          const content = (row as Record<string, unknown>)['full_content'] as string;
+          const absPath = resolve(gsdDir, relPath);
+          fileContentCache.set(absPath, content);
+        }
+        dbContentLoaded = rows.length > 0;
+      } catch {
+        // DB query failed — fall through to native batch parse
+      }
+    }
+  }
+
+  if (!dbContentLoaded) {
   const batchFiles = nativeBatchParseGsdFiles(gsdDir);
   if (batchFiles) {
     for (const f of batchFiles) {
@@ -138,6 +167,7 @@ async function _deriveStateImpl(basePath: string): Promise<GSDState> {
       fileContentCache.set(absPath, f.rawContent);
     }
   }
+  }
 
   /**
    * Load file content from batch cache first, falling back to disk read.

package/src/resources/extensions/gsd/templates/preferences.md

@@ -1,5 +1,6 @@
 ---
 version: 1
+mode:
 always_use_skills: []
 prefer_skills: []
 avoid_skills: []