guild-agents 1.3.0 → 1.4.0

package/README.md

@@ -93,11 +93,27 @@ guild list              # List agents and skills
 guild run <skill>       # Preview a skill's execution plan (dry-run)
 guild logs              # View execution traces
 guild logs clean        # Remove old traces (--days N, --all)
+guild stats             # Token usage and cost estimates
+guild eval              # Run structural skill evaluations
+guild eval --triggers   # Run trigger accuracy tests (keyword matcher)
+guild eval --semantic   # Run trigger tests with LLM semantic matcher
+guild eval --suggest    # Show description improvement suggestions
 guild workspace init <name> <members...>  # Create a workspace
 guild workspace add <path>                # Add a member repo
 guild workspace status                    # Show workspace state
 ```
 
+## Skill Evaluations
+
+Guild includes a built-in evaluation framework for validating skill quality:
+
+- **Structural evals** (`guild eval`) -- assert workflow structure: steps exist, roles are correct, gates are present
+- **Trigger tests** (`guild eval --triggers`) -- verify that user prompts route to the correct skill using keyword overlap scoring
+- **Semantic matcher** (`guild eval --semantic`) -- optional LLM-based scoring via Anthropic Haiku for higher-fidelity trigger testing (requires `ANTHROPIC_API_KEY`)
+- **Description suggestions** (`guild eval --suggest`) -- analyzes keyword gaps in skill descriptions based on failed triggers
+
+Every trigger run automatically records results to `benchmarks/benchmark.json` (rolling 30-entry history) and generates `benchmarks/benchmark.md` with per-skill accuracy, precision, recall, and delta vs previous run. Regressions (>5% accuracy drop with 2+ tests flipped) are flagged automatically.
+
 ## Under the Hood
 
 Guild coordinates 10 specialized agents through the pipeline. Each agent handles one phase.

package/bin/guild.js

@@ -8,6 +8,7 @@
  *   guild status         — view project status
  *   guild doctor         — verify setup and report issues
  *   guild list           — list installed agents and skills
+ *   guild stats          — view token usage and cost stats
  */
 
 import { program } from 'commander';
@@ -168,6 +169,51 @@ logsCmd
     }
   });
 
+// guild eval
+program
+  .command('eval')
+  .description('Run skill structural evaluations')
+  .argument('[skill]', 'Skill name to evaluate (or all if omitted)')
+  .option('--triggers', 'Run trigger tests instead of structural evals')
+  .option('--semantic', 'Use LLM-based semantic matcher for trigger tests')
+  .option('--suggest', 'Show description improvement suggestions')
+  .action(async (skill, options) => {
+    try {
+      if (options.triggers || options.semantic || options.suggest) {
+        const { runEvalTriggers } = await import('../src/commands/eval.js');
+        await runEvalTriggers(skill, {
+          semantic: options.semantic || false,
+          suggest: options.suggest || false,
+        });
+      } else {
+        const { runEval } = await import('../src/commands/eval.js');
+        await runEval(skill);
+      }
+    } catch (err) {
+      console.error(err.message);
+      process.exit(1);
+    }
+  });
+
+// guild stats
+program
+  .command('stats')
+  .description('View token usage stats and cost estimates')
+  .option('--period <period>', 'Filter by period: today, week, month, all', 'month')
+  .option('--compare', 'Compare cost across model profiles')
+  .option('--reset', 'Delete all usage history')
+  .option('-f, --force', 'Skip confirmation prompt (for --reset)')
+  .option('--export <format>', 'Export data (csv)')
+  .action(async (options) => {
+    try {
+      const { runStats } = await import('../src/commands/stats.js');
+      await runStats(options);
+    } catch (err) {
+      console.error(err.message);
+      process.exit(1);
+    }
+  });
+
 // guild workspace
 const workspaceCmd = program
   .command('workspace')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "guild-agents",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Specification-driven development CLI for Claude Code — think before you build",
   "type": "module",
   "files": [

package/src/commands/eval.js

@@ -0,0 +1,225 @@
+/**
+ * eval.js — Run skill structural evaluations.
+ *
+ * Usage:
+ *   guild eval                — Run all skills that have evals
+ *   guild eval build-feature  — Run evals for a specific skill
+ */
+
+import * as p from '@clack/prompts';
+import chalk from 'chalk';
+import { readdirSync, readFileSync, writeFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { loadEvals, runEvals } from '../utils/eval-runner.js';
+import { loadTriggers, runTriggerTests, computeAccuracy, loadAllSkillDescriptions } from '../utils/trigger-runner.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SKILLS_DIR = join(__dirname, '..', 'templates', 'skills');
+
+/**
+ * Runs skill evaluations.
+ * @param {string} [skillName] - Specific skill to evaluate, or all if omitted
+ */
+export async function runEval(skillName) {
+  const skills = skillName
+    ? [skillName]
+    : readdirSync(SKILLS_DIR, { withFileTypes: true })
+        .filter(d => d.isDirectory())
+        .map(d => d.name)
+        .filter(name => loadEvals(name) !== null);
+
+  p.intro(chalk.bold.cyan(`Guild Eval — ${skillName || 'all skills'}`));
+
+  let totalPassed = 0;
+  let totalFailed = 0;
+
+  for (const skill of skills) {
+    try {
+      const results = runEvals(skill);
+      for (const evalResult of results.results) {
+        if (evalResult.passed) {
+          p.log.success(`${chalk.gray(skill)} ${evalResult.description}`);
+          totalPassed++;
+        } else {
+          p.log.error(`${chalk.gray(skill)} ${evalResult.description}`);
+          for (const exp of evalResult.expectations.filter(e => !e.passed)) {
+            p.log.info(chalk.red(`  ↳ ${exp.text}: ${exp.evidence}`));
+          }
+          totalFailed++;
+        }
+      }
+    } catch (err) {
+      p.log.error(`${skill}: ${err.message}`);
+      totalFailed++;
+    }
+  }
+
+  const summary = `${totalPassed + totalFailed} evals: ${chalk.green(`${totalPassed} passed`)}${totalFailed > 0 ? `, ${chalk.red(`${totalFailed} failed`)}` : ''}`;
+  p.outro(summary);
+
+  if (totalFailed > 0) process.exit(1);
+}
+
+/**
+ * Runs trigger evaluations with optional semantic matcher, benchmarks, and suggestions.
+ * @param {string} [skillName] - Specific skill or all
+ * @param {object} [options] - CLI options
+ * @param {boolean} [options.semantic=false] - Use semantic matcher
+ * @param {boolean} [options.suggest=false] - Show description suggestions
+ */
+export async function runEvalTriggers(skillName, options = {}) {
+  const { semantic = false, suggest = false } = options;
+  const allSkills = loadAllSkillDescriptions();
+
+  // Warn if semantic mode but no API key
+  if (semantic && !process.env.ANTHROPIC_API_KEY) {
+    p.log.warn(chalk.yellow('ANTHROPIC_API_KEY not set — semantic matcher requires it'));
+    process.exit(1);
+  }
+
+  // Lazy-load semantic matcher only when needed
+  let scoreMatchSemantic;
+  if (semantic) {
+    const mod = await import('../utils/semantic-matcher.js');
+    scoreMatchSemantic = mod.scoreMatchSemantic;
+  }
+
+  const skills = skillName
+    ? [skillName]
+    : readdirSync(SKILLS_DIR, { withFileTypes: true })
+        .filter(d => d.isDirectory())
+        .map(d => d.name)
+        .filter(name => loadTriggers(name) !== null);
+
+  const matcherLabel = semantic ? 'semantic' : 'keyword';
+  p.intro(chalk.bold.cyan(`Guild Trigger Tests [${matcherLabel}] — ${skillName || 'all skills'}`));
+
+  let totalSkills = 0;
+  let totalTests = 0;
+  let totalCorrect = 0;
+  const allResults = [];
+  const benchmarkSkills = [];
+
+  for (const skill of skills) {
+    const triggers = loadTriggers(skill);
+    if (!triggers) {
+      p.log.warn(`${skill}: no triggers.json`);
+      continue;
+    }
+
+    const results = await runTriggerTests(triggers, allSkills, {
+      semantic,
+      scoreMatchSemantic,
+    });
+    const acc = computeAccuracy(results);
+    totalSkills++;
+    totalTests += acc.total;
+    totalCorrect += acc.tp + acc.tn;
+
+    const icon = acc.accuracy === 1 ? chalk.green('✓') : acc.accuracy >= 0.75 ? chalk.yellow('~') : chalk.red('✗');
+    p.log.info(`${icon} ${chalk.bold(skill)}  accuracy=${(acc.accuracy * 100).toFixed(0)}%  precision=${(acc.precision * 100).toFixed(0)}%  recall=${(acc.recall * 100).toFixed(0)}%`);
+
+    // Show failures
+    for (const r of results) {
+      if (r.expected !== r.actual) {
+        const label = r.expected ? chalk.red('MISS') : chalk.yellow('FALSE+');
+        let detail = `(score=${r.score.toFixed(2)}`;
+        if (r.rank !== null) detail += `, rank=#${r.rank}`;
+        if (r.reasoning) detail += `, reason: ${r.reasoning}`;
+        detail += ')';
+        p.log.info(chalk.gray(`    ${label} "${r.prompt}" ${detail}`));
+      }
+    }
+
+    allResults.push({ skill, results, triggers });
+    benchmarkSkills.push({
+      name: skill,
+      accuracy: acc.accuracy,
+      precision: acc.precision,
+      recall: acc.recall,
+      tp: acc.tp,
+      fp: acc.fp,
+      fn: acc.fn,
+      tn: acc.tn,
+    });
+  }
+
+  const overallAcc = totalTests > 0 ? ((totalCorrect / totalTests) * 100).toFixed(0) : 0;
+
+  // Record benchmark
+  const { recordBenchmark, generateReport, detectRegressions } = await import('../utils/benchmark.js');
+  const benchmarkDir = join(__dirname, '..', '..', 'benchmarks');
+  const benchmarkPath = join(benchmarkDir, 'benchmark.json');
+  const reportPath = join(benchmarkDir, 'benchmark.md');
+
+  const entry = {
+    timestamp: new Date().toISOString(),
+    matcher: matcherLabel,
+    model: semantic ? (process.env.GUILD_SEMANTIC_MODEL || 'claude-haiku-4-5-20251001') : null,
+    skills: benchmarkSkills,
+    aggregate: {
+      accuracy: totalTests > 0 ? totalCorrect / totalTests : 0,
+      precision: benchmarkSkills.reduce((s, sk) => s + sk.precision, 0) / (benchmarkSkills.length || 1),
+      recall: benchmarkSkills.reduce((s, sk) => s + sk.recall, 0) / (benchmarkSkills.length || 1),
+      total: totalTests,
+    },
+  };
+
+  recordBenchmark(entry, benchmarkPath);
+
+  // Load previous entry for comparison
+  const entries = JSON.parse(readFileSync(benchmarkPath, 'utf8'));
+  const previous = entries.length >= 2 ? entries[entries.length - 2] : null;
+
+  const report = generateReport(entry, previous);
+  writeFileSync(reportPath, report);
+  p.log.info(chalk.gray(`Benchmark recorded → benchmarks/benchmark.json`));
+
+  // Check for regressions
+  const regressions = detectRegressions(entry, previous);
+  if (regressions.length > 0) {
+    p.log.warn(chalk.yellow.bold('Regressions detected:'));
+    for (const reg of regressions) {
+      p.log.warn(chalk.yellow(`  ${reg.skill}: ${(reg.previousAccuracy * 100).toFixed(0)}% → ${(reg.currentAccuracy * 100).toFixed(0)}% (${reg.flippedTests} tests flipped)`));
+    }
+  }
+
+  // Description suggestions
+  if (suggest) {
+    const { analyzeGaps, generateSuggestions } = await import('../utils/description-analyzer.js');
+
+    const gapsList = [];
+    for (const { skill, results, triggers } of allResults) {
+      const skillDesc = allSkills.find(s => s.name === skill);
+      const gaps = analyzeGaps(results, skillDesc?.description || triggers.description);
+      if (gaps.missingKeywords.length > 0) {
+        gapsList.push({
+          skill,
+          currentDescription: skillDesc?.description || triggers.description,
+          ...gaps,
+        });
+      }
+    }
+
+    const suggestions = generateSuggestions(gapsList);
+    if (suggestions.length > 0) {
+      p.log.info('');
+      p.log.info(chalk.bold.cyan('Description Suggestions:'));
+      for (const sug of suggestions) {
+        const highWords = sug.suggestedKeywords.filter(k => k.confidence === 'high').map(k => k.word);
+        const medWords = sug.suggestedKeywords.filter(k => k.confidence === 'medium').map(k => k.word);
+        const parts = [];
+        if (highWords.length > 0) parts.push(`${highWords.join(', ')} (high)`);
+        if (medWords.length > 0) parts.push(`${medWords.join(', ')} (medium)`);
+        p.log.warn(`  ${chalk.bold(sug.skill)} — ${sug.suggestedKeywords.length} missing keywords`);
+        p.log.info(chalk.gray(`    Missing: ${parts.join(', ')}`));
+        p.log.info(chalk.gray(`    Current: "${sug.currentDescription}"`));
+      }
+    } else {
+      p.log.success('No description gaps found');
+    }
+  }
+
+  p.outro(`${totalSkills} skills, ${totalTests} tests, ${overallAcc}% overall accuracy`);
+}

package/src/commands/stats.js

@@ -0,0 +1,147 @@
+/**
+ * stats.js — Token usage stats command.
+ */
+
+import * as p from '@clack/prompts';
+import chalk from 'chalk';
+import { existsSync, unlinkSync, copyFileSync } from 'fs';
+import { join } from 'path';
+import { loadUsage, aggregate, estimateWithProfile } from '../utils/accounting.js';
+import { getModelShortName } from '../utils/pricing.js';
+
+const USAGE_PATH = join('.claude', 'guild', 'usage.json');
+
+function fmt(n) {
+  return n.toLocaleString('en-US');
+}
+
+function usd(n) {
+  return `$${n.toFixed(2)}`;
+}
+
+function pct(part, total) {
+  if (total === 0) return '0%';
+  return `${Math.round((part / total) * 100)}%`;
+}
+
+const PERIOD_LABELS = {
+  today: 'Today',
+  week: 'Last 7 days',
+  month: 'Last 30 days',
+  all: 'All time',
+};
+
+export function formatCsv(entries) {
+  const headers = 'timestamp,workflow,agent,tier,model,inputTokens,outputTokens,totalTokens,estimatedCostUSD';
+  const rows = entries.map(e =>
+    `${e.timestamp},${e.workflow},${e.agent},${e.tier},${e.model},${e.inputTokens},${e.outputTokens},${e.totalTokens},${e.estimatedCostUSD.toFixed(6)}`
+  );
+  return [headers, ...rows].join('\n') + '\n';
+}
+
+export async function runStats(options = {}) {
+  const root = process.cwd();
+
+  if (options.reset) {
+    return handleReset(root, options.force);
+  }
+
+  if (options.export === 'csv') {
+    const usage = loadUsage(root);
+    if (usage.entries.length === 0) {
+      console.log('No usage data to export.');
+      return;
+    }
+    process.stdout.write(formatCsv(usage.entries));
+    return;
+  }
+
+  const period = options.period || 'month';
+  const totals = aggregate(root, period);
+
+  p.intro(chalk.bold.cyan(`Guild Usage Stats — ${PERIOD_LABELS[period] || period}`));
+
+  if (totals.totalTokens === 0) {
+    p.log.info('No usage data yet. Token tracking will begin when workflows record usage.');
+    p.outro('');
+    return;
+  }
+
+  p.log.step('Summary');
+  p.log.info(`  Workflows executed:  ${chalk.bold(fmt(totals.workflowCount))}`);
+  p.log.info(`  Total tokens:        ${chalk.bold(fmt(totals.totalTokens))}`);
+  p.log.info(`  Estimated cost:      ${chalk.bold.green(usd(totals.totalCostUSD))}`);
+
+  if (Object.keys(totals.tokensByTier).length > 0) {
+    p.log.step('By tier');
+    for (const [tier, tokens] of Object.entries(totals.tokensByTier)) {
+      p.log.info(`  ${tier.padEnd(12)} ${fmt(tokens).padStart(10)} tok  (${pct(tokens, totals.totalTokens).padStart(4)})`);
+    }
+  }
+
+  if (Object.keys(totals.tokensByModel).length > 0) {
+    p.log.step('By model');
+    for (const [model, tokens] of Object.entries(totals.tokensByModel)) {
+      p.log.info(`  ${getModelShortName(model).padEnd(12)} ${fmt(tokens).padStart(10)} tok`);
+    }
+  }
+
+  if (Object.keys(totals.tokensByWorkflow).length > 0) {
+    p.log.step('Top workflows');
+    const sorted = Object.entries(totals.tokensByWorkflow).sort((a, b) => b[1] - a[1]);
+    for (const [wf, tokens] of sorted) {
+      p.log.info(`  ${wf.padEnd(20)} ${fmt(tokens).padStart(10)} tok`);
+    }
+  }
+
+  if (options.compare) {
+    const usage = loadUsage(root);
+    const filtered = usage.entries;
+    const maxCost = estimateWithProfile(filtered, 'max');
+    const proCost = estimateWithProfile(filtered, 'pro');
+    const allOpusCost = estimateWithProfile(filtered, 'all-opus');
+
+    p.log.step('Profile comparison');
+    p.log.info(`  ${'max'.padEnd(12)} ${usd(maxCost).padStart(10)}    —`);
+    p.log.info(`  ${'pro'.padEnd(12)} ${usd(proCost).padStart(10)}    ${diffLabel(proCost, maxCost)}`);
+    p.log.info(`  ${'all-opus'.padEnd(12)} ${usd(allOpusCost).padStart(10)}    ${diffLabel(allOpusCost, maxCost)}`);
+  }
+
+  p.outro('');
+}
+
+function diffLabel(cost, baseline) {
+  if (baseline === 0) return '';
+  const diff = ((cost - baseline) / baseline) * 100;
+  const sign = diff >= 0 ? '+' : '';
+  return chalk.gray(`${sign}${Math.round(diff)}%`);
+}
+
+async function handleReset(root, force) {
+  const filePath = join(root, USAGE_PATH);
+
+  p.intro(chalk.bold.cyan('Guild — Reset Usage Stats'));
+
+  if (!existsSync(filePath)) {
+    p.log.info('No usage data found. Nothing to reset.');
+    p.outro('');
+    return;
+  }
+
+  if (!force) {
+    const confirmed = await p.confirm({
+      message: 'This will delete all usage history. Continue?',
+      initialValue: false,
+    });
+
+    if (p.isCancel(confirmed) || !confirmed) {
+      p.cancel('Reset cancelled.');
+      return;
+    }
+  }
+
+  copyFileSync(filePath, filePath + '.bak');
+  unlinkSync(filePath);
+  p.log.success(`${chalk.green('✓')} Usage history deleted. Backup saved as usage.json.bak.`);
+  p.outro('');
+}

package/src/templates/skills/build-feature/evals/triggers.json

@@ -0,0 +1,16 @@
+{
+  "skill": "build-feature",
+  "matcherType": "keyword",
+  "description": "Full pipeline: evaluation -> spec -> implementation -> review -> QA",
+  "threshold": 0.3,
+  "tests": [
+    { "prompt": "build a new feature with full pipeline", "shouldTrigger": true },
+    { "prompt": "implement this feature end to end", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "run the full implementation pipeline", "shouldTrigger": true },
+    { "prompt": "I want to ship this end to end", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "review my code", "shouldTrigger": false },
+    { "prompt": "create a pull request", "shouldTrigger": false },
+    { "prompt": "save my session", "shouldTrigger": false },
+    { "prompt": "debug this bug", "shouldTrigger": false }
+  ]
+}

package/src/templates/skills/council/evals/triggers.json

@@ -0,0 +1,16 @@
+{
+  "skill": "council",
+  "matcherType": "keyword",
+  "description": "Convenes multiple agents to debate an important decision",
+  "threshold": 0.3,
+  "tests": [
+    { "prompt": "convene a council to debate this decision", "shouldTrigger": true },
+    { "prompt": "I need multiple agents to debate this", "shouldTrigger": true },
+    { "prompt": "let the council decide", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "I need help making a decision", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "build a new feature", "shouldTrigger": false },
+    { "prompt": "review my code", "shouldTrigger": false },
+    { "prompt": "save my session", "shouldTrigger": false },
+    { "prompt": "debug this bug", "shouldTrigger": false }
+  ]
+}

package/src/templates/skills/create-pr/evals/evals.json

@@ -0,0 +1,44 @@
+{
+  "skill": "create-pr",
+  "evals": [
+    {
+      "id": "cpr-has-core-steps",
+      "description": "PR creation has verify, gather, generate, create steps",
+      "expectations": [
+        { "text": "Has verify-branch step", "assertion": "step-exists:verify-branch" },
+        { "text": "Has gather-context step", "assertion": "step-exists:gather-context" },
+        { "text": "Has generate-description step", "assertion": "step-exists:generate-description" },
+        { "text": "Has create-pr step", "assertion": "step-exists:create-pr" }
+      ]
+    },
+    {
+      "id": "cpr-all-system-role",
+      "description": "All steps use system role (no agent delegation)",
+      "expectations": [
+        { "text": "verify-branch is system", "assertion": "step-role:verify-branch:system" },
+        { "text": "gather-context is system", "assertion": "step-role:gather-context:system" },
+        { "text": "generate-description is system", "assertion": "step-role:generate-description:system" },
+        { "text": "create-pr is system", "assertion": "step-role:create-pr:system" },
+        { "text": "post-creation is system", "assertion": "step-role:post-creation:system" }
+      ]
+    },
+    {
+      "id": "cpr-gates",
+      "description": "Gates at description generation and post-creation",
+      "expectations": [
+        { "text": "Generate-description has gate", "assertion": "gate-exists:generate-description" },
+        { "text": "Post-creation has gate", "assertion": "gate-exists:post-creation" }
+      ]
+    },
+    {
+      "id": "cpr-dependencies",
+      "description": "Steps have correct dependency chain",
+      "expectations": [
+        { "text": "gather-context requires branch-state", "assertion": "step-requires:gather-context:branch-state" },
+        { "text": "generate-description requires commit-list", "assertion": "step-requires:generate-description:commit-list" },
+        { "text": "create-pr requires pr-description", "assertion": "step-requires:create-pr:pr-description" },
+        { "text": "post-creation requires pr-url", "assertion": "step-requires:post-creation:pr-url" }
+      ]
+    }
+  ]
+}

package/src/templates/skills/create-pr/evals/triggers.json

@@ -0,0 +1,16 @@
+{
+  "skill": "create-pr",
+  "matcherType": "keyword",
+  "description": "Create a pull request from the current branch with structured summary",
+  "threshold": 0.3,
+  "tests": [
+    { "prompt": "create a pull request", "shouldTrigger": true },
+    { "prompt": "open a PR for this branch", "shouldTrigger": true },
+    { "prompt": "push and create PR", "shouldTrigger": true },
+    { "prompt": "I'm ready to submit this for review", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "review my code changes", "shouldTrigger": false },
+    { "prompt": "start a new feature", "shouldTrigger": false },
+    { "prompt": "deploy to production", "shouldTrigger": false },
+    { "prompt": "save my session", "shouldTrigger": false }
+  ]
+}

package/src/templates/skills/debug/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: debug
-description: "Discipline skill — systematic debugging process. Use when encountering any bug, test failure, or unexpected behavior, before proposing fixes."
+description: "Discipline skill — systematic debugging process. Use when encountering any bug, debug issue, test failure, broken function, or unexpected behavior, before proposing fixes."
 user-invocable: true
 ---
 

package/src/templates/skills/debug/evals/triggers.json

@@ -0,0 +1,16 @@
+{
+  "skill": "debug",
+  "matcherType": "keyword",
+  "description": "Discipline skill — systematic debugging process. Use when encountering any bug, debug issue, test failure, broken function, or unexpected behavior, before proposing fixes.",
+  "threshold": 0.3,
+  "tests": [
+    { "prompt": "I have a bug in the login flow", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "tests are failing unexpectedly", "shouldTrigger": true },
+    { "prompt": "unexpected behavior in the API", "shouldTrigger": true },
+    { "prompt": "help me debug this function", "shouldTrigger": true },
+    { "prompt": "create a new feature", "shouldTrigger": false },
+    { "prompt": "review my code", "shouldTrigger": false },
+    { "prompt": "save my session", "shouldTrigger": false },
+    { "prompt": "what phase am I in", "shouldTrigger": false }
+  ]
+}

package/src/templates/skills/dev-flow/evals/evals.json

@@ -0,0 +1,36 @@
+{
+  "skill": "dev-flow",
+  "evals": [
+    {
+      "id": "df-has-steps",
+      "description": "Dev flow has read-state and present-flow steps",
+      "expectations": [
+        { "text": "Has read-state step", "assertion": "step-exists:read-state" },
+        { "text": "Has present-flow step", "assertion": "step-exists:present-flow" }
+      ]
+    },
+    {
+      "id": "df-all-system",
+      "description": "All steps are system role",
+      "expectations": [
+        { "text": "read-state is system", "assertion": "step-role:read-state:system" },
+        { "text": "present-flow is system", "assertion": "step-role:present-flow:system" }
+      ]
+    },
+    {
+      "id": "df-presentation-gate",
+      "description": "Present-flow step has a gate for user confirmation",
+      "expectations": [
+        { "text": "present-flow has gate", "assertion": "gate-exists:present-flow" }
+      ]
+    },
+    {
+      "id": "df-dependencies",
+      "description": "Present-flow requires session state",
+      "expectations": [
+        { "text": "present-flow requires session-state", "assertion": "step-requires:present-flow:session-state" },
+        { "text": "present-flow requires current-phase", "assertion": "step-requires:present-flow:current-phase" }
+      ]
+    }
+  ]
+}

package/src/templates/skills/dev-flow/evals/triggers.json

@@ -0,0 +1,16 @@
+{
+  "skill": "dev-flow",
+  "matcherType": "keyword",
+  "description": "Shows current pipeline phase and what comes next",
+  "threshold": 0.3,
+  "tests": [
+    { "prompt": "what phase am I in", "shouldTrigger": true },
+    { "prompt": "show the current pipeline phase", "shouldTrigger": true },
+    { "prompt": "what comes next in the flow", "shouldTrigger": true },
+    { "prompt": "where did I leave off", "shouldTrigger": true, "keywordExpected": false },
+    { "prompt": "create a pull request", "shouldTrigger": false },
+    { "prompt": "review my code", "shouldTrigger": false },
+    { "prompt": "fix this bug", "shouldTrigger": false },
+    { "prompt": "run the tests", "shouldTrigger": false }
+  ]
+}