guild-agents 1.3.0 → 1.4.0

package/src/utils/accounting.js

@@ -0,0 +1,139 @@
+/**
+ * accounting.js — Token usage recording, persistence, and aggregation.
+ *
+ * Persists usage data to .claude/guild/usage.json.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { estimateCost } from './pricing.js';
+
+const USAGE_PATH = join('.claude', 'guild', 'usage.json');
+
+export function emptyUsage() {
+  return {
+    version: 1,
+    lastUpdated: new Date().toISOString(),
+    entries: [],
+    totals: {
+      totalTokens: 0,
+      totalInputTokens: 0,
+      totalOutputTokens: 0,
+      totalCostUSD: 0,
+      tokensByModel: {},
+      tokensByTier: {},
+      tokensByWorkflow: {},
+      workflowCount: 0,
+    },
+  };
+}
+
+export function createEntry({ workflow, agent, tier, model, inputTokens, outputTokens }) {
+  const totalTokens = inputTokens + outputTokens;
+  return {
+    timestamp: new Date().toISOString(),
+    workflow,
+    agent,
+    tier,
+    model,
+    inputTokens,
+    outputTokens,
+    totalTokens,
+    estimatedCostUSD: estimateCost(model, inputTokens, outputTokens),
+  };
+}
+
+export function loadUsage(root) {
+  const filePath = join(root, USAGE_PATH);
+  if (!existsSync(filePath)) return emptyUsage();
+  try {
+    return JSON.parse(readFileSync(filePath, 'utf8'));
+  } catch {
+    return emptyUsage();
+  }
+}
+
+export function saveUsage(root, usage) {
+  const filePath = join(root, USAGE_PATH);
+  mkdirSync(dirname(filePath), { recursive: true });
+  usage.lastUpdated = new Date().toISOString();
+  writeFileSync(filePath, JSON.stringify(usage, null, 2) + '\n');
+}
+
+function updateTotals(totals, entry) {
+  totals.totalTokens += entry.totalTokens;
+  totals.totalInputTokens += entry.inputTokens;
+  totals.totalOutputTokens += entry.outputTokens;
+  totals.totalCostUSD += entry.estimatedCostUSD;
+  totals.tokensByModel[entry.model] = (totals.tokensByModel[entry.model] || 0) + entry.totalTokens;
+  totals.tokensByTier[entry.tier] = (totals.tokensByTier[entry.tier] || 0) + entry.totalTokens;
+  totals.tokensByWorkflow[entry.workflow] = (totals.tokensByWorkflow[entry.workflow] || 0) + entry.totalTokens;
+  totals.workflowCount += 1;
+}
+
+export function recordStep(root, params) {
+  const usage = loadUsage(root);
+  const entry = createEntry(params);
+  usage.entries.push(entry);
+  updateTotals(usage.totals, entry);
+  saveUsage(root, usage);
+}
+
+const PROFILES = {
+  max: { reasoning: 'claude-opus-4-6', execution: 'claude-sonnet-4-5', routine: 'claude-haiku-4-5' },
+  pro: { reasoning: 'claude-sonnet-4-5', execution: 'claude-sonnet-4-5', routine: 'claude-haiku-4-5' },
+  'all-opus': { reasoning: 'claude-opus-4-6', execution: 'claude-opus-4-6', routine: 'claude-opus-4-6' },
+};
+
+export function aggregate(root, period) {
+  const usage = loadUsage(root);
+  const now = new Date();
+  let cutoff;
+
+  switch (period) {
+    case 'today':
+      cutoff = new Date(now.getFullYear(), now.getMonth(), now.getDate());
+      break;
+    case 'week':
+      cutoff = new Date(now);
+      cutoff.setDate(cutoff.getDate() - 7);
+      break;
+    case 'month':
+      cutoff = new Date(now);
+      cutoff.setDate(cutoff.getDate() - 30);
+      break;
+    default:
+      cutoff = new Date(0);
+  }
+
+  const filtered = usage.entries.filter(e => new Date(e.timestamp) >= cutoff);
+
+  const totals = {
+    totalTokens: 0,
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    totalCostUSD: 0,
+    tokensByModel: {},
+    tokensByTier: {},
+    tokensByWorkflow: {},
+    workflowCount: 0,
+  };
+
+  for (const entry of filtered) {
+    updateTotals(totals, entry);
+  }
+
+  return totals;
+}
+
+export function estimateWithProfile(entries, profileName) {
+  const profile = PROFILES[profileName];
+  if (!profile) return 0;
+
+  let cost = 0;
+  for (const entry of entries) {
+    const model = profile[entry.tier] || entry.model;
+    cost += estimateCost(model, entry.inputTokens, entry.outputTokens);
+  }
+  return cost;
+}

package/src/utils/benchmark.js

@@ -0,0 +1,128 @@
+/**
+ * benchmark.js — Records, reports, and detects regressions in eval benchmarks.
+ *
+ * Persists results to benchmarks/benchmark.json with 30-entry rotation.
+ * Generates benchmarks/benchmark.md as a human-readable report.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { dirname } from 'path';
+
+const MAX_ENTRIES = 30;
+
+/**
+ * Appends a benchmark entry to the JSON file, rotating old entries.
+ * @param {object} entry - Benchmark entry with timestamp, matcher, skills, aggregate
+ * @param {string} filePath - Path to benchmark.json
+ */
+export function recordBenchmark(entry, filePath) {
+  const dir = dirname(filePath);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  let entries = [];
+  if (existsSync(filePath)) {
+    entries = JSON.parse(readFileSync(filePath, 'utf8'));
+  }
+
+  entries.push(entry);
+
+  if (entries.length > MAX_ENTRIES) {
+    entries = entries.slice(entries.length - MAX_ENTRIES);
+  }
+
+  writeFileSync(filePath, JSON.stringify(entries, null, 2));
+}
+
+/**
+ * Generates a markdown report from a benchmark entry.
+ * @param {object} current - Current benchmark entry
+ * @param {object|null} previous - Previous entry for delta comparison
+ * @returns {string} Markdown report
+ */
+export function generateReport(current, previous) {
+  const lines = [];
+  const date = current.timestamp;
+  const matcher = current.matcher;
+  const model = current.model ? ` (${current.model})` : '';
+
+  lines.push(`# Eval Benchmark — ${date}`);
+  lines.push(`Matcher: ${matcher}${model} | Skills: ${current.skills.length} | Total tests: ${current.aggregate.total}`);
+  lines.push('');
+  lines.push('| Skill | Accuracy | Precision | Recall | Delta |');
+  lines.push('|-------|----------|-----------|--------|-------|');
+
+  for (const skill of current.skills) {
+    let delta = '—';
+    if (previous) {
+      const prev = previous.skills.find(s => s.name === skill.name);
+      if (prev) {
+        const diff = (skill.accuracy - prev.accuracy) * 100;
+        if (Math.abs(diff) >= 0.1) {
+          const sign = diff > 0 ? '+' : '';
+          const warn = diff < -5 ? ' !!' : '';
+          delta = `${sign}${diff.toFixed(1)}%${warn}`;
+        }
+      }
+    }
+
+    lines.push(`| ${skill.name} | ${(skill.accuracy * 100).toFixed(1)}% | ${(skill.precision * 100).toFixed(1)}% | ${(skill.recall * 100).toFixed(1)}% | ${delta} |`);
+  }
+
+  lines.push('');
+  lines.push('## Aggregate');
+
+  let aggDelta = '';
+  if (previous) {
+    const diff = (current.aggregate.accuracy - previous.aggregate.accuracy) * 100;
+    if (Math.abs(diff) >= 0.1) {
+      const sign = diff > 0 ? '+' : '';
+      aggDelta = ` (Delta ${sign}${diff.toFixed(1)}%)`;
+    }
+  }
+
+  lines.push(`Accuracy: ${(current.aggregate.accuracy * 100).toFixed(1)}%${aggDelta}`);
+  lines.push(`Precision: ${(current.aggregate.precision * 100).toFixed(1)}%`);
+  lines.push(`Recall: ${(current.aggregate.recall * 100).toFixed(1)}%`);
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+/**
+ * Detects regressions between two benchmark entries.
+ * A regression is: accuracy dropped >5% AND at least 2 tests flipped.
+ * @param {object} current
+ * @param {object|null} previous
+ * @returns {Array<{ skill: string, currentAccuracy: number, previousAccuracy: number, delta: number, flippedTests: number }>}
+ */
+export function detectRegressions(current, previous) {
+  if (!previous) return [];
+
+  const regressions = [];
+
+  for (const skill of current.skills) {
+    const prev = previous.skills.find(s => s.name === skill.name);
+    if (!prev) continue;
+
+    const delta = skill.accuracy - prev.accuracy;
+    if (delta > -0.05) continue;
+
+    const currentCorrect = skill.tp + skill.tn;
+    const prevCorrect = prev.tp + prev.tn;
+    const flippedTests = Math.abs(currentCorrect - prevCorrect);
+
+    if (flippedTests < 2) continue;
+
+    regressions.push({
+      skill: skill.name,
+      currentAccuracy: skill.accuracy,
+      previousAccuracy: prev.accuracy,
+      delta,
+      flippedTests,
+    });
+  }
+
+  return regressions;
+}

package/src/utils/description-analyzer.js

@@ -0,0 +1,92 @@
+/**
+ * description-analyzer.js — Analyzes keyword gaps in skill descriptions.
+ *
+ * Uses token analysis to identify which keywords are missing from
+ * skill descriptions based on failed trigger tests. No LLM required.
+ */
+
+import { tokenize } from './trigger-matcher.js';
+
+const STOP_WORDS = new Set([
+  'the', 'is', 'at', 'in', 'on', 'to', 'of', 'for', 'and', 'or', 'an',
+  'it', 'by', 'as', 'be', 'do', 'if', 'no', 'so', 'up', 'we', 'my',
+  'use', 'when', 'with', 'from', 'this', 'that', 'will', 'can', 'has',
+  'not', 'are', 'was', 'but', 'all', 'any', 'its', 'you', 'your',
+  'want', 'need', 'just', 'let', 'get', 'make', 'help', 'me',
+]);
+
+/**
+ * Checks if a token matches any description token (full or substring).
+ */
+function tokenMatchesDescription(token, descTokens) {
+  for (const dt of descTokens) {
+    if (dt === token || dt.includes(token) || token.includes(dt)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Analyzes gaps between failed trigger prompts and a skill description.
+ * @param {Array} triggerResults - Results from runTriggerTests
+ * @param {string} description - Skill description
+ * @returns {{ missingKeywords: string[], failedPrompts: string[] }}
+ */
+export function analyzeGaps(triggerResults, description) {
+  const failedPositives = triggerResults.filter(r => r.expected && !r.actual);
+
+  if (failedPositives.length === 0) {
+    return { missingKeywords: [], failedPrompts: [] };
+  }
+
+  const descTokens = tokenize(description).filter(w => !STOP_WORDS.has(w));
+  const missingKeywords = [];
+  const failedPrompts = [];
+
+  for (const result of failedPositives) {
+    failedPrompts.push(result.prompt);
+    const promptTokens = tokenize(result.prompt).filter(w => !STOP_WORDS.has(w));
+
+    for (const token of promptTokens) {
+      if (!tokenMatchesDescription(token, descTokens)) {
+        missingKeywords.push(token);
+      }
+    }
+  }
+
+  return { missingKeywords, failedPrompts };
+}
+
+/**
+ * Generates keyword suggestions from gap analysis results.
+ * @param {Array<{ skill: string, currentDescription: string, missingKeywords: string[], failedPrompts: string[] }>} gapsList
+ * @returns {Array<{ skill: string, currentDescription: string, suggestedKeywords: Array<{ word: string, confidence: string }> }>}
+ */
+export function generateSuggestions(gapsList) {
+  const suggestions = [];
+
+  for (const gaps of gapsList) {
+    if (gaps.missingKeywords.length === 0) continue;
+
+    const freq = new Map();
+    for (const word of gaps.missingKeywords) {
+      freq.set(word, (freq.get(word) || 0) + 1);
+    }
+
+    const suggestedKeywords = [...freq.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .map(([word, count]) => ({
+        word,
+        confidence: count >= 2 ? 'high' : 'medium',
+      }));
+
+    suggestions.push({
+      skill: gaps.skill,
+      currentDescription: gaps.currentDescription,
+      suggestedKeywords,
+    });
+  }
+
+  return suggestions;
+}

package/src/utils/pricing.js

@@ -0,0 +1,28 @@
+/**
+ * pricing.js — Model pricing table and cost calculation.
+ *
+ * Prices per million tokens (USD).
+ * Source: https://docs.anthropic.com/en/docs/about-claude/models
+ */
+
+export const DEFAULT_PRICING = {
+  'claude-opus-4-6': { input: 15.00, output: 75.00 },
+  'claude-sonnet-4-5': { input: 3.00, output: 15.00 },
+  'claude-haiku-4-5': { input: 0.80, output: 4.00 },
+};
+
+const SHORT_NAMES = {
+  'claude-opus-4-6': 'Opus',
+  'claude-sonnet-4-5': 'Sonnet',
+  'claude-haiku-4-5': 'Haiku',
+};
+
+export function estimateCost(model, inputTokens, outputTokens) {
+  const pricing = DEFAULT_PRICING[model];
+  if (!pricing) return 0;
+  return (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000;
+}
+
+export function getModelShortName(model) {
+  return SHORT_NAMES[model] || model;
+}

package/src/utils/semantic-matcher.js

@@ -0,0 +1,91 @@
+/**
+ * semantic-matcher.js — LLM-based trigger scoring via Anthropic Haiku.
+ *
+ * Calls the Anthropic Messages API to score how well a user prompt
+ * matches a skill. Optional complement to the keyword matcher.
+ */
+
+export const SEMANTIC_MODEL_DEFAULT = 'claude-haiku-4-5-20251001';
+
+const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
+
+const SYSTEM_PROMPT = `You are a skill-routing classifier. Given a user prompt and a skill name + description, score how likely the user wants to trigger this skill.
+
+Respond with ONLY a JSON object, no other text:
+{"score": <0-100>, "reasoning": "<one sentence>"}
+
+Score guide:
+- 90-100: Clear, direct match
+- 60-89: Likely match, related intent
+- 30-59: Possible but ambiguous
+- 0-29: Unrelated`;
+
+/**
+ * Scores a prompt against a skill using the Anthropic Messages API.
+ * @param {string} prompt - User prompt to classify
+ * @param {string} skillName - Skill identifier
+ * @param {string} skillDescription - Skill description text
+ * @returns {Promise<{ score: number, reasoning: string, error?: boolean }>}
+ */
+export async function scoreMatchSemantic(prompt, skillName, skillDescription) {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  const model = process.env.GUILD_SEMANTIC_MODEL || SEMANTIC_MODEL_DEFAULT;
+
+  try {
+    const response = await fetch(ANTHROPIC_API_URL, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 100,
+        system: SYSTEM_PROMPT,
+        messages: [
+          {
+            role: 'user',
+            content: `User prompt: "${prompt}"\nSkill: ${skillName}\nDescription: ${skillDescription}`,
+          },
+        ],
+      }),
+    });
+
+    if (!response.ok) {
+      return { score: 0, reasoning: `API error: ${response.status} ${response.statusText}`, error: true };
+    }
+
+    const data = await response.json();
+    const text = data.content[0].text;
+
+    return parseResponse(text);
+  } catch (err) {
+    return { score: 0, reasoning: err.message, error: true };
+  }
+}
+
+/**
+ * Parses the LLM response, extracting JSON with fallback.
+ * @param {string} text
+ * @returns {{ score: number, reasoning: string, error?: boolean }}
+ */
+function parseResponse(text) {
+  // Try direct parse first
+  try {
+    const parsed = JSON.parse(text);
+    return { score: parsed.score / 100, reasoning: parsed.reasoning };
+  } catch {
+    // Fallback: extract first JSON object from text
+    const match = text.match(/\{[^}]+\}/);
+    if (match) {
+      try {
+        const parsed = JSON.parse(match[0]);
+        return { score: parsed.score / 100, reasoning: parsed.reasoning };
+      } catch {
+        // Fall through
+      }
+    }
+    return { score: 0, reasoning: 'parse-error', error: true };
+  }
+}

package/src/utils/trigger-matcher.js

@@ -0,0 +1,64 @@
+/**
+ * trigger-matcher.js — Scores prompts against skill descriptions.
+ *
+ * Uses keyword overlap scoring to determine how well a user prompt
+ * matches a skill's description. No LLM calls — purely programmatic.
+ */
+
+/**
+ * Tokenizes text into lowercase words, stripping punctuation.
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function tokenize(text) {
+  return text
+    .toLowerCase()
+    .replace(/[—–\-/]/g, ' ')
+    .replace(/[^\w\s]/g, '')
+    .split(/\s+/)
+    .filter(w => w.length > 1);
+}
+
+const STOP_WORDS = new Set([
+  'the', 'is', 'at', 'in', 'on', 'to', 'of', 'for', 'and', 'or', 'an',
+  'it', 'by', 'as', 'be', 'do', 'if', 'no', 'so', 'up', 'we', 'my',
+  'use', 'when', 'with', 'from', 'this', 'that', 'will', 'can', 'has',
+  'not', 'are', 'was', 'but', 'all', 'any', 'its', 'you', 'your',
+  'skill', 'discipline',
+]);
+
+/**
+ * Scores how well a prompt matches a description.
+ * Returns 0-1.
+ */
+export function scoreMatch(prompt, description) {
+  const promptTokens = tokenize(prompt).filter(w => !STOP_WORDS.has(w));
+  if (promptTokens.length === 0) return 0;
+
+  const descTokens = new Set(tokenize(description).filter(w => !STOP_WORDS.has(w)));
+
+  let matches = 0;
+  for (const token of promptTokens) {
+    if (descTokens.has(token)) {
+      matches++;
+    } else {
+      for (const dt of descTokens) {
+        if (dt.includes(token) || token.includes(dt)) {
+          matches += 0.5;
+          break;
+        }
+      }
+    }
+  }
+
+  return matches / promptTokens.length;
+}
+
+/**
+ * Ranks all skills by match score descending.
+ */
+export function rankSkills(prompt, skills) {
+  return skills
+    .map(s => ({ ...s, score: scoreMatch(prompt, s.description) }))
+    .sort((a, b) => b.score - a.score);
+}

package/src/utils/trigger-runner.js

@@ -0,0 +1,132 @@
+/**
+ * trigger-runner.js — Loads and executes trigger tests for skills.
+ */
+
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { rankSkills } from './trigger-matcher.js';
+import { extractFrontmatterBlock, parseYamlFrontmatter } from './workflow-parser.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const TEMPLATES_DIR = join(__dirname, '..', 'templates', 'skills');
+
+/**
+ * Loads triggers.json for a skill template.
+ * @param {string} skillName
+ * @returns {object|null}
+ */
+export function loadTriggers(skillName) {
+  const triggersPath = join(TEMPLATES_DIR, skillName, 'evals', 'triggers.json');
+  if (!existsSync(triggersPath)) return null;
+  return JSON.parse(readFileSync(triggersPath, 'utf8'));
+}
+
+/**
+ * Loads all skill names and descriptions from templates.
+ * @returns {{ name: string, description: string }[]}
+ */
+export function loadAllSkillDescriptions() {
+  const skillDirs = readdirSync(TEMPLATES_DIR, { withFileTypes: true })
+    .filter(d => d.isDirectory())
+    .map(d => d.name);
+
+  const skills = [];
+  for (const name of skillDirs) {
+    const skillPath = join(TEMPLATES_DIR, name, 'SKILL.md');
+    if (!existsSync(skillPath)) continue;
+    const content = readFileSync(skillPath, 'utf8');
+    const block = extractFrontmatterBlock(content);
+    if (!block) continue;
+    const fm = parseYamlFrontmatter(block.yaml);
+    if (fm.description) {
+      skills.push({ name, description: fm.description });
+    }
+  }
+  return skills;
+}
+
+/**
+ * Runs trigger tests for a skill.
+ *
+ * When matcherType is "keyword" and a test has keywordExpected defined,
+ * that value overrides shouldTrigger for accuracy calculation. This lets
+ * tests document the ideal (semantic) expectation while being honest
+ * about what keyword matching can achieve.
+ *
+ * @param {object} triggers - Trigger test config from triggers.json
+ * @param {Array} allSkills - All skill descriptions
+ * @param {object} [options] - Options
+ * @param {boolean} [options.semantic=false] - Use semantic matcher
+ * @param {Function} [options.scoreMatchSemantic] - Semantic scoring function (injected for testability)
+ */
+export async function runTriggerTests(triggers, allSkills, options = {}) {
+  const { semantic = false, scoreMatchSemantic: semanticFn } = options;
+  const threshold = triggers.threshold || 0.3;
+  const isKeyword = !semantic && triggers.matcherType === 'keyword';
+  const results = [];
+
+  for (const test of triggers.tests) {
+    let actual, score, rank, reasoning;
+
+    if (semantic && semanticFn) {
+      const targetSkill = allSkills.find(s => s.name === triggers.skill);
+      const semanticResult = await semanticFn(test.prompt, triggers.skill, targetSkill?.description || triggers.description);
+      score = semanticResult.score;
+      actual = score >= threshold;
+      rank = null;
+      reasoning = semanticResult.reasoning;
+    } else {
+      const ranked = rankSkills(test.prompt, allSkills);
+      const targetRank = ranked.findIndex(s => s.name === triggers.skill);
+      score = targetRank >= 0 ? ranked[targetRank].score : 0;
+      actual = targetRank === 0 && score >= threshold;
+      rank = targetRank + 1;
+    }
+
+    const hasOverride = isKeyword && test.keywordExpected !== undefined;
+    const expected = hasOverride ? test.keywordExpected : test.shouldTrigger;
+
+    const result = {
+      prompt: test.prompt,
+      expected,
+      actual,
+      score,
+      rank,
+      matcherUsed: semantic ? 'semantic' : 'keyword',
+    };
+
+    if (reasoning) {
+      result.reasoning = reasoning;
+    }
+
+    if (hasOverride) {
+      result.semanticExpected = test.shouldTrigger;
+    }
+
+    results.push(result);
+  }
+
+  return results;
+}
+
+/**
+ * Computes precision, recall, and accuracy from trigger test results.
+ */
+export function computeAccuracy(results) {
+  if (results.length === 0) return { precision: 0, recall: 0, accuracy: 0, total: 0, tp: 0, fp: 0, fn: 0, tn: 0 };
+
+  let tp = 0, fp = 0, fn = 0, tn = 0;
+  for (const r of results) {
+    if (r.expected && r.actual) tp++;
+    else if (!r.expected && r.actual) fp++;
+    else if (r.expected && !r.actual) fn++;
+    else tn++;
+  }
+
+  const precision = (tp + fp) > 0 ? tp / (tp + fp) : 0;
+  const recall = (tp + fn) > 0 ? tp / (tp + fn) : 0;
+  const accuracy = (tp + tn) / results.length;
+
+  return { precision, recall, accuracy, total: results.length, tp, fp, fn, tn };
+}