guild-agents 1.3.0 → 1.5.0

package/src/utils/benchmark.js

@@ -0,0 +1,128 @@
+/**
+ * benchmark.js — Records, reports, and detects regressions in eval benchmarks.
+ *
+ * Persists results to benchmarks/benchmark.json with 30-entry rotation.
+ * Generates benchmarks/benchmark.md as a human-readable report.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { dirname } from 'path';
+
+const MAX_ENTRIES = 30;
+
+/**
+ * Appends a benchmark entry to the JSON file, rotating old entries.
+ * @param {object} entry - Benchmark entry with timestamp, matcher, skills, aggregate
+ * @param {string} filePath - Path to benchmark.json
+ */
+export function recordBenchmark(entry, filePath) {
+  const dir = dirname(filePath);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  let entries = [];
+  if (existsSync(filePath)) {
+    entries = JSON.parse(readFileSync(filePath, 'utf8'));
+  }
+
+  entries.push(entry);
+
+  if (entries.length > MAX_ENTRIES) {
+    entries = entries.slice(entries.length - MAX_ENTRIES);
+  }
+
+  writeFileSync(filePath, JSON.stringify(entries, null, 2));
+}
+
+/**
+ * Generates a markdown report from a benchmark entry.
+ * @param {object} current - Current benchmark entry
+ * @param {object|null} previous - Previous entry for delta comparison
+ * @returns {string} Markdown report
+ */
+export function generateReport(current, previous) {
+  const lines = [];
+  const date = current.timestamp;
+  const matcher = current.matcher;
+  const model = current.model ? ` (${current.model})` : '';
+
+  lines.push(`# Eval Benchmark — ${date}`);
+  lines.push(`Matcher: ${matcher}${model} | Skills: ${current.skills.length} | Total tests: ${current.aggregate.total}`);
+  lines.push('');
+  lines.push('| Skill | Accuracy | Precision | Recall | Delta |');
+  lines.push('|-------|----------|-----------|--------|-------|');
+
+  for (const skill of current.skills) {
+    let delta = '—';
+    if (previous) {
+      const prev = previous.skills.find(s => s.name === skill.name);
+      if (prev) {
+        const diff = (skill.accuracy - prev.accuracy) * 100;
+        if (Math.abs(diff) >= 0.1) {
+          const sign = diff > 0 ? '+' : '';
+          const warn = diff < -5 ? ' !!' : '';
+          delta = `${sign}${diff.toFixed(1)}%${warn}`;
+        }
+      }
+    }
+
+    lines.push(`| ${skill.name} | ${(skill.accuracy * 100).toFixed(1)}% | ${(skill.precision * 100).toFixed(1)}% | ${(skill.recall * 100).toFixed(1)}% | ${delta} |`);
+  }
+
+  lines.push('');
+  lines.push('## Aggregate');
+
+  let aggDelta = '';
+  if (previous) {
+    const diff = (current.aggregate.accuracy - previous.aggregate.accuracy) * 100;
+    if (Math.abs(diff) >= 0.1) {
+      const sign = diff > 0 ? '+' : '';
+      aggDelta = ` (Delta ${sign}${diff.toFixed(1)}%)`;
+    }
+  }
+
+  lines.push(`Accuracy: ${(current.aggregate.accuracy * 100).toFixed(1)}%${aggDelta}`);
+  lines.push(`Precision: ${(current.aggregate.precision * 100).toFixed(1)}%`);
+  lines.push(`Recall: ${(current.aggregate.recall * 100).toFixed(1)}%`);
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+/**
+ * Detects regressions between two benchmark entries.
+ * A regression is: accuracy dropped >5% AND at least 2 tests flipped.
+ * @param {object} current
+ * @param {object|null} previous
+ * @returns {Array<{ skill: string, currentAccuracy: number, previousAccuracy: number, delta: number, flippedTests: number }>}
+ */
+export function detectRegressions(current, previous) {
+  if (!previous) return [];
+
+  const regressions = [];
+
+  for (const skill of current.skills) {
+    const prev = previous.skills.find(s => s.name === skill.name);
+    if (!prev) continue;
+
+    const delta = skill.accuracy - prev.accuracy;
+    if (delta > -0.05) continue;
+
+    const currentCorrect = skill.tp + skill.tn;
+    const prevCorrect = prev.tp + prev.tn;
+    const flippedTests = Math.abs(currentCorrect - prevCorrect);
+
+    if (flippedTests < 2) continue;
+
+    regressions.push({
+      skill: skill.name,
+      currentAccuracy: skill.accuracy,
+      previousAccuracy: prev.accuracy,
+      delta,
+      flippedTests,
+    });
+  }
+
+  return regressions;
+}

package/src/utils/description-analyzer.js

@@ -0,0 +1,92 @@
+/**
+ * description-analyzer.js — Analyzes keyword gaps in skill descriptions.
+ *
+ * Uses token analysis to identify which keywords are missing from
+ * skill descriptions based on failed trigger tests. No LLM required.
+ */
+
+import { tokenize } from './trigger-matcher.js';
+
+const STOP_WORDS = new Set([
+  'the', 'is', 'at', 'in', 'on', 'to', 'of', 'for', 'and', 'or', 'an',
+  'it', 'by', 'as', 'be', 'do', 'if', 'no', 'so', 'up', 'we', 'my',
+  'use', 'when', 'with', 'from', 'this', 'that', 'will', 'can', 'has',
+  'not', 'are', 'was', 'but', 'all', 'any', 'its', 'you', 'your',
+  'want', 'need', 'just', 'let', 'get', 'make', 'help', 'me',
+]);
+
+/**
+ * Checks if a token matches any description token (full or substring).
+ */
+function tokenMatchesDescription(token, descTokens) {
+  for (const dt of descTokens) {
+    if (dt === token || dt.includes(token) || token.includes(dt)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Analyzes gaps between failed trigger prompts and a skill description.
+ * @param {Array} triggerResults - Results from runTriggerTests
+ * @param {string} description - Skill description
+ * @returns {{ missingKeywords: string[], failedPrompts: string[] }}
+ */
+export function analyzeGaps(triggerResults, description) {
+  const failedPositives = triggerResults.filter(r => r.expected && !r.actual);
+
+  if (failedPositives.length === 0) {
+    return { missingKeywords: [], failedPrompts: [] };
+  }
+
+  const descTokens = tokenize(description).filter(w => !STOP_WORDS.has(w));
+  const missingKeywords = [];
+  const failedPrompts = [];
+
+  for (const result of failedPositives) {
+    failedPrompts.push(result.prompt);
+    const promptTokens = tokenize(result.prompt).filter(w => !STOP_WORDS.has(w));
+
+    for (const token of promptTokens) {
+      if (!tokenMatchesDescription(token, descTokens)) {
+        missingKeywords.push(token);
+      }
+    }
+  }
+
+  return { missingKeywords, failedPrompts };
+}
+
+/**
+ * Generates keyword suggestions from gap analysis results.
+ * @param {Array<{ skill: string, currentDescription: string, missingKeywords: string[], failedPrompts: string[] }>} gapsList
+ * @returns {Array<{ skill: string, currentDescription: string, suggestedKeywords: Array<{ word: string, confidence: string }> }>}
+ */
+export function generateSuggestions(gapsList) {
+  const suggestions = [];
+
+  for (const gaps of gapsList) {
+    if (gaps.missingKeywords.length === 0) continue;
+
+    const freq = new Map();
+    for (const word of gaps.missingKeywords) {
+      freq.set(word, (freq.get(word) || 0) + 1);
+    }
+
+    const suggestedKeywords = [...freq.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .map(([word, count]) => ({
+        word,
+        confidence: count >= 2 ? 'high' : 'medium',
+      }));
+
+    suggestions.push({
+      skill: gaps.skill,
+      currentDescription: gaps.currentDescription,
+      suggestedKeywords,
+    });
+  }
+
+  return suggestions;
+}

package/src/utils/dispatch-protocol.js

@@ -34,14 +34,11 @@ export const DEFAULT_FAILURE_STRATEGY = 'abort';
  */
 export const DEFAULT_AGENT_TIERS = {
   'advisor': 'reasoning',
-  'product-owner': 'reasoning',
   'tech-lead': 'reasoning',
   'code-reviewer': 'reasoning',
   'developer': 'execution',
   'bugfix': 'execution',
-  'db-migration': 'execution',
   'qa': 'execution',
-  'platform-expert': 'execution',
   'learnings-extractor': 'routine',
 };
 

package/src/utils/executor.js

@@ -3,7 +3,7 @@
  *
  * Drives a plan to completion by iterating through steps, dispatching
  * agent steps to a provider function and system steps to local commands.
- * Sequential execution only (v1.1); parallel groups deferred to v1.2.
+ * Supports parallel execution (v1.2) and delegation to sub-skills.
  */
 
 import { execFile } from 'child_process';
@@ -11,8 +11,15 @@ import {
   advanceStep,
   getNextSteps,
   isPlanComplete,
+  MAX_DELEGATION_DEPTH,
+  createExecutionPlan,
 } from './orchestrator.js';
-import { buildStepContext, recordStepTrace } from './orchestrator-io.js';
+import {
+  buildStepContext,
+  recordStepTrace,
+  loadWorkflow,
+  resolveStepDispatch,
+} from './orchestrator-io.js';
 
 const SYSTEM_STEP_TIMEOUT = 120_000; // 2 minutes
 
@@ -70,7 +77,7 @@ async function executeSystemStep(step, options = {}) {
   }
 
   if (step.delegatesTo) {
-    return { status: 'passed', output: `Delegation to "${step.delegatesTo}" skipped (v1.1)` };
+    return { status: 'passed', output: `System step with delegation — handled by executeDelegation` };
   }
 
   return { status: 'passed', output: 'System step completed' };
@@ -92,12 +99,111 @@ function findStepInPlan(plan, stepId) {
   return null;
 }
 
+/**
+ * Dispatches a single step (agent or system) and returns its result.
+ *
+ * @param {object} step - Step definition
+ * @param {object} dispatch - Dispatch info for this step
+ * @param {object} context - Execution context
+ * @param {import('./orchestrator.js').ExecutionPlan} context.currentPlan - Current plan state
+ * @param {Function} context.provider - Agent step provider
+ * @param {string} context.projectRoot - Working directory
+ * @param {string} context.skillBody - Skill body text
+ * @param {object} context.executeOptions - Full options passed to execute()
+ * @returns {Promise<{ status: string, output: string, outcome?: object, error?: string }>}
+ */
+async function dispatchStep(step, dispatch, context) {
+  const { currentPlan, provider, projectRoot, skillBody, executeOptions } = context;
+
+  if (step.role === 'system' && step.delegatesTo) {
+    return executeDelegation(step, executeOptions);
+  }
+
+  if (step.role === 'system') {
+    return executeSystemStep(step, { projectRoot });
+  }
+
+  const stepContext = buildStepContext(step, currentPlan, { skillBody });
+  return provider(step, dispatch, stepContext);
+}
+
+/**
+ * Executes a delegation step by loading and running the sub-skill.
+ *
+ * @param {object} step - Delegation step (with delegatesTo field)
+ * @param {object} options - Execute options from parent
+ * @returns {Promise<{ status: string, output: string, error?: string }>}
+ */
+async function executeDelegation(step, options) {
+  const {
+    provider,
+    trace,
+    projectRoot,
+    profile = 'max',
+    onStepStart,
+    onStepEnd,
+    delegationDepth = 0,
+  } = options;
+
+  if (delegationDepth >= MAX_DELEGATION_DEPTH) {
+    return {
+      status: 'failed',
+      output: '',
+      error: `Delegation depth limit (${MAX_DELEGATION_DEPTH}) exceeded at step "${step.id}" delegating to "${step.delegatesTo}"`,
+    };
+  }
+
+  let subSkill;
+  try {
+    subSkill = loadWorkflow(step.delegatesTo);
+  } catch (err) {
+    return {
+      status: 'failed',
+      output: '',
+      error: `Failed to load delegated skill "${step.delegatesTo}": ${err.message}`,
+    };
+  }
+
+  const subPlan = createExecutionPlan(subSkill.workflow, {
+    skillName: subSkill.name || step.delegatesTo,
+  });
+
+  const subDispatchMap = {};
+  for (const group of subPlan.groups) {
+    for (const s of group.steps) {
+      subDispatchMap[s.id] = resolveStepDispatch(s, { profile, projectRoot });
+    }
+  }
+
+  const finalSubPlan = await execute(subPlan, subDispatchMap, {
+    provider,
+    trace,
+    projectRoot,
+    skillBody: subSkill.body || '',
+    onStepStart,
+    onStepEnd,
+    delegationDepth: delegationDepth + 1,
+    profile,
+  });
+
+  if (finalSubPlan.status === 'completed') {
+    return { status: 'passed', output: `Delegation to "${step.delegatesTo}" completed` };
+  }
+
+  return {
+    status: 'failed',
+    output: '',
+    error: `Delegated skill "${step.delegatesTo}" ended with status: ${finalSubPlan.status}`,
+  };
+}
+
 /**
  * Executes a workflow plan to completion.
  *
  * Drives the orchestrator state machine by repeatedly calling getNextSteps,
  * dispatching each step (agent via provider, system via local commands),
- * and advancing the plan with the result.
+ * and advancing the plan with the result. Parallel groups are dispatched
+ * concurrently via Promise.all.
  *
  * @param {import('./orchestrator.js').ExecutionPlan} plan - Initial execution plan
  * @param {Object.<string, import('./orchestrator-io.js').StepDispatchInfo>} dispatchInfoMap - Dispatch info per step
@@ -108,6 +214,8 @@ function findStepInPlan(plan, stepId) {
  * @param {string} [options.skillBody=''] - Skill body text for context building
  * @param {Function} [options.onStepStart] - Callback before each step: (step, dispatch) => void
  * @param {Function} [options.onStepEnd] - Callback after each step: (step, result) => void
+ * @param {number} [options.delegationDepth=0] - Current delegation nesting depth
+ * @param {string} [options.profile='max'] - Model profile for delegation dispatch
  * @returns {Promise<import('./orchestrator.js').ExecutionPlan>} Final plan state
  */
 export async function execute(plan, dispatchInfoMap, options = {}) {
@@ -127,7 +235,6 @@ export async function execute(plan, dispatchInfoMap, options = {}) {
   while (!isPlanComplete(currentPlan)) {
     const { steps, skipped } = getNextSteps(currentPlan);
 
-    // Advance skipped steps first
     for (const stepId of skipped) {
       currentPlan = advanceStep(currentPlan, stepId, { status: 'skipped' });
 
@@ -140,7 +247,6 @@ export async function execute(plan, dispatchInfoMap, options = {}) {
       }
     }
 
-    // If no executable steps remain, check completion again
     if (steps.length === 0) {
       if (isPlanComplete(currentPlan)) break;
       if (++emptyIterations > MAX_EMPTY_ITERATIONS) {
@@ -151,30 +257,34 @@ export async function execute(plan, dispatchInfoMap, options = {}) {
     }
     emptyIterations = 0;
 
-    // v1.1: sequential execution — one step at a time
-    const step = steps[0];
-    const dispatch = dispatchInfoMap[step.id] || {};
+    const dispatchContext = {
+      currentPlan,
+      provider,
+      projectRoot,
+      skillBody,
+      executeOptions: options,
+    };
 
-    onStepStart?.(step, dispatch);
+    const settled = await Promise.all(
+      steps.map(async (step) => {
+        const dispatch = dispatchInfoMap[step.id] || {};
+        onStepStart?.(step, dispatch);
+        const result = await dispatchStep(step, dispatch, dispatchContext);
+        return { step, dispatch, result };
+      })
+    );
 
-    let result;
-    if (step.role === 'system') {
-      result = await executeSystemStep(step, { projectRoot });
-    } else {
-      const context = buildStepContext(step, currentPlan, { skillBody });
-      result = await provider(step, dispatch, context);
-    }
+    for (const { step, dispatch, result } of settled) {
+      currentPlan = advanceStep(currentPlan, step.id, result);
 
-    currentPlan = advanceStep(currentPlan, step.id, result);
+      if (trace) {
+        recordStepTrace(trace, step, currentPlan.stepStates[step.id], dispatch);
+      }
 
-    if (trace) {
-      recordStepTrace(trace, step, currentPlan.stepStates[step.id], dispatch);
+      onStepEnd?.(step, result);
     }
-
-    onStepEnd?.(step, result);
   }
 
-  // Mark plan as completed if all steps reached terminal state and plan is still running
   if (currentPlan.status === 'running' && isPlanComplete(currentPlan)) {
     currentPlan = { ...currentPlan, status: 'completed' };
   }

package/src/utils/pricing.js

@@ -0,0 +1,28 @@
+/**
+ * pricing.js — Model pricing table and cost calculation.
+ *
+ * Prices per million tokens (USD).
+ * Source: https://docs.anthropic.com/en/docs/about-claude/models
+ */
+
+export const DEFAULT_PRICING = {
+  'claude-opus-4-6': { input: 15.00, output: 75.00 },
+  'claude-sonnet-4-5': { input: 3.00, output: 15.00 },
+  'claude-haiku-4-5': { input: 0.80, output: 4.00 },
+};
+
+const SHORT_NAMES = {
+  'claude-opus-4-6': 'Opus',
+  'claude-sonnet-4-5': 'Sonnet',
+  'claude-haiku-4-5': 'Haiku',
+};
+
+export function estimateCost(model, inputTokens, outputTokens) {
+  const pricing = DEFAULT_PRICING[model];
+  if (!pricing) return 0;
+  return (inputTokens * pricing.input + outputTokens * pricing.output) / 1_000_000;
+}
+
+export function getModelShortName(model) {
+  return SHORT_NAMES[model] || model;
+}

package/src/utils/semantic-matcher.js

@@ -0,0 +1,91 @@
+/**
+ * semantic-matcher.js — LLM-based trigger scoring via Anthropic Haiku.
+ *
+ * Calls the Anthropic Messages API to score how well a user prompt
+ * matches a skill. Optional complement to the keyword matcher.
+ */
+
+export const SEMANTIC_MODEL_DEFAULT = 'claude-haiku-4-5-20251001';
+
+const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
+
+const SYSTEM_PROMPT = `You are a skill-routing classifier. Given a user prompt and a skill name + description, score how likely the user wants to trigger this skill.
+
+Respond with ONLY a JSON object, no other text:
+{"score": <0-100>, "reasoning": "<one sentence>"}
+
+Score guide:
+- 90-100: Clear, direct match
+- 60-89: Likely match, related intent
+- 30-59: Possible but ambiguous
+- 0-29: Unrelated`;
+
+/**
+ * Scores a prompt against a skill using the Anthropic Messages API.
+ * @param {string} prompt - User prompt to classify
+ * @param {string} skillName - Skill identifier
+ * @param {string} skillDescription - Skill description text
+ * @returns {Promise<{ score: number, reasoning: string, error?: boolean }>}
+ */
+export async function scoreMatchSemantic(prompt, skillName, skillDescription) {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  const model = process.env.GUILD_SEMANTIC_MODEL || SEMANTIC_MODEL_DEFAULT;
+
+  try {
+    const response = await fetch(ANTHROPIC_API_URL, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': apiKey,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({
+        model,
+        max_tokens: 100,
+        system: SYSTEM_PROMPT,
+        messages: [
+          {
+            role: 'user',
+            content: `User prompt: "${prompt}"\nSkill: ${skillName}\nDescription: ${skillDescription}`,
+          },
+        ],
+      }),
+    });
+
+    if (!response.ok) {
+      return { score: 0, reasoning: `API error: ${response.status} ${response.statusText}`, error: true };
+    }
+
+    const data = await response.json();
+    const text = data.content[0].text;
+
+    return parseResponse(text);
+  } catch (err) {
+    return { score: 0, reasoning: err.message, error: true };
+  }
+}
+
+/**
+ * Parses the LLM response, extracting JSON with fallback.
+ * @param {string} text
+ * @returns {{ score: number, reasoning: string, error?: boolean }}
+ */
+function parseResponse(text) {
+  // Try direct parse first
+  try {
+    const parsed = JSON.parse(text);
+    return { score: parsed.score / 100, reasoning: parsed.reasoning };
+  } catch {
+    // Fallback: extract first JSON object from text
+    const match = text.match(/\{[^}]+\}/);
+    if (match) {
+      try {
+        const parsed = JSON.parse(match[0]);
+        return { score: parsed.score / 100, reasoning: parsed.reasoning };
+      } catch {
+        // Fall through
+      }
+    }
+    return { score: 0, reasoning: 'parse-error', error: true };
+  }
+}

package/src/utils/trigger-matcher.js

@@ -0,0 +1,64 @@
+/**
+ * trigger-matcher.js — Scores prompts against skill descriptions.
+ *
+ * Uses keyword overlap scoring to determine how well a user prompt
+ * matches a skill's description. No LLM calls — purely programmatic.
+ */
+
+/**
+ * Tokenizes text into lowercase words, stripping punctuation.
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function tokenize(text) {
+  return text
+    .toLowerCase()
+    .replace(/[—–\-/]/g, ' ')
+    .replace(/[^\w\s]/g, '')
+    .split(/\s+/)
+    .filter(w => w.length > 1);
+}
+
+const STOP_WORDS = new Set([
+  'the', 'is', 'at', 'in', 'on', 'to', 'of', 'for', 'and', 'or', 'an',
+  'it', 'by', 'as', 'be', 'do', 'if', 'no', 'so', 'up', 'we', 'my',
+  'use', 'when', 'with', 'from', 'this', 'that', 'will', 'can', 'has',
+  'not', 'are', 'was', 'but', 'all', 'any', 'its', 'you', 'your',
+  'skill', 'discipline',
+]);
+
+/**
+ * Scores how well a prompt matches a description.
+ * Returns 0-1.
+ */
+export function scoreMatch(prompt, description) {
+  const promptTokens = tokenize(prompt).filter(w => !STOP_WORDS.has(w));
+  if (promptTokens.length === 0) return 0;
+
+  const descTokens = new Set(tokenize(description).filter(w => !STOP_WORDS.has(w)));
+
+  let matches = 0;
+  for (const token of promptTokens) {
+    if (descTokens.has(token)) {
+      matches++;
+    } else {
+      for (const dt of descTokens) {
+        if (dt.includes(token) || token.includes(dt)) {
+          matches += 0.5;
+          break;
+        }
+      }
+    }
+  }
+
+  return matches / promptTokens.length;
+}
+
+/**
+ * Ranks all skills by match score descending.
+ */
+export function rankSkills(prompt, skills) {
+  return skills
+    .map(s => ({ ...s, score: scoreMatch(prompt, s.description) }))
+    .sort((a, b) => b.score - a.score);
+}