@yeaft/webchat-agent 0.1.410 → 0.1.412

package/unify/eval/cases/tool-use.js

@@ -0,0 +1,356 @@
+/**
+ * eval/cases/tool-use.js — Tool use eval cases
+ *
+ * Tests whether the model correctly decides when and how to call tools.
+ * These are the most important evals — they catch regressions in:
+ *   - Tool selection (right tool for the job)
+ *   - Parameter extraction (correct input from natural language)
+ *   - Tool avoidance (not calling tools when unnecessary)
+ *   - Multi-tool orchestration (using multiple tools in sequence)
+ */
+
+import { defineTool } from '../../tools/types.js';
+import {
+  noError,
+  toolWasCalled,
+  toolCalledWith,
+  toolNotCalled,
+  toolSucceeded,
+  turnCountInRange,
+  containsText,
+  custom,
+} from '../runner.js';
+
+// ─── Mock Tools for Evals ────────────────────────────────────
+
+const searchTool = defineTool({
+  name: 'search',
+  description: 'Search the web for information. Returns search results as text.',
+  parameters: {
+    type: 'object',
+    properties: {
+      query: { type: 'string', description: 'The search query' },
+    },
+    required: ['query'],
+  },
+  modes: ['chat', 'work'],
+  async execute(input) {
+    const q = (input.query || '').slice(0, 200);
+    return JSON.stringify({
+      results: [
+        { title: `Result for: ${q}`, snippet: `Information about ${q}` },
+      ],
+    });
+  },
+});
+
+const calculatorTool = defineTool({
+  name: 'calculator',
+  description: 'Perform mathematical calculations. Supports basic arithmetic and common functions.',
+  parameters: {
+    type: 'object',
+    properties: {
+      expression: { type: 'string', description: 'Math expression to evaluate (e.g. "2 + 3 * 4")' },
+    },
+    required: ['expression'],
+  },
+  modes: ['chat', 'work'],
+  async execute(input) {
+    try {
+      // Safe eval for basic math
+      const result = Function(`"use strict"; return (${input.expression})`)();
+      return String(result);
+    } catch {
+      return `Error: invalid expression "${input.expression}"`;
+    }
+  },
+});
+
+const readFileTool = defineTool({
+  name: 'read_file',
+  description: 'Read the contents of a file at the given path.',
+  parameters: {
+    type: 'object',
+    properties: {
+      path: { type: 'string', description: 'File path to read' },
+    },
+    required: ['path'],
+  },
+  modes: ['chat', 'work'],
+  async execute(input) {
+    // Mock file system
+    const files = {
+      'package.json': '{ "name": "my-app", "version": "1.0.0", "dependencies": { "express": "^4.18" } }',
+      'README.md': '# My App\n\nA sample application built with Express.',
+      'src/index.js': 'const express = require("express");\nconst app = express();\napp.listen(3000);',
+    };
+    return files[input.path] || `Error: file not found "${input.path}"`;
+  },
+});
+
+const writeFileTool = defineTool({
+  name: 'write_file',
+  description: 'Write content to a file at the given path.',
+  parameters: {
+    type: 'object',
+    properties: {
+      path: { type: 'string', description: 'File path to write' },
+      content: { type: 'string', description: 'Content to write' },
+    },
+    required: ['path', 'content'],
+  },
+  modes: ['work'],
+  async execute(input) {
+    return `Successfully wrote ${input.content.length} bytes to ${input.path}`;
+  },
+});
+
+const bashTool = defineTool({
+  name: 'bash',
+  description: 'Execute a bash command and return the output.',
+  parameters: {
+    type: 'object',
+    properties: {
+      command: { type: 'string', description: 'The bash command to execute' },
+    },
+    required: ['command'],
+  },
+  modes: ['work'],
+  async execute(input) {
+    // Mock bash responses
+    const responses = {
+      'git status': 'On branch main\nnothing to commit, working tree clean',
+      'ls': 'package.json\nREADME.md\nsrc/',
+      'npm test': 'Tests passed: 42/42',
+    };
+    // Match any command that starts with a known command
+    for (const [cmd, response] of Object.entries(responses)) {
+      if (input.command.startsWith(cmd)) return response;
+    }
+    return `$ ${input.command}\n(command executed successfully)`;
+  },
+});
+
+const allTools = [searchTool, calculatorTool, readFileTool, writeFileTool, bashTool];
+
+// ─── Eval Cases ──────────────────────────────────────────────
+
+export const toolUseCases = [
+
+  // ─── Basic Tool Selection ─────────────────────────────
+
+  {
+    id: 'tool-select-search',
+    suite: 'tools',
+    description: 'Model should use search tool for factual questions',
+    prompt: 'What is the current population of Tokyo?',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('search', { weight: 10 }),
+      toolCalledWith('search', (input) =>
+        input.query && input.query.toLowerCase().includes('tokyo'),
+        { id: 'search-mentions-tokyo', weight: 6 },
+      ),
+      toolNotCalled('calculator', { weight: 3 }),
+      toolNotCalled('read_file', { weight: 3 }),
+    ],
+  },
+
+  {
+    id: 'tool-select-calculator',
+    suite: 'tools',
+    description: 'Model should use calculator for math problems',
+    prompt: 'What is 1847 * 293 + 7621?',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('calculator', { weight: 10 }),
+      toolCalledWith('calculator', (input) =>
+        input.expression && /1847/.test(input.expression) && /293/.test(input.expression),
+        { id: 'calc-correct-expr', weight: 8 },
+      ),
+      toolNotCalled('search', { weight: 3 }),
+    ],
+  },
+
+  {
+    id: 'tool-select-read-file',
+    suite: 'tools',
+    description: 'Model should use read_file to examine project files',
+    prompt: 'What dependencies does the project have? Check package.json',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('read_file', { weight: 10 }),
+      toolCalledWith('read_file', (input) =>
+        input.path === 'package.json',
+        { id: 'reads-package-json', weight: 8 },
+      ),
+      containsText('express', { id: 'mentions-express', weight: 5 }),
+    ],
+  },
+
+  // ─── Tool Avoidance ───────────────────────────────────
+
+  {
+    id: 'tool-avoid-simple-chat',
+    suite: 'tools',
+    description: 'Model should NOT use tools for simple conversation',
+    prompt: 'Hello! How are you doing today?',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolNotCalled('search', { weight: 8 }),
+      toolNotCalled('calculator', { weight: 8 }),
+      toolNotCalled('read_file', { weight: 8 }),
+      turnCountInRange(1, 1, { weight: 5 }),
+    ],
+  },
+
+  {
+    id: 'tool-avoid-known-knowledge',
+    suite: 'tools',
+    description: 'Model should NOT search for common knowledge it already has',
+    prompt: 'What is the capital of France?',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolNotCalled('search', { weight: 8, id: 'no-search-for-common-knowledge' }),
+      containsText('Paris', { weight: 7 }),
+      turnCountInRange(1, 1, { weight: 3 }),
+    ],
+  },
+
+  {
+    id: 'tool-avoid-simple-math',
+    suite: 'tools',
+    description: 'Model should NOT use calculator for trivial math (2+2)',
+    prompt: 'What is 2 + 2?',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolNotCalled('calculator', { weight: 6, id: 'no-calc-for-trivial' }),
+      containsText('4', { weight: 5 }),
+    ],
+  },
+
+  // ─── Multi-Tool Orchestration ─────────────────────────
+
+  {
+    id: 'tool-multi-read-then-write',
+    suite: 'tools',
+    description: 'Model should read a file then modify it (sequential tools)',
+    prompt: 'Read src/index.js and add a health check endpoint at /health',
+    mode: 'work',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('read_file', { weight: 8, id: 'reads-first' }),
+      toolWasCalled('write_file', { weight: 8, id: 'writes-after' }),
+      toolCalledWith('read_file', (input) =>
+        input.path === 'src/index.js',
+        { id: 'reads-correct-file', weight: 6 },
+      ),
+      toolCalledWith('write_file', (input) =>
+        input.path === 'src/index.js' && input.content && input.content.includes('health'),
+        { id: 'writes-health-endpoint', weight: 8 },
+      ),
+      custom('read-before-write', 'Read happens before write', 5, (result) => {
+        const readIdx = result.toolCalls.findIndex(tc => tc.name === 'read_file');
+        const writeIdx = result.toolCalls.findIndex(tc => tc.name === 'write_file');
+        const ordered = readIdx >= 0 && writeIdx >= 0 && readIdx < writeIdx;
+        return { pass: ordered, score: ordered ? 1 : 0 };
+      }),
+    ],
+  },
+
+  {
+    id: 'tool-multi-bash-workflow',
+    suite: 'tools',
+    description: 'Model should run git status and npm test in work mode',
+    prompt: 'Check the git status and run the tests',
+    mode: 'work',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('bash', { weight: 8 }),
+      custom('git-status-called', 'Ran git status', 7, (result) => {
+        const gitCall = result.toolCalls.find(tc =>
+          tc.name === 'bash' && tc.input.command && tc.input.command.includes('git status'),
+        );
+        return { pass: !!gitCall, score: gitCall ? 1 : 0 };
+      }),
+      custom('npm-test-called', 'Ran npm test', 7, (result) => {
+        const testCall = result.toolCalls.find(tc =>
+          tc.name === 'bash' && tc.input.command && tc.input.command.includes('test'),
+        );
+        return { pass: !!testCall, score: testCall ? 1 : 0 };
+      }),
+    ],
+  },
+
+  // ─── Mode Awareness ───────────────────────────────────
+
+  {
+    id: 'tool-mode-chat-no-write',
+    suite: 'tools',
+    description: 'In chat mode, write_file should not be available (work-only tool)',
+    prompt: 'Write "hello" to a file called greeting.txt',
+    mode: 'chat',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolNotCalled('write_file', {
+        weight: 10,
+        id: 'no-write-in-chat',
+        description: 'write_file is work-only and should not be called in chat mode',
+      }),
+    ],
+  },
+
+  // ─── Error Handling ───────────────────────────────────
+
+  {
+    id: 'tool-error-recovery',
+    suite: 'tools',
+    description: 'Model should handle tool errors gracefully and explain to user',
+    prompt: 'Read the file at /nonexistent/path/file.txt',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('read_file', { weight: 8 }),
+      custom('acknowledges-error', 'Model acknowledges the file was not found', 7, (result) => {
+        const hasError = result.toolResults.some(tr =>
+          tr.name === 'read_file' && tr.output.includes('not found'),
+        );
+        const acknowledges = result.fullText.toLowerCase().includes('not found') ||
+                            result.fullText.toLowerCase().includes('doesn\'t exist') ||
+                            result.fullText.toLowerCase().includes('does not exist') ||
+                            result.fullText.toLowerCase().includes('error') ||
+                            result.fullText.toLowerCase().includes('unable');
+        return { pass: hasError && acknowledges, score: (hasError && acknowledges) ? 1 : 0 };
+      }),
+    ],
+  },
+
+  // ─── Parameter Extraction ─────────────────────────────
+
+  {
+    id: 'tool-param-extraction-complex',
+    suite: 'tools',
+    description: 'Model should extract correct parameters from complex natural language',
+    prompt: 'Search for "best practices for TypeScript error handling in 2026"',
+    registryTools: allTools,
+    criteria: [
+      noError,
+      toolWasCalled('search', { weight: 8 }),
+      toolCalledWith('search', (input) =>
+        input.query &&
+        input.query.toLowerCase().includes('typescript') &&
+        input.query.toLowerCase().includes('error'),
+        { id: 'extracts-key-terms', weight: 8 },
+      ),
+    ],
+  },
+];

package/unify/eval/run-eval.js

@@ -0,0 +1,250 @@
+#!/usr/bin/env node
+
+/**
+ * eval/run-eval.js — CLI runner for Yeaft evals
+ *
+ * Usage:
+ *   # Run all evals against default model (requires API key)
+ *   node agent/unify/eval/run-eval.js
+ *
+ *   # Run specific suite
+ *   node agent/unify/eval/run-eval.js --suite tools
+ *
+ *   # Compare multiple models
+ *   node agent/unify/eval/run-eval.js --models claude-sonnet-4-20250514,gpt-5
+ *
+ *   # Save baseline
+ *   node agent/unify/eval/run-eval.js --save-baseline initial
+ *
+ *   # Compare against baseline
+ *   node agent/unify/eval/run-eval.js --compare-baseline baselines/initial.json
+ *
+ *   # Dry run (MockAdapter, no API calls)
+ *   node agent/unify/eval/run-eval.js --dry-run
+ *
+ * Environment:
+ *   YEAFT_API_KEY          — Anthropic API key
+ *   YEAFT_OPENAI_API_KEY   — OpenAI API key
+ */
+
+import { parseArgs } from 'util';
+import { join } from 'path';
+import { homedir } from 'os';
+
+import { toolUseCases } from './cases/tool-use.js';
+import { memoryCases } from './cases/memory.js';
+import { skillsCases } from './cases/skills.js';
+import { e2eCases } from './cases/e2e.js';
+import {
+  runEvals,
+  printResults,
+  printComparison,
+  saveBaseline,
+  loadBaseline,
+  compareToBaseline,
+} from './runner.js';
+
+// ─── Parse CLI args ──────────────────────────────────────────
+
+const { values: args } = parseArgs({
+  options: {
+    suite: { type: 'string', short: 's', default: 'all' },
+    models: { type: 'string', short: 'm', default: '' },
+    'save-baseline': { type: 'string', default: '' },
+    'compare-baseline': { type: 'string', default: '' },
+    'dry-run': { type: 'boolean', default: false },
+    help: { type: 'boolean', short: 'h', default: false },
+  },
+});
+
+if (args.help) {
+  console.log(`
+Yeaft Eval Runner
+
+Usage:
+  node agent/unify/eval/run-eval.js [options]
+
+Options:
+  -s, --suite <name>         Run specific suite: tools, memory, skills, e2e, all (default: all)
+  -m, --models <list>        Comma-separated model IDs (default: auto-detect from API keys)
+  --save-baseline <name>     Save results as named baseline
+  --compare-baseline <path>  Compare results against a baseline file
+  --dry-run                  Run with MockAdapter (no API calls, for testing the harness)
+  -h, --help                 Show this help
+
+Environment:
+  YEAFT_API_KEY              Anthropic API key (enables Claude models)
+  YEAFT_OPENAI_API_KEY       OpenAI API key (enables GPT models)
+
+Examples:
+  # Quick dry run to verify harness works
+  node agent/unify/eval/run-eval.js --dry-run
+
+  # Run tool evals against Claude Sonnet
+  node agent/unify/eval/run-eval.js --suite tools --models claude-sonnet-4-20250514
+
+  # Full eval, save baseline
+  node agent/unify/eval/run-eval.js --save-baseline v0.1.411
+
+  # Check for regressions
+  node agent/unify/eval/run-eval.js --compare-baseline baselines/v0.1.411.json
+`);
+  process.exit(0);
+}
+
+// ─── Collect cases ───────────────────────────────────────────
+
+const allCases = {
+  tools: toolUseCases,
+  memory: memoryCases,
+  skills: skillsCases,
+  e2e: e2eCases,
+};
+
+let cases;
+if (args.suite === 'all') {
+  cases = [...toolUseCases, ...memoryCases, ...skillsCases, ...e2eCases];
+} else if (allCases[args.suite]) {
+  cases = allCases[args.suite];
+} else {
+  console.error(`Unknown suite: ${args.suite}. Available: tools, memory, skills, e2e, all`);
+  process.exit(1);
+}
+
+console.log(`\nYeaft Eval Runner`);
+console.log(`Suite: ${args.suite} (${cases.length} cases)`);
+
+// ─── Build adapters ──────────────────────────────────────────
+
+const adapters = [];
+
+if (args['dry-run']) {
+  // MockAdapter that gives simple responses
+  console.log('Mode: DRY RUN (MockAdapter)\n');
+
+  class DryRunAdapter {
+    async *stream(params) {
+      // Check if tools are available
+      const hasTools = params.tools && params.tools.length > 0;
+
+      // Simple heuristic: if prompt mentions search/find → call search tool
+      const prompt = params.messages[params.messages.length - 1]?.content || '';
+      const lp = prompt.toLowerCase();
+
+      if (hasTools && (lp.includes('search') || lp.includes('population') || lp.includes('find'))) {
+        const searchTool = params.tools.find(t => t.name === 'search');
+        if (searchTool) {
+          yield { type: 'text_delta', text: 'Let me search for that. ' };
+          yield { type: 'tool_call', id: 'mock-tc-1', name: 'search', input: { query: prompt } };
+          yield { type: 'usage', inputTokens: 100, outputTokens: 20 };
+          yield { type: 'stop', stopReason: 'tool_use' };
+          return;
+        }
+      }
+
+      if (hasTools && (lp.includes('calculate') || lp.includes('math') || /\d+\s*[\+\-\*\/]\s*\d+/.test(lp))) {
+        const calcTool = params.tools.find(t => t.name === 'calculator');
+        if (calcTool) {
+          const expr = prompt.match(/[\d\s\+\-\*\/\(\)]+/)?.[0]?.trim() || '0';
+          yield { type: 'text_delta', text: 'Let me calculate. ' };
+          yield { type: 'tool_call', id: 'mock-tc-1', name: 'calculator', input: { expression: expr } };
+          yield { type: 'usage', inputTokens: 100, outputTokens: 20 };
+          yield { type: 'stop', stopReason: 'tool_use' };
+          return;
+        }
+      }
+
+      // Default: just respond with text
+      yield { type: 'text_delta', text: `I understand you asked: "${prompt.slice(0, 50)}". ` };
+      yield { type: 'text_delta', text: 'Here is my response.' };
+      yield { type: 'usage', inputTokens: 100, outputTokens: 15 };
+      yield { type: 'stop', stopReason: 'end_turn' };
+    }
+    async call() {
+      return { text: '{}', usage: { inputTokens: 10, outputTokens: 5 } };
+    }
+  }
+
+  adapters.push({ name: 'dry-run-mock', adapter: new DryRunAdapter() });
+
+} else {
+  // Real adapters
+  const { createLLMAdapter } = await import('../llm/adapter.js');
+  const modelList = args.models
+    ? args.models.split(',').map(m => m.trim())
+    : [];
+
+  // Auto-detect from API keys if no models specified
+  if (modelList.length === 0) {
+    if (process.env.YEAFT_API_KEY) modelList.push('claude-sonnet-4-20250514');
+    if (process.env.YEAFT_OPENAI_API_KEY) modelList.push('gpt-5');
+  }
+
+  if (modelList.length === 0) {
+    console.error('\nNo models available. Set YEAFT_API_KEY or YEAFT_OPENAI_API_KEY, or use --dry-run.');
+    process.exit(1);
+  }
+
+  console.log(`Models: ${modelList.join(', ')}\n`);
+
+  for (const model of modelList) {
+    try {
+      const config = {
+        model,
+        apiKey: process.env.YEAFT_API_KEY,
+        openaiApiKey: process.env.YEAFT_OPENAI_API_KEY,
+      };
+      const adapter = await createLLMAdapter(config);
+      adapters.push({ name: model, adapter, config });
+    } catch (err) {
+      console.error(`Failed to create adapter for ${model}: ${err.message}`);
+    }
+  }
+}
+
+if (adapters.length === 0) {
+  console.error('No adapters available. Exiting.');
+  process.exit(1);
+}
+
+// ─── Run evals ───────────────────────────────────────────────
+
+console.log(`Running ${cases.length} eval cases across ${adapters.length} adapter(s)...\n`);
+
+const scores = await runEvals({ cases, adapters });
+
+// ─── Display results ─────────────────────────────────────────
+
+printResults(scores);
+
+// ─── Save baseline if requested ──────────────────────────────
+
+const baselineDir = join(homedir(), '.yeaft', 'eval', 'baselines');
+
+if (args['save-baseline']) {
+  const path = saveBaseline(scores, baselineDir, args['save-baseline']);
+  console.log(`\nBaseline saved: ${path}`);
+}
+
+// ─── Compare to baseline if requested ────────────────────────
+
+if (args['compare-baseline']) {
+  try {
+    const baseline = loadBaseline(args['compare-baseline']);
+    const comparison = compareToBaseline(scores, baseline);
+    printComparison(comparison);
+
+    if (comparison.regressions.length > 0) {
+      process.exit(1); // Exit with error code for CI
+    }
+  } catch (err) {
+    console.error(`\nFailed to load baseline: ${err.message}`);
+    process.exit(1);
+  }
+}
+
+// ─── Summary ─────────────────────────────────────────────────
+
+const passed = scores.filter(s => s.totalScore >= 80).length;
+const total = scores.length;
+console.log(`\n${passed}/${total} evals passed (≥80 score)`);