@yeaft/webchat-agent 0.1.411 → 0.1.413

package/unify/eval/runner.js

@@ -0,0 +1,525 @@
+/**
+ * eval/runner.js — Eval harness for Yeaft Unify
+ *
+ * Runs eval cases against one or more models and produces scored results.
+ * Each eval case defines:
+ *   - A prompt (what to send)
+ *   - Expected behavior (tool calls, output assertions, timing)
+ *   - Scoring rubric (pass/fail/score per criterion)
+ *
+ * Results are deterministic and comparable across models for regression detection.
+ *
+ * Usage:
+ *   import { runEvals, printResults, saveBaseline, compareToBaseline } from './runner.js';
+ *   const results = await runEvals({ models: ['claude-sonnet-4-20250514'], suite: 'tools' });
+ *   printResults(results);
+ */
+
+import { Engine } from '../engine.js';
+import { NullTrace } from '../debug-trace.js';
+import { buildSystemPrompt } from '../prompts.js';
+import { createEmptyRegistry } from '../tools/registry.js';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
+import { join } from 'path';
+
+// ─── Types ───────────────────────────────────────────────────
+
+/**
+ * @typedef {Object} EvalCase
+ * @property {string} id — unique identifier (e.g. "tool-use-search-basic")
+ * @property {string} suite — group name (e.g. "tools", "memory", "skills")
+ * @property {string} description — human-readable description
+ * @property {string} prompt — the user prompt to send
+ * @property {string} [mode] — 'chat' | 'work' (default: 'chat')
+ * @property {object[]} [messages] — prior conversation messages
+ * @property {object[]} [tools] — tool definitions to register
+ * @property {object} [registryTools] — ToolDef objects to register in ToolRegistry
+ * @property {Function} [setupEngine] — custom engine setup hook
+ * @property {EvalCriterion[]} criteria — scoring criteria
+ */
+
+/**
+ * @typedef {Object} EvalCriterion
+ * @property {string} id — criterion identifier
+ * @property {string} description — what this checks
+ * @property {number} weight — importance weight (1-10)
+ * @property {(result: EvalRunResult) => CriterionScore} score — scoring function
+ */
+
+/**
+ * @typedef {Object} CriterionScore
+ * @property {boolean} pass — did it pass?
+ * @property {number} score — 0.0 to 1.0
+ * @property {string} [reason] — explanation
+ */
+
+/**
+ * @typedef {Object} EvalRunResult
+ * @property {string} caseId
+ * @property {string} model
+ * @property {object[]} events — all engine events
+ * @property {string} fullText — concatenated text_delta
+ * @property {object[]} toolCalls — tool_call events
+ * @property {object[]} toolResults — tool_end events
+ * @property {number} turnCount
+ * @property {number} inputTokens
+ * @property {number} outputTokens
+ * @property {number} latencyMs
+ * @property {string|null} error — error message if failed
+ */
+
+/**
+ * @typedef {Object} EvalScore
+ * @property {string} caseId
+ * @property {string} model
+ * @property {number} totalScore — weighted average 0-100
+ * @property {Record<string, CriterionScore>} criteria
+ * @property {EvalRunResult} raw
+ */
+
+// ─── Runner ──────────────────────────────────────────────────
+
+/**
+ * Run a single eval case against a model adapter.
+ *
+ * @param {EvalCase} evalCase
+ * @param {{ adapter: object, model: string, config?: object }} options
+ * @returns {Promise<EvalRunResult>}
+ */
+export async function runSingleEval(evalCase, { adapter, model, config = {} }) {
+  const trace = new NullTrace();
+  const engineConfig = { model, maxOutputTokens: 4096, ...config };
+
+  // Build engine — optionally with ToolRegistry
+  const engineOpts = { adapter, trace, config: engineConfig };
+
+  if (evalCase.registryTools) {
+    const registry = createEmptyRegistry();
+    for (const tool of evalCase.registryTools) {
+      registry.register(tool);
+    }
+    engineOpts.toolRegistry = registry;
+  }
+
+  const engine = new Engine(engineOpts);
+
+  // Register legacy tools if provided
+  if (evalCase.tools) {
+    for (const tool of evalCase.tools) {
+      engine.registerTool(tool);
+    }
+  }
+
+  // Custom setup hook
+  if (evalCase.setupEngine) {
+    evalCase.setupEngine(engine);
+  }
+
+  const events = [];
+  let fullText = '';
+  const toolCalls = [];
+  const toolResults = [];
+  let turnCount = 0;
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let error = null;
+
+  const startTime = Date.now();
+
+  try {
+    for await (const event of engine.query({
+      prompt: evalCase.prompt,
+      mode: evalCase.mode || 'chat',
+      messages: evalCase.messages || [],
+    })) {
+      events.push(event);
+
+      switch (event.type) {
+        case 'text_delta':
+          fullText += event.text;
+          break;
+        case 'tool_call':
+          toolCalls.push(event);
+          break;
+        case 'tool_end':
+          toolResults.push(event);
+          break;
+        case 'turn_start':
+          turnCount++;
+          break;
+        case 'usage':
+          inputTokens += event.inputTokens || 0;
+          outputTokens += event.outputTokens || 0;
+          break;
+        case 'error':
+          error = event.error?.message || 'Unknown error';
+          break;
+      }
+    }
+  } catch (err) {
+    error = err.message;
+  }
+
+  const latencyMs = Date.now() - startTime;
+
+  return {
+    caseId: evalCase.id,
+    model,
+    events,
+    fullText,
+    toolCalls,
+    toolResults,
+    turnCount,
+    inputTokens,
+    outputTokens,
+    latencyMs,
+    error,
+  };
+}
+
+/**
+ * Score an eval run against its criteria.
+ *
+ * @param {EvalCase} evalCase
+ * @param {EvalRunResult} runResult
+ * @returns {EvalScore}
+ */
+export function scoreEval(evalCase, runResult) {
+  const criteriaResults = {};
+  let weightedSum = 0;
+  let totalWeight = 0;
+
+  for (const criterion of evalCase.criteria) {
+    const result = criterion.score(runResult);
+    criteriaResults[criterion.id] = result;
+    weightedSum += result.score * criterion.weight;
+    totalWeight += criterion.weight;
+  }
+
+  return {
+    caseId: evalCase.id,
+    model: runResult.model,
+    totalScore: totalWeight > 0 ? Math.round((weightedSum / totalWeight) * 100) : 0,
+    criteria: criteriaResults,
+    raw: runResult,
+  };
+}
+
+/**
+ * Run multiple eval cases against multiple adapters.
+ *
+ * @param {{
+ *   cases: EvalCase[],
+ *   adapters: { name: string, adapter: object, config?: object }[],
+ * }} params
+ * @returns {Promise<EvalScore[]>}
+ */
+export async function runEvals({ cases, adapters }) {
+  const allScores = [];
+
+  for (const { name: model, adapter, config } of adapters) {
+    for (const evalCase of cases) {
+      const runResult = await runSingleEval(evalCase, { adapter, model, config });
+      const score = scoreEval(evalCase, runResult);
+      allScores.push(score);
+    }
+  }
+
+  return allScores;
+}
+
+// ─── Baseline Management ─────────────────────────────────────
+
+/**
+ * Save eval scores as a baseline for future comparison.
+ *
+ * @param {EvalScore[]} scores
+ * @param {string} baselineDir — directory to store baselines
+ * @param {string} [name] — baseline name (default: timestamp)
+ */
+export function saveBaseline(scores, baselineDir, name) {
+  mkdirSync(baselineDir, { recursive: true });
+  const filename = `${name || new Date().toISOString().replace(/[:.]/g, '-')}.json`;
+  const baseline = {
+    timestamp: new Date().toISOString(),
+    name: name || 'unnamed',
+    scores: scores.map(s => ({
+      caseId: s.caseId,
+      model: s.model,
+      totalScore: s.totalScore,
+      criteria: s.criteria,
+      // Omit raw to keep baseline files small
+    })),
+  };
+  writeFileSync(join(baselineDir, filename), JSON.stringify(baseline, null, 2));
+  return join(baselineDir, filename);
+}
+
+/**
+ * Load a baseline file.
+ *
+ * @param {string} path
+ * @returns {object}
+ */
+export function loadBaseline(path) {
+  return JSON.parse(readFileSync(path, 'utf8'));
+}
+
+/**
+ * Compare current scores against a baseline.
+ * Returns a list of regressions (cases where score dropped).
+ *
+ * @param {EvalScore[]} current
+ * @param {object} baseline — loaded baseline object
+ * @param {number} [threshold=5] — score drop threshold to flag as regression
+ * @returns {{ regressions: object[], improvements: object[], unchanged: object[] }}
+ */
+export function compareToBaseline(current, baseline, threshold = 5) {
+  const baselineMap = new Map();
+  for (const s of baseline.scores) {
+    baselineMap.set(`${s.model}::${s.caseId}`, s);
+  }
+
+  const regressions = [];
+  const improvements = [];
+  const unchanged = [];
+
+  for (const score of current) {
+    const key = `${score.model}::${score.caseId}`;
+    const base = baselineMap.get(key);
+
+    if (!base) {
+      unchanged.push({ ...score, baseScore: null, delta: null, status: 'new' });
+      continue;
+    }
+
+    const delta = score.totalScore - base.totalScore;
+
+    if (delta < -threshold) {
+      regressions.push({
+        caseId: score.caseId,
+        model: score.model,
+        currentScore: score.totalScore,
+        baseScore: base.totalScore,
+        delta,
+        criteria: score.criteria,
+        baseCriteria: base.criteria,
+      });
+    } else if (delta > threshold) {
+      improvements.push({
+        caseId: score.caseId,
+        model: score.model,
+        currentScore: score.totalScore,
+        baseScore: base.totalScore,
+        delta,
+      });
+    } else {
+      unchanged.push({
+        caseId: score.caseId,
+        model: score.model,
+        currentScore: score.totalScore,
+        baseScore: base.totalScore,
+        delta,
+      });
+    }
+  }
+
+  return { regressions, improvements, unchanged };
+}
+
+// ─── Display ─────────────────────────────────────────────────
+
+/**
+ * Print eval results as a formatted table.
+ *
+ * @param {EvalScore[]} scores
+ */
+export function printResults(scores) {
+  // Group by model
+  const byModel = new Map();
+  for (const s of scores) {
+    if (!byModel.has(s.model)) byModel.set(s.model, []);
+    byModel.get(s.model).push(s);
+  }
+
+  for (const [model, modelScores] of byModel) {
+    console.log(`\n${'═'.repeat(60)}`);
+    console.log(`  Model: ${model}`);
+    console.log(`${'═'.repeat(60)}`);
+
+    for (const s of modelScores) {
+      const icon = s.totalScore >= 80 ? '✅' : s.totalScore >= 50 ? '⚠️' : '❌';
+      console.log(`\n  ${icon} ${s.caseId} — ${s.totalScore}/100`);
+
+      for (const [critId, crit] of Object.entries(s.criteria)) {
+        const critIcon = crit.pass ? '  ✓' : '  ✗';
+        console.log(`    ${critIcon} ${critId}: ${Math.round(crit.score * 100)}%${crit.reason ? ` — ${crit.reason}` : ''}`);
+      }
+    }
+
+    const avg = Math.round(modelScores.reduce((sum, s) => sum + s.totalScore, 0) / modelScores.length);
+    console.log(`\n  Average: ${avg}/100`);
+  }
+}
+
+/**
+ * Print baseline comparison results.
+ *
+ * @param {{ regressions: object[], improvements: object[], unchanged: object[] }} comparison
+ */
+export function printComparison(comparison) {
+  const { regressions, improvements, unchanged } = comparison;
+
+  console.log(`\n${'═'.repeat(60)}`);
+  console.log('  Baseline Comparison');
+  console.log(`${'═'.repeat(60)}`);
+
+  if (regressions.length > 0) {
+    console.log(`\n  🔴 REGRESSIONS (${regressions.length}):`);
+    for (const r of regressions) {
+      console.log(`    ${r.model} :: ${r.caseId}: ${r.baseScore} → ${r.currentScore} (${r.delta})`);
+    }
+  }
+
+  if (improvements.length > 0) {
+    console.log(`\n  🟢 IMPROVEMENTS (${improvements.length}):`);
+    for (const i of improvements) {
+      console.log(`    ${i.model} :: ${i.caseId}: ${i.baseScore} → ${i.currentScore} (+${i.delta})`);
+    }
+  }
+
+  console.log(`\n  ⚪ Unchanged: ${unchanged.length} cases`);
+
+  if (regressions.length > 0) {
+    console.log('\n  ⛔ REGRESSION DETECTED — eval failed');
+  } else {
+    console.log('\n  ✅ No regressions detected');
+  }
+}
+
+// ─── Criterion Helpers ───────────────────────────────────────
+
+/** Check that no error occurred. */
+export const noError = {
+  id: 'no-error',
+  description: 'No error during execution',
+  weight: 10,
+  score: (result) => ({
+    pass: !result.error,
+    score: result.error ? 0 : 1,
+    reason: result.error || undefined,
+  }),
+};
+
+/** Check that response contains expected text. */
+export function containsText(text, opts = {}) {
+  return {
+    id: opts.id || `contains-${text.slice(0, 20)}`,
+    description: opts.description || `Response contains "${text}"`,
+    weight: opts.weight || 5,
+    score: (result) => {
+      const found = result.fullText.toLowerCase().includes(text.toLowerCase());
+      return { pass: found, score: found ? 1 : 0 };
+    },
+  };
+}
+
+/** Check that a specific tool was called. */
+export function toolWasCalled(toolName, opts = {}) {
+  return {
+    id: opts.id || `tool-called-${toolName}`,
+    description: opts.description || `Tool "${toolName}" was called`,
+    weight: opts.weight || 8,
+    score: (result) => {
+      const called = result.toolCalls.some(tc => tc.name === toolName);
+      return { pass: called, score: called ? 1 : 0 };
+    },
+  };
+}
+
+/** Check that a tool was called with specific input. */
+export function toolCalledWith(toolName, inputMatcher, opts = {}) {
+  return {
+    id: opts.id || `tool-input-${toolName}`,
+    description: opts.description || `Tool "${toolName}" called with correct input`,
+    weight: opts.weight || 6,
+    score: (result) => {
+      const call = result.toolCalls.find(tc => tc.name === toolName);
+      if (!call) return { pass: false, score: 0, reason: 'Tool not called' };
+      const match = inputMatcher(call.input);
+      return { pass: match, score: match ? 1 : 0 };
+    },
+  };
+}
+
+/** Check that a tool was NOT called. */
+export function toolNotCalled(toolName, opts = {}) {
+  return {
+    id: opts.id || `tool-not-called-${toolName}`,
+    description: opts.description || `Tool "${toolName}" was NOT called`,
+    weight: opts.weight || 5,
+    score: (result) => {
+      const called = result.toolCalls.some(tc => tc.name === toolName);
+      return { pass: !called, score: called ? 0 : 1 };
+    },
+  };
+}
+
+/** Check that a tool succeeded (not error). */
+export function toolSucceeded(toolName, opts = {}) {
+  return {
+    id: opts.id || `tool-success-${toolName}`,
+    description: opts.description || `Tool "${toolName}" succeeded`,
+    weight: opts.weight || 7,
+    score: (result) => {
+      const toolResult = result.toolResults.find(tr => tr.name === toolName);
+      if (!toolResult) return { pass: false, score: 0, reason: 'Tool not found in results' };
+      return { pass: !toolResult.isError, score: toolResult.isError ? 0 : 1, reason: toolResult.isError ? toolResult.output : undefined };
+    },
+  };
+}
+
+/** Check total turn count is within bounds. */
+export function turnCountInRange(min, max, opts = {}) {
+  return {
+    id: opts.id || `turns-${min}-${max}`,
+    description: opts.description || `Turn count between ${min} and ${max}`,
+    weight: opts.weight || 3,
+    score: (result) => {
+      const inRange = result.turnCount >= min && result.turnCount <= max;
+      return { pass: inRange, score: inRange ? 1 : 0, reason: `${result.turnCount} turns` };
+    },
+  };
+}
+
+/** Check that response does NOT contain text. */
+export function doesNotContain(text, opts = {}) {
+  return {
+    id: opts.id || `not-contains-${text.slice(0, 20)}`,
+    description: opts.description || `Response does NOT contain "${text}"`,
+    weight: opts.weight || 4,
+    score: (result) => {
+      const found = result.fullText.toLowerCase().includes(text.toLowerCase());
+      return { pass: !found, score: found ? 0 : 1 };
+    },
+  };
+}
+
+/** Check response length is within bounds (characters). */
+export function responseLengthInRange(min, max, opts = {}) {
+  return {
+    id: opts.id || `length-${min}-${max}`,
+    description: opts.description || `Response length between ${min} and ${max} chars`,
+    weight: opts.weight || 2,
+    score: (result) => {
+      const len = result.fullText.length;
+      const inRange = len >= min && len <= max;
+      return { pass: inRange, score: inRange ? 1 : 0, reason: `${len} chars` };
+    },
+  };
+}
+
+/** Custom criterion with arbitrary scoring function. */
+export function custom(id, description, weight, scoreFn) {
+  return { id, description, weight, score: scoreFn };
+}

package/unify/index.js

@@ -16,7 +16,8 @@ export {
   LLMAbortError,
   createLLMAdapter,
 } from './llm/adapter.js';
-export { MODEL_REGISTRY, resolveModel, listModels, isKnownModel } from './models.js';
+export { AdapterRouter } from './llm/router.js';
+export { MODEL_REGISTRY, resolveModel, listModels, isKnownModel, getProviderForModel, parseModelRef } from './models.js';
 export { buildSystemPrompt, SUPPORTED_LANGUAGES } from './prompts.js';
 export { Engine } from './engine.js';
 export { ConversationStore, parseMessage, estimateTokens } from './conversation/persist.js';

package/unify/init.js

@@ -23,56 +23,28 @@ const SUBDIRS = [
   'skills',
 ];
 
-/** Default config.md content (YAML frontmatter + markdown body). */
-const DEFAULT_CONFIG = `---
-model: claude-sonnet-4-20250514
-language: en
-debug: false
-maxContextTokens: 200000
----
-
-# Yeaft Config
-
-Edit the YAML frontmatter above to change settings.
-The \`model\` field is a model ID (e.g. \`gpt-5\`, \`claude-sonnet-4-20250514\`).
-Yeaft auto-detects the correct API adapter and endpoint from the model ID.
-
-## Language
-
-Set \`language\` in frontmatter or \`YEAFT_LANGUAGE\` env var:
-- \`en\` — English system prompts (default)
-- \`zh\` — Chinese system prompts (中文系统提示)
-
-## Model IDs
-
-- \`claude-sonnet-4-20250514\` (default)
-- \`claude-opus-4-20250514\`
-- \`gpt-5\`, \`gpt-5.4\`, \`gpt-4.1\`, \`gpt-4.1-mini\`
-- \`o3\`, \`o4-mini\`
-- \`deepseek-chat\`, \`deepseek-reasoner\`
-- \`gemini-2.5-pro\`, \`gemini-2.5-flash\`
-
-## API Keys
-
-Store API keys in \`~/.yeaft/.env\` (recommended) or export as env vars:
-
-\`\`\`bash
-# ~/.yeaft/.env
-YEAFT_API_KEY=sk-ant-...          # Anthropic
-YEAFT_OPENAI_API_KEY=sk-...       # OpenAI / DeepSeek / Gemini
-\`\`\`
-
-## Environment Variables
-
-Shell env vars take precedence over .env and config.md:
-
-- \`YEAFT_MODEL\` — override model ID
-- \`YEAFT_LANGUAGE\` — language for system prompts (en/zh)
-- \`YEAFT_API_KEY\` — Anthropic API key
-- \`YEAFT_OPENAI_API_KEY\` — OpenAI-compatible API key
-- \`YEAFT_PROXY_URL\` — CopilotProxy URL (default: http://localhost:6628)
-- \`YEAFT_DEBUG\` — enable debug mode (1/true)
-- \`YEAFT_DIR\` — data directory (default: ~/.yeaft)
+/** Default config.json — generated on first init as default configuration. */
+const DEFAULT_CONFIG_JSON = `{
+  "providers": [
+    {
+      "name": "my-proxy",
+      "baseUrl": "http://localhost:6628/v1",
+      "apiKey": "proxy",
+      "models": [
+        "claude-sonnet-4-20250514",
+        "claude-haiku-3-20250414",
+        "gpt-5",
+        "deepseek-chat"
+      ]
+    }
+  ],
+  "primaryModel": "my-proxy/claude-sonnet-4-20250514",
+  "fastModel": "my-proxy/claude-sonnet-4-20250514",
+  "language": "en",
+  "debug": false,
+  "maxContextTokens": 200000,
+  "messageTokenBudget": 8192
+}
 `;
 
 /** Default MEMORY.md content. */
@@ -88,6 +60,23 @@ This file stores persistent memory entries. The agent will read and update this
 
 `;
 
+/** Default mcp.json example — generated as reference for MCP server configuration. */
+const DEFAULT_MCP_EXAMPLE = `{
+  "_comment": "MCP server configuration. Rename to mcp.json to enable.",
+  "_docs": "Each server needs 'name' + 'command'. Optional: 'args', 'env'.",
+  "servers": [
+    {
+      "name": "example-github",
+      "command": "npx",
+      "args": ["@modelcontextprotocol/server-github"],
+      "env": {
+        "GITHUB_TOKEN": "ghp_your_token_here"
+      }
+    }
+  ]
+}
+`;
+
 /** Default conversation/index.md content. */
 const DEFAULT_CONVERSATION_INDEX = `---
 lastMessageId: null
@@ -125,10 +114,11 @@ export function initYeaftDir(dir) {
   }
 
   // Create default files if they don't exist
-  const configPath = join(root, 'config.md');
-  if (!existsSync(configPath)) {
-    writeFileSync(configPath, DEFAULT_CONFIG, 'utf8');
-    created.push(configPath);
+  // config.json — default configuration (user edits this directly)
+  const configJsonPath = join(root, 'config.json');
+  if (!existsSync(configJsonPath)) {
+    writeFileSync(configJsonPath, DEFAULT_CONFIG_JSON, 'utf8');
+    created.push(configJsonPath);
   }
 
   const memoryPath = join(root, 'memory', 'MEMORY.md');
@@ -143,5 +133,12 @@ export function initYeaftDir(dir) {
     created.push(indexPath);
   }
 
+  // mcp.json.example — reference template for MCP server configuration
+  const mcpExamplePath = join(root, 'mcp.json.example');
+  if (!existsSync(mcpExamplePath)) {
+    writeFileSync(mcpExamplePath, DEFAULT_MCP_EXAMPLE, 'utf8');
+    created.push(mcpExamplePath);
+  }
+
   return { dir: root, created };
 }

package/unify/llm/adapter.js

@@ -143,10 +143,22 @@ export class LLMAdapter {
 /**
  * Create an LLM adapter based on configuration.
  *
+ * If config.providers exists (config.json path), creates an AdapterRouter
+ * that routes requests to the correct provider based on model ID.
+ *
+ * Otherwise falls back to legacy single-adapter creation from env vars.
+ *
  * @param {object} config — From loadConfig()
  * @returns {Promise<LLMAdapter>}
  */
 export async function createLLMAdapter(config) {
+  // ─── New path: config.json with providers ─────────────
+  if (config.providers && config.providers.length > 0) {
+    const { AdapterRouter } = await import('./router.js');
+    return new AdapterRouter({ providers: config.providers });
+  }
+
+  // ─── Legacy path: single adapter from env vars ────────
   // Normalize adapter name — accept 'chat-completions' as alias for 'openai'
   const adapter = config.adapter === 'chat-completions' ? 'openai' : config.adapter;