@swarmclawai/swarmclaw 0.6.7 → 0.6.8

package/src/lib/server/daemon-state.ts

@@ -1,4 +1,4 @@
-import { loadQueue, loadSchedules, loadSessions, saveSessions, loadConnectors, saveConnectors, loadWebhookRetryQueue, upsertWebhookRetry, deleteWebhookRetry, loadWebhooks, loadAgents, appendWebhookLog, loadCredentials, decryptKey } from './storage'
+import { loadQueue, loadSchedules, loadSessions, saveSessions, loadConnectors, saveConnectors, loadWebhookRetryQueue, upsertWebhookRetry, deleteWebhookRetry, loadWebhooks, loadAgents, loadSettings, appendWebhookLog, loadCredentials, decryptKey } from './storage'
 import { notify } from './ws-hub'
 import { processNext, cleanupFinishedTaskSessions, validateCompletedTasksQueue, recoverStalledRunningTasks } from './queue'
 import { startScheduler, stopScheduler } from './scheduler'
@@ -80,6 +80,7 @@ const ds: {
   healthIntervalId: ReturnType<typeof setInterval> | null
   memoryConsolidationTimeoutId: ReturnType<typeof setTimeout> | null
   memoryConsolidationIntervalId: ReturnType<typeof setInterval> | null
+  evalSchedulerIntervalId: ReturnType<typeof setInterval> | null
   /** Session IDs we've already alerted as stale (alert-once semantics). */
   staleSessionIds: Set<string>
   connectorRestartState: Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>
@@ -97,6 +98,7 @@ const ds: {
   healthIntervalId: null,
   memoryConsolidationTimeoutId: null,
   memoryConsolidationIntervalId: null,
+  evalSchedulerIntervalId: null,
   staleSessionIds: new Set<string>(),
   connectorRestartState: new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>(),
   openclawDownAgentIds: new Set<string>(),
@@ -118,6 +120,7 @@ if (ds.healthIntervalId === undefined) ds.healthIntervalId = null
 if (ds.manualStopRequested === undefined) ds.manualStopRequested = false
 if (ds.memoryConsolidationTimeoutId === undefined) ds.memoryConsolidationTimeoutId = null
 if (ds.memoryConsolidationIntervalId === undefined) ds.memoryConsolidationIntervalId = null
+if (ds.evalSchedulerIntervalId === undefined) ds.evalSchedulerIntervalId = null
 
 export function ensureDaemonStarted(source = 'unknown'): boolean {
   if (ds.running) return false
@@ -140,6 +143,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
     startHealthMonitor()
     startHeartbeatService()
     startMemoryConsolidation()
+    startEvalScheduler()
     return
   }
   ds.running = true
@@ -155,6 +159,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
     startHealthMonitor()
     startHeartbeatService()
     startMemoryConsolidation()
+    startEvalScheduler()
   } catch (err: unknown) {
     ds.running = false
     notify('daemon')
@@ -182,6 +187,7 @@ export function stopDaemon(options?: { source?: string; manualStop?: boolean })
   stopHealthMonitor()
   stopHeartbeatService()
   stopMemoryConsolidation()
+  stopEvalScheduler()
   stopAllConnectors().catch(() => {})
 }
 
@@ -785,6 +791,69 @@ function stopMemoryConsolidation() {
   }
 }
 
+// --- Eval scheduler ---
+
+const EVAL_DEFAULT_INTERVAL_MS = 24 * 3600_000 // 24 hours
+
+function parseCronToMs(cron: string | null | undefined): number | null {
+  if (!cron || typeof cron !== 'string') return null
+  // Simple heuristic: extract hours from common cron patterns like "0 */6 * * *"
+  const hourMatch = cron.match(/\*\/(\d+)/)
+  if (hourMatch) return parseInt(hourMatch[1], 10) * 3600_000
+  return EVAL_DEFAULT_INTERVAL_MS
+}
+
+async function runEvalSchedulerTick() {
+  try {
+    const settings = loadSettings()
+    if (!settings.autonomyEvalEnabled) return
+
+    const { runEvalSuite } = await import('./eval/runner')
+    const agents = loadAgents()
+    const heartbeatAgentIds = Object.keys(agents).filter(
+      (id) => agents[id].heartbeatEnabled === true,
+    )
+
+    for (const agentId of heartbeatAgentIds) {
+      try {
+        const result = await runEvalSuite(agentId)
+        console.log(
+          `[daemon:eval] Agent ${agents[agentId].name}: ${result.percentage}% (${result.totalScore}/${result.maxScore})`,
+        )
+        createNotification({
+          title: `Eval: ${agents[agentId].name} scored ${result.percentage}%`,
+          message: `${result.runs.length} scenarios, ${result.totalScore}/${result.maxScore} points`,
+          type: result.percentage >= 60 ? 'info' : 'warning',
+        })
+      } catch (err: unknown) {
+        console.error(`[daemon:eval] Failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err))
+      }
+    }
+  } catch (err: unknown) {
+    console.error('[daemon:eval] Scheduler tick error:', err instanceof Error ? err.message : String(err))
+  }
+}
+
+function startEvalScheduler() {
+  if (ds.evalSchedulerIntervalId) return
+  try {
+    const settings = loadSettings()
+    if (!settings.autonomyEvalEnabled) return
+    const intervalMs = parseCronToMs(settings.autonomyEvalCron) || EVAL_DEFAULT_INTERVAL_MS
+    ds.evalSchedulerIntervalId = setInterval(runEvalSchedulerTick, intervalMs)
+    console.log(`[daemon:eval] Eval scheduler started (interval=${Math.round(intervalMs / 3600_000)}h)`)
+  } catch {
+    // Eval scheduling is optional — don't block daemon start
+  }
+}
+
+function stopEvalScheduler() {
+  if (ds.evalSchedulerIntervalId) {
+    clearInterval(ds.evalSchedulerIntervalId)
+    ds.evalSchedulerIntervalId = null
+  }
+}
+
 export async function runDaemonHealthCheckNow() {
   await runHealthChecks()
 }

package/src/lib/server/eval/runner.ts

@@ -0,0 +1,126 @@
+import { genId } from '@/lib/id'
+import type { EvalScenario, EvalRun, EvalSuiteResult } from './types'
+import { getScenario, EVAL_SCENARIOS } from './scenarios'
+import { scoreCriteria } from './scorer'
+import { saveEvalRun } from './store'
+import { loadSessions, saveSessions, loadAgents, loadCredentials, decryptKey } from '../storage'
+import { executeSessionChatTurn } from '../chat-execution'
+import type { Session } from '@/types'
+
+export async function runEvalScenario(scenarioId: string, agentId: string): Promise<EvalRun> {
+  const scenario = getScenario(scenarioId)
+  if (!scenario) throw new Error(`Unknown eval scenario: ${scenarioId}`)
+
+  const agents = loadAgents() as Record<string, Record<string, unknown>>
+  const agent = agents[agentId]
+  if (!agent) throw new Error(`Unknown agent: ${agentId}`)
+
+  const runId = genId()
+  const sessionId = `eval-${runId}`
+  const now = Date.now()
+
+  const run: EvalRun = {
+    id: runId,
+    scenarioId,
+    agentId,
+    status: 'running',
+    startedAt: now,
+    score: 0,
+    maxScore: scenario.scoringCriteria.reduce((sum, c) => sum + c.weight, 0),
+    details: [],
+    sessionId,
+  }
+
+  // Create temporary eval session
+  const sessions = loadSessions() as Record<string, Session>
+  const evalSession: Session = {
+    id: sessionId,
+    name: `Eval: ${scenario.name}`,
+    cwd: process.cwd(),
+    user: 'eval-runner',
+    provider: (agent.provider as Session['provider']) ?? 'anthropic',
+    model: (agent.model as string) ?? '',
+    credentialId: (agent.credentialId as string | null) ?? null,
+    apiEndpoint: (agent.apiEndpoint as string | null) ?? null,
+    claudeSessionId: null,
+    agentId,
+    tools: scenario.tools,
+    messages: [],
+    createdAt: now,
+    lastActiveAt: now,
+  }
+  sessions[sessionId] = evalSession
+  saveSessions(sessions)
+
+  try {
+    const result = await executeSessionChatTurn({
+      sessionId,
+      message: scenario.userMessage,
+      internal: true,
+      source: 'eval',
+    })
+
+    const judgeProvider = typeof agent.provider === 'string' ? agent.provider : undefined
+    const judgeModel = typeof agent.model === 'string' ? agent.model : undefined
+    let judgeApiKey: string | null = null
+    if (typeof agent.credentialId === 'string' && agent.credentialId) {
+      const creds = loadCredentials()
+      const cred = creds[agent.credentialId]
+      if (cred) {
+        try { judgeApiKey = decryptKey(cred.encryptedKey) } catch { /* skip undecryptable */ }
+      }
+    }
+    const judgeOpts = judgeProvider && judgeModel ? {
+      provider: judgeProvider,
+      model: judgeModel,
+      apiKey: judgeApiKey,
+      apiEndpoint: typeof agent.apiEndpoint === 'string' ? agent.apiEndpoint : undefined,
+    } : undefined
+
+    run.details = await scoreCriteria(
+      scenario.scoringCriteria,
+      result.text,
+      result.toolEvents || [],
+      judgeOpts,
+    )
+    run.score = run.details.reduce((sum, d) => sum + d.score, 0)
+    run.status = 'completed'
+    run.endedAt = Date.now()
+  } catch (err: unknown) {
+    run.status = 'failed'
+    run.error = err instanceof Error ? err.message : String(err)
+    run.endedAt = Date.now()
+  } finally {
+    // Clean up eval session
+    const currentSessions = loadSessions() as Record<string, Session>
+    delete currentSessions[sessionId]
+    saveSessions(currentSessions)
+  }
+
+  saveEvalRun(run)
+  return run
+}
+
+export async function runEvalSuite(agentId: string, categories?: string[]): Promise<EvalSuiteResult> {
+  const scenarios: EvalScenario[] = categories
+    ? EVAL_SCENARIOS.filter(s => categories.includes(s.category))
+    : EVAL_SCENARIOS
+
+  const runs: EvalRun[] = []
+  for (const scenario of scenarios) {
+    const evalRun = await runEvalScenario(scenario.id, agentId)
+    runs.push(evalRun)
+  }
+
+  const totalScore = runs.reduce((sum, r) => sum + r.score, 0)
+  const maxScore = runs.reduce((sum, r) => sum + r.maxScore, 0)
+
+  return {
+    agentId,
+    totalScore,
+    maxScore,
+    percentage: maxScore > 0 ? Math.round((totalScore / maxScore) * 100) : 0,
+    runs,
+    completedAt: Date.now(),
+  }
+}

package/src/lib/server/eval/scenarios.ts

@@ -0,0 +1,218 @@
+import type { EvalScenario } from './types'
+
+export const EVAL_SCENARIOS: EvalScenario[] = [
+  {
+    id: 'coding-prime',
+    name: 'Prime Number Function',
+    category: 'coding',
+    description: 'Create and test a function that checks if a number is prime',
+    userMessage: 'Create a function that checks if a number is prime and test it with a few examples including 2, 7, 10, and 97.',
+    expectedBehaviors: [
+      'Writes a correct isPrime function',
+      'Tests with the specified numbers',
+      'Returns correct results for each test case',
+    ],
+    scoringCriteria: [
+      { name: 'uses_shell', weight: 2, evaluator: 'tool_used', expected: 'shell' },
+      { name: 'uses_files', weight: 2, evaluator: 'tool_used', expected: 'files' },
+      { name: 'mentions_prime', weight: 1, evaluator: 'contains', expected: 'prime' },
+      { name: 'tests_number_2', weight: 1, evaluator: 'contains', expected: '2' },
+      { name: 'tests_number_97', weight: 1, evaluator: 'contains', expected: '97' },
+      { name: 'correctness', weight: 3, evaluator: 'llm_judge', expected: 'Did the response correctly implement an isPrime function and test it with 2, 7, 10, and 97, producing correct results (2=prime, 7=prime, 10=not prime, 97=prime)?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['shell', 'files'],
+  },
+  {
+    id: 'research-frameworks',
+    name: 'Node.js Framework Comparison',
+    category: 'research',
+    description: 'Research and compare top Node.js web frameworks by GitHub stars',
+    userMessage: 'Find the top 3 Node.js web frameworks by GitHub stars and compare them. Include star counts and key differences.',
+    expectedBehaviors: [
+      'Searches the web for framework information',
+      'Identifies at least 3 frameworks (e.g. Express, Next.js, Fastify)',
+      'Provides star counts and meaningful comparison',
+    ],
+    scoringCriteria: [
+      { name: 'uses_web_search', weight: 2, evaluator: 'tool_used', expected: 'web_search' },
+      { name: 'mentions_express', weight: 1, evaluator: 'regex', expected: 'express(\\.js)?' },
+      { name: 'mentions_next', weight: 1, evaluator: 'regex', expected: 'next(\\.js)?' },
+      { name: 'includes_stars', weight: 1, evaluator: 'regex', expected: '\\d+[kK,.]?\\d*\\s*(stars|\\*)' },
+      { name: 'quality', weight: 5, evaluator: 'llm_judge', expected: 'Did the response identify at least 3 Node.js web frameworks, include GitHub star counts, and provide a meaningful comparison of their key differences?' },
+    ],
+    timeoutMs: 90_000,
+    tools: ['web_search', 'web_fetch'],
+  },
+  {
+    id: 'companionship-overwhelmed',
+    name: 'Emotional Support - Overwhelmed',
+    category: 'companionship',
+    description: 'Provide empathetic support when user is overwhelmed with work',
+    userMessage: "I'm feeling overwhelmed with work deadlines. Everything is piling up and I don't know where to start.",
+    expectedBehaviors: [
+      'Acknowledges the feeling of being overwhelmed',
+      'Shows empathy without being dismissive',
+      'Offers actionable suggestions for managing workload',
+    ],
+    scoringCriteria: [
+      { name: 'empathy', weight: 4, evaluator: 'llm_judge', expected: 'Does the response show genuine empathy and acknowledge the user\'s feelings of being overwhelmed without being dismissive or jumping straight to advice?' },
+      { name: 'actionable_advice', weight: 3, evaluator: 'llm_judge', expected: 'Does the response offer practical, actionable suggestions for managing workload or reducing the feeling of being overwhelmed (e.g. prioritization, breaking tasks down, time management)?' },
+      { name: 'appropriate_tone', weight: 3, evaluator: 'llm_judge', expected: 'Is the tone warm, supportive, and human-like rather than clinical, robotic, or overly formal?' },
+    ],
+    timeoutMs: 30_000,
+    tools: [],
+  },
+  {
+    id: 'multi-step-project',
+    name: 'Project Directory Setup',
+    category: 'multi-step',
+    description: 'Create a project directory, write a README, and initialize git',
+    userMessage: "Create a project directory called 'demo-project' in /tmp, write a README.md with a title and description, and initialize a git repository in it.",
+    expectedBehaviors: [
+      'Creates the demo-project directory',
+      'Writes a README.md file with content',
+      'Initializes a git repository',
+    ],
+    scoringCriteria: [
+      { name: 'uses_shell', weight: 2, evaluator: 'tool_used', expected: 'shell' },
+      { name: 'uses_files', weight: 2, evaluator: 'tool_used', expected: 'files' },
+      { name: 'mentions_mkdir', weight: 1, evaluator: 'regex', expected: 'demo-project' },
+      { name: 'mentions_readme', weight: 1, evaluator: 'contains', expected: 'README' },
+      { name: 'mentions_git_init', weight: 1, evaluator: 'contains', expected: 'git init' },
+      { name: 'completeness', weight: 3, evaluator: 'llm_judge', expected: 'Did the response successfully complete all 3 steps: create the demo-project directory, write a README.md with meaningful content, and initialize a git repository?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['shell', 'files'],
+  },
+  {
+    id: 'memory-store-recall',
+    name: 'Memory Store and Recall',
+    category: 'memory',
+    description: 'Store a fact in memory and demonstrate recall capability',
+    userMessage: 'Remember that my favorite programming language is Rust and I prefer functional programming patterns. Then confirm what you just stored.',
+    expectedBehaviors: [
+      'Uses memory tool to store the information',
+      'Confirms what was stored',
+      'Accurately reflects the stored preferences',
+    ],
+    scoringCriteria: [
+      { name: 'uses_memory', weight: 3, evaluator: 'tool_used', expected: 'memory' },
+      { name: 'mentions_rust', weight: 2, evaluator: 'contains', expected: 'Rust' },
+      { name: 'mentions_functional', weight: 2, evaluator: 'contains', expected: 'functional' },
+      { name: 'confirmation', weight: 3, evaluator: 'llm_judge', expected: 'Did the response confirm storing the user\'s preference for Rust and functional programming, and accurately summarize what was stored?' },
+    ],
+    timeoutMs: 30_000,
+    tools: ['memory'],
+  },
+  {
+    id: 'planning-blog',
+    name: 'Blog Platform Planning',
+    category: 'planning',
+    description: 'Create a detailed plan for building a blog platform',
+    userMessage: 'Build me a detailed plan for a blog platform with posts, comments, and user authentication. Break it into tasks I can work through.',
+    expectedBehaviors: [
+      'Creates structured tasks or a plan',
+      'Covers posts, comments, and authentication',
+      'Breaks work into manageable pieces',
+    ],
+    scoringCriteria: [
+      { name: 'uses_tasks', weight: 2, evaluator: 'tool_used', expected: 'manage_tasks' },
+      { name: 'mentions_posts', weight: 1, evaluator: 'contains', expected: 'post' },
+      { name: 'mentions_comments', weight: 1, evaluator: 'contains', expected: 'comment' },
+      { name: 'mentions_auth', weight: 1, evaluator: 'regex', expected: 'auth(entication|orization)?' },
+      { name: 'plan_quality', weight: 5, evaluator: 'llm_judge', expected: 'Is the plan well-structured with clear, actionable tasks that cover the three main features (posts, comments, user auth)? Are tasks broken into manageable pieces with logical ordering?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['manage_tasks'],
+  },
+  {
+    id: 'tool-usage-weather',
+    name: 'Web Search - Weather',
+    category: 'tool-usage',
+    description: 'Search the web for current weather information',
+    userMessage: 'Search the web for today\'s weather in London and tell me the temperature and conditions.',
+    expectedBehaviors: [
+      'Uses web search tool',
+      'Reports temperature',
+      'Reports weather conditions',
+    ],
+    scoringCriteria: [
+      { name: 'uses_web_search', weight: 3, evaluator: 'tool_used', expected: 'web_search' },
+      { name: 'mentions_temperature', weight: 2, evaluator: 'regex', expected: '\\d+\\s*[°]?\\s*[CcFf]' },
+      { name: 'mentions_london', weight: 1, evaluator: 'contains', expected: 'London' },
+      { name: 'quality', weight: 4, evaluator: 'llm_judge', expected: 'Did the response provide specific, current weather information for London including temperature and conditions (e.g. sunny, cloudy, rain)?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['web_search'],
+  },
+  {
+    id: 'coding-fizzbuzz',
+    name: 'FizzBuzz Implementation',
+    category: 'coding',
+    description: 'Write and run a FizzBuzz implementation in Python',
+    userMessage: 'Write a FizzBuzz implementation in Python that prints numbers 1 to 30 and run it.',
+    expectedBehaviors: [
+      'Writes correct FizzBuzz logic',
+      'Runs the code successfully',
+      'Output contains Fizz, Buzz, and FizzBuzz',
+    ],
+    scoringCriteria: [
+      { name: 'uses_shell', weight: 2, evaluator: 'tool_used', expected: 'shell' },
+      { name: 'uses_files', weight: 2, evaluator: 'tool_used', expected: 'files' },
+      { name: 'contains_fizz', weight: 1, evaluator: 'contains', expected: 'Fizz' },
+      { name: 'contains_buzz', weight: 1, evaluator: 'contains', expected: 'Buzz' },
+      { name: 'contains_fizzbuzz', weight: 1, evaluator: 'contains', expected: 'FizzBuzz' },
+      { name: 'correctness', weight: 3, evaluator: 'llm_judge', expected: 'Did the response implement FizzBuzz correctly (multiples of 3 print Fizz, multiples of 5 print Buzz, multiples of both print FizzBuzz) and successfully execute it?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['shell', 'files'],
+  },
+  {
+    id: 'research-comparison',
+    name: 'LLM Pricing Comparison',
+    category: 'research',
+    description: 'Compare pricing of major LLM models',
+    userMessage: 'Compare the pricing of OpenAI GPT-4o and Anthropic Claude 3.5 Sonnet. Include input and output token costs.',
+    expectedBehaviors: [
+      'Searches for current pricing',
+      'Includes both models',
+      'Reports input and output token costs',
+    ],
+    scoringCriteria: [
+      { name: 'uses_web_search', weight: 2, evaluator: 'tool_used', expected: 'web_search' },
+      { name: 'mentions_gpt4o', weight: 1, evaluator: 'regex', expected: 'GPT-?4[oO]' },
+      { name: 'mentions_claude', weight: 1, evaluator: 'regex', expected: 'Claude\\s*3\\.?5' },
+      { name: 'mentions_pricing', weight: 1, evaluator: 'regex', expected: '\\$\\d+' },
+      { name: 'quality', weight: 5, evaluator: 'llm_judge', expected: 'Did the response provide accurate and specific pricing for both GPT-4o and Claude 3.5 Sonnet, including input and output token costs, with a clear comparison?' },
+    ],
+    timeoutMs: 90_000,
+    tools: ['web_search', 'web_fetch'],
+  },
+  {
+    id: 'multi-step-analyze',
+    name: 'Package.json Analysis',
+    category: 'multi-step',
+    description: 'Read and analyze the current project\'s package.json',
+    userMessage: 'Read the package.json in the current directory and list all dependencies. Group them into regular dependencies and dev dependencies.',
+    expectedBehaviors: [
+      'Reads package.json using shell or files tool',
+      'Lists regular dependencies',
+      'Lists dev dependencies',
+      'Groups them clearly',
+    ],
+    scoringCriteria: [
+      { name: 'uses_shell_or_files', weight: 2, evaluator: 'tool_used', expected: 'shell' },
+      { name: 'mentions_dependencies', weight: 1, evaluator: 'contains', expected: 'dependencies' },
+      { name: 'mentions_dev_deps', weight: 1, evaluator: 'regex', expected: 'dev[Dd]ependencies|dev dependencies' },
+      { name: 'mentions_package_json', weight: 1, evaluator: 'contains', expected: 'package.json' },
+      { name: 'quality', weight: 5, evaluator: 'llm_judge', expected: 'Did the response successfully read package.json, list the dependencies, and clearly group them into regular and dev dependencies?' },
+    ],
+    timeoutMs: 60_000,
+    tools: ['shell', 'files'],
+  },
+]
+
+export function getScenario(id: string): EvalScenario | undefined {
+  return EVAL_SCENARIOS.find(s => s.id === id)
+}

package/src/lib/server/eval/scorer.ts

@@ -0,0 +1,96 @@
+import type { ScoringCriterion, EvalCriterionResult } from './types'
+import type { MessageToolEvent } from '@/types'
+
+export async function scoreCriteria(
+  criteria: ScoringCriterion[],
+  responseText: string,
+  toolEvents: MessageToolEvent[],
+  judgeOpts?: { provider: string; model: string; apiKey: string | null; apiEndpoint?: string | null },
+): Promise<EvalCriterionResult[]> {
+  const results: EvalCriterionResult[] = []
+
+  for (const criterion of criteria) {
+    switch (criterion.evaluator) {
+      case 'contains': {
+        const found = responseText.toLowerCase().includes(criterion.expected.toLowerCase())
+        results.push({
+          criterion: criterion.name,
+          score: found ? criterion.weight : 0,
+          maxScore: criterion.weight,
+          evidence: found ? `Found "${criterion.expected}" in response` : `"${criterion.expected}" not found in response`,
+        })
+        break
+      }
+
+      case 'regex': {
+        const regex = new RegExp(criterion.expected, 'i')
+        const matched = regex.test(responseText)
+        results.push({
+          criterion: criterion.name,
+          score: matched ? criterion.weight : 0,
+          maxScore: criterion.weight,
+          evidence: matched ? `Pattern /${criterion.expected}/i matched` : `Pattern /${criterion.expected}/i did not match`,
+        })
+        break
+      }
+
+      case 'tool_used': {
+        const used = toolEvents.some(e => e.name === criterion.expected)
+        results.push({
+          criterion: criterion.name,
+          score: used ? criterion.weight : 0,
+          maxScore: criterion.weight,
+          evidence: used ? `Tool "${criterion.expected}" was used` : `Tool "${criterion.expected}" was not used`,
+        })
+        break
+      }
+
+      case 'llm_judge': {
+        if (!judgeOpts) {
+          results.push({
+            criterion: criterion.name,
+            score: 0,
+            maxScore: criterion.weight,
+            evidence: 'No judge provider configured; skipped',
+          })
+          break
+        }
+
+        try {
+          const { buildChatModel } = await import('../build-llm')
+          const { HumanMessage } = await import('@langchain/core/messages')
+
+          const llm = buildChatModel({
+            provider: judgeOpts.provider,
+            model: judgeOpts.model,
+            apiKey: judgeOpts.apiKey,
+            apiEndpoint: judgeOpts.apiEndpoint,
+          })
+
+          const judgePrompt = `Rate the following AI response on a scale of 0-10.\n\nCriterion: ${criterion.expected}\n\nResponse:\n${responseText}\n\nReply with ONLY a number 0-10.`
+          const result = await llm.invoke([new HumanMessage(judgePrompt)])
+          const scoreText = typeof result.content === 'string' ? result.content : ''
+          const parsed = parseInt(scoreText.trim(), 10)
+          const rawScore = Number.isFinite(parsed) ? Math.max(0, Math.min(10, parsed)) : 5
+
+          results.push({
+            criterion: criterion.name,
+            score: (rawScore / 10) * criterion.weight,
+            maxScore: criterion.weight,
+            evidence: `LLM judge: ${rawScore}/10`,
+          })
+        } catch (err: unknown) {
+          results.push({
+            criterion: criterion.name,
+            score: 0,
+            maxScore: criterion.weight,
+            evidence: `LLM judge error: ${err instanceof Error ? err.message : String(err)}`,
+          })
+        }
+        break
+      }
+    }
+  }
+
+  return results
+}

package/src/lib/server/eval/store.ts

@@ -0,0 +1,37 @@
+import Database from 'better-sqlite3'
+import path from 'path'
+import type { EvalRun } from './types'
+
+const DB_PATH = path.join(process.cwd(), 'data', 'eval-runs.db')
+
+let db: Database.Database | null = null
+
+function getDb(): Database.Database {
+  if (!db) {
+    db = new Database(DB_PATH)
+    db.pragma('journal_mode = WAL')
+    db.exec(`CREATE TABLE IF NOT EXISTS eval_runs (
+      id TEXT PRIMARY KEY,
+      data TEXT NOT NULL
+    )`)
+  }
+  return db
+}
+
+export function saveEvalRun(run: EvalRun): void {
+  getDb().prepare('INSERT OR REPLACE INTO eval_runs (id, data) VALUES (?, ?)').run(run.id, JSON.stringify(run))
+}
+
+export function getEvalRun(id: string): EvalRun | null {
+  const row = getDb().prepare('SELECT data FROM eval_runs WHERE id = ?').get(id) as { data: string } | undefined
+  return row ? JSON.parse(row.data) as EvalRun : null
+}
+
+export function listEvalRuns(limit = 50): EvalRun[] {
+  const rows = getDb().prepare('SELECT data FROM eval_runs ORDER BY rowid DESC LIMIT ?').all(limit) as { data: string }[]
+  return rows.map(r => JSON.parse(r.data) as EvalRun)
+}
+
+export function listEvalRunsByAgent(agentId: string, limit = 50): EvalRun[] {
+  return listEvalRuns(limit * 2).filter(r => r.agentId === agentId).slice(0, limit)
+}

package/src/lib/server/eval/types.ts

@@ -0,0 +1,48 @@
+export interface ScoringCriterion {
+  name: string
+  weight: number
+  evaluator: 'contains' | 'regex' | 'tool_used' | 'llm_judge'
+  expected: string
+}
+
+export interface EvalScenario {
+  id: string
+  name: string
+  category: 'coding' | 'research' | 'companionship' | 'multi-step' | 'memory' | 'planning' | 'tool-usage' | 'long-lived'
+  description: string
+  userMessage: string
+  expectedBehaviors: string[]
+  scoringCriteria: ScoringCriterion[]
+  timeoutMs: number
+  tools: string[]
+}
+
+export interface EvalRun {
+  id: string
+  scenarioId: string
+  agentId: string
+  status: 'pending' | 'running' | 'completed' | 'failed'
+  startedAt: number
+  endedAt?: number
+  score: number
+  maxScore: number
+  details: EvalCriterionResult[]
+  sessionId?: string
+  error?: string
+}
+
+export interface EvalCriterionResult {
+  criterion: string
+  score: number
+  maxScore: number
+  evidence?: string
+}
+
+export interface EvalSuiteResult {
+  agentId: string
+  totalScore: number
+  maxScore: number
+  percentage: number
+  runs: EvalRun[]
+  completedAt: number
+}

package/src/lib/server/execution-log.ts

@@ -8,14 +8,18 @@ import { genId } from '@/lib/id'
 // ---------------------------------------------------------------------------
 
 export type LogCategory =
-  | 'trigger'       // what kicked off the action
-  | 'decision'      // reasoning / model choice
-  | 'tool_call'     // tool invocation with input
-  | 'tool_result'   // tool output
-  | 'outbound'      // messages sent to users/platforms
-  | 'file_op'       // file read/write/delete with checksums
-  | 'commit'        // git commit activity
-  | 'error'         // errors during execution
+  | 'trigger'            // what kicked off the action
+  | 'decision'           // reasoning / model choice
+  | 'tool_call'          // tool invocation with input
+  | 'tool_result'        // tool output
+  | 'outbound'           // messages sent to users/platforms
+  | 'file_op'            // file read/write/delete with checksums
+  | 'commit'             // git commit activity
+  | 'error'              // errors during execution
+  | 'mission_start'      // new mission/goal started
+  | 'mission_checkpoint' // periodic mission state snapshot
+  | 'mission_complete'   // mission reached ok status
+  | 'budget_warning'     // mission approaching or exceeding budget
 
 export interface ExecutionLogEntry {
   id: string