@swarmclawai/swarmclaw 0.6.7 → 0.7.0

package/src/lib/server/cost.ts

@@ -1,3 +1,6 @@
+import type { Agent, UsageRecord } from '@/types'
+import { loadSessions, loadUsage } from './storage'
+
 // Model cost table: [inputCostPer1M, outputCostPer1M] in USD
 const MODEL_COSTS: Record<string, [number, number]> = {
   // Anthropic
@@ -11,7 +14,7 @@ const MODEL_COSTS: Record<string, [number, number]> = {
   'gpt-4.1': [2, 8],
   'gpt-4.1-mini': [0.4, 1.6],
   'gpt-4.1-nano': [0.1, 0.4],
-  'o3': [10, 40],
+  o3: [10, 40],
   'o3-mini': [1.1, 4.4],
   'o4-mini': [1.1, 4.4],
   // OpenAI embeddings
@@ -19,6 +22,38 @@ const MODEL_COSTS: Record<string, [number, number]> = {
   'text-embedding-3-large': [0.13, 0],
 }
 
+const ONE_HOUR_MS = 60 * 60 * 1000
+const WARNING_RATIO = 0.8
+
+type GenericRecord = Record<string, unknown>
+type SessionsMap = Record<string, GenericRecord>
+type UsageMap = Record<string, unknown>
+
+function parsePositiveBudget(value: unknown): number | null {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return null
+  return value
+}
+
+function toDateBoundaries(now: number) {
+  const d = new Date(now)
+  const dayStart = new Date(d)
+  dayStart.setHours(0, 0, 0, 0)
+  const monthStart = new Date(d.getFullYear(), d.getMonth(), 1)
+  return {
+    hourStartTs: now - ONE_HOUR_MS,
+    dayStartTs: dayStart.getTime(),
+    monthStartTs: monthStart.getTime(),
+  }
+}
+
+function getAgentSessionIds(agentId: string, sessions: SessionsMap): Set<string> {
+  const ids = new Set<string>()
+  for (const [sid, session] of Object.entries(sessions)) {
+    if (session?.agentId === agentId) ids.add(sid)
+  }
+  return ids
+}
+
 export function estimateCost(model: string, inputTokens: number, outputTokens: number): number {
   const costs = MODEL_COSTS[model]
   if (!costs) return 0
@@ -30,41 +65,141 @@ export function getModelCosts(): Record<string, [number, number]> {
   return { ...MODEL_COSTS }
 }
 
-// --- Agent Monthly Budget ---
-
-import { loadUsage, loadSessions } from './storage'
-import type { Agent, UsageRecord } from '@/types'
+export interface AgentSpendWindows {
+  hourly: number
+  daily: number
+  monthly: number
+}
 
-/**
- * Sum the estimated cost for an agent in the current calendar month.
- * Usage records are keyed by sessionId; we resolve agentId through sessions.
- */
-export function getAgentMonthlySpend(agentId: string): number {
-  const sessions = loadSessions()
-  // Build a set of sessionIds linked to this agent
-  const agentSessionIds = new Set<string>()
-  for (const [sid, session] of Object.entries(sessions)) {
-    if (session?.agentId === agentId) agentSessionIds.add(sid)
+export function getAgentSpendWindows(
+  agentId: string,
+  now = Date.now(),
+  opts?: { sessions?: SessionsMap; usage?: UsageMap },
+): AgentSpendWindows {
+  const sessions = opts?.sessions ?? (loadSessions() as SessionsMap)
+  const usage = opts?.usage ?? (loadUsage() as UsageMap)
+  const agentSessionIds = getAgentSessionIds(agentId, sessions)
+  if (agentSessionIds.size === 0) {
+    return { hourly: 0, daily: 0, monthly: 0 }
   }
-  if (agentSessionIds.size === 0) return 0
 
-  const now = new Date()
-  const monthStart = new Date(now.getFullYear(), now.getMonth(), 1).getTime()
+  const { hourStartTs, dayStartTs, monthStartTs } = toDateBoundaries(now)
+  const spend: AgentSpendWindows = { hourly: 0, daily: 0, monthly: 0 }
 
-  const usage = loadUsage()
-  let total = 0
   for (const sid of agentSessionIds) {
-    const records = usage[sid]
-    if (!Array.isArray(records)) continue
-    for (const record of records) {
+    const raw = usage[sid]
+    if (!Array.isArray(raw)) continue
+    for (const record of raw) {
       const r = record as UsageRecord
-      if (typeof r.timestamp !== 'number' || r.timestamp < monthStart) continue
-      if (typeof r.estimatedCost === 'number' && Number.isFinite(r.estimatedCost) && r.estimatedCost > 0) {
-        total += r.estimatedCost
-      }
+      const ts = typeof r?.timestamp === 'number' ? r.timestamp : 0
+      if (ts <= 0) continue
+      const cost = typeof r?.estimatedCost === 'number' ? r.estimatedCost : 0
+      if (!Number.isFinite(cost) || cost <= 0) continue
+
+      if (ts >= monthStartTs) spend.monthly += cost
+      if (ts >= dayStartTs) spend.daily += cost
+      if (ts >= hourStartTs) spend.hourly += cost
     }
   }
-  return total
+
+  return spend
+}
+
+export function getAgentMonthlySpend(
+  agentId: string,
+  now = Date.now(),
+  opts?: { sessions?: SessionsMap; usage?: UsageMap },
+): number {
+  return getAgentSpendWindows(agentId, now, opts).monthly
+}
+
+export function getAgentDailySpend(
+  agentId: string,
+  now = Date.now(),
+  opts?: { sessions?: SessionsMap; usage?: UsageMap },
+): number {
+  return getAgentSpendWindows(agentId, now, opts).daily
+}
+
+export function getAgentHourlySpend(
+  agentId: string,
+  now = Date.now(),
+  opts?: { sessions?: SessionsMap; usage?: UsageMap },
+): number {
+  return getAgentSpendWindows(agentId, now, opts).hourly
+}
+
+export type AgentBudgetWindow = 'hourly' | 'daily' | 'monthly'
+
+export interface AgentBudgetStatus {
+  window: AgentBudgetWindow
+  spend: number
+  budget: number
+  ratio: number
+  message: string
+}
+
+export interface AgentBudgetCheckSummary {
+  ok: boolean
+  spend: AgentSpendWindows
+  exceeded: AgentBudgetStatus[]
+  warnings: AgentBudgetStatus[]
+}
+
+function budgetWindowLabel(window: AgentBudgetWindow): string {
+  if (window === 'hourly') return 'hourly'
+  if (window === 'daily') return 'daily'
+  return 'monthly'
+}
+
+function buildBudgetStatus(
+  agentName: string,
+  window: AgentBudgetWindow,
+  spend: number,
+  budget: number,
+  exceeded: boolean,
+): AgentBudgetStatus {
+  const ratio = budget > 0 ? spend / budget : 0
+  const label = budgetWindowLabel(window)
+  const message = exceeded
+    ? `Agent "${agentName}" has reached its ${label} budget: $${spend.toFixed(4)} spent of $${budget.toFixed(2)} cap.`
+    : `Agent "${agentName}" is nearing its ${label} budget: $${spend.toFixed(4)} of $${budget.toFixed(2)} (${Math.round(ratio * 100)}%).`
+  return { window, spend, budget, ratio, message }
+}
+
+export function checkAgentBudgetLimits(
+  agent: Agent,
+  now = Date.now(),
+  opts?: { sessions?: SessionsMap; usage?: UsageMap },
+): AgentBudgetCheckSummary {
+  const budgets: Partial<Record<AgentBudgetWindow, number>> = {
+    hourly: parsePositiveBudget(agent.hourlyBudget) ?? undefined,
+    daily: parsePositiveBudget(agent.dailyBudget) ?? undefined,
+    monthly: parsePositiveBudget(agent.monthlyBudget) ?? undefined,
+  }
+  const spend = getAgentSpendWindows(agent.id, now, opts)
+  const exceeded: AgentBudgetStatus[] = []
+  const warnings: AgentBudgetStatus[] = []
+
+  for (const window of ['hourly', 'daily', 'monthly'] as const) {
+    const budget = budgets[window]
+    if (!budget) continue
+    const windowSpend = spend[window]
+    if (windowSpend >= budget) {
+      exceeded.push(buildBudgetStatus(agent.name, window, windowSpend, budget, true))
+      continue
+    }
+    if (windowSpend >= budget * WARNING_RATIO) {
+      warnings.push(buildBudgetStatus(agent.name, window, windowSpend, budget, false))
+    }
+  }
+
+  return {
+    ok: exceeded.length === 0,
+    spend,
+    exceeded,
+    warnings,
+  }
 }
 
 export interface BudgetCheckResult {
@@ -75,14 +210,10 @@ export interface BudgetCheckResult {
 }
 
 /**
- * Check whether an agent is within its monthly budget.
- * Returns ok: true if no budget is set or spend is under the cap.
+ * Backwards-compatible monthly-only budget check.
  */
 export function checkBudget(agent: Agent): BudgetCheckResult {
-  const budget = typeof agent.monthlyBudget === 'number' && Number.isFinite(agent.monthlyBudget) && agent.monthlyBudget > 0
-    ? agent.monthlyBudget
-    : 0
-
+  const budget = parsePositiveBudget(agent.monthlyBudget) ?? 0
   if (budget <= 0) {
     return { ok: true, spend: 0, budget: 0 }
   }

package/src/lib/server/daemon-state.ts

@@ -1,4 +1,4 @@
-import { loadQueue, loadSchedules, loadSessions, saveSessions, loadConnectors, saveConnectors, loadWebhookRetryQueue, upsertWebhookRetry, deleteWebhookRetry, loadWebhooks, loadAgents, appendWebhookLog, loadCredentials, decryptKey } from './storage'
+import { loadQueue, loadSchedules, loadSessions, saveSessions, loadConnectors, saveConnectors, loadWebhookRetryQueue, upsertWebhookRetry, deleteWebhookRetry, loadWebhooks, loadAgents, loadSettings, appendWebhookLog, loadCredentials, decryptKey } from './storage'
 import { notify } from './ws-hub'
 import { processNext, cleanupFinishedTaskSessions, validateCompletedTasksQueue, recoverStalledRunningTasks } from './queue'
 import { startScheduler, stopScheduler } from './scheduler'
@@ -17,9 +17,11 @@ import { hasOpenClawAgents, ensureGatewayConnected, disconnectGateway, getGatewa
 import { enqueueSessionRun } from './session-run-manager'
 import { WORKSPACE_DIR } from './data-dir'
 import { genId } from '@/lib/id'
+import path from 'node:path'
 import type { WebhookRetryEntry } from '@/types'
 import { createNotification } from '@/lib/server/create-notification'
 import { pingProvider, OPENAI_COMPATIBLE_DEFAULTS } from '@/lib/server/provider-health'
+import { runIntegrityMonitor } from '@/lib/server/integrity-monitor'
 
 const QUEUE_CHECK_INTERVAL = 30_000 // 30 seconds
 const BROWSER_SWEEP_INTERVAL = 60_000 // 60 seconds
@@ -80,6 +82,7 @@ const ds: {
   healthIntervalId: ReturnType<typeof setInterval> | null
   memoryConsolidationTimeoutId: ReturnType<typeof setTimeout> | null
   memoryConsolidationIntervalId: ReturnType<typeof setInterval> | null
+  evalSchedulerIntervalId: ReturnType<typeof setInterval> | null
   /** Session IDs we've already alerted as stale (alert-once semantics). */
   staleSessionIds: Set<string>
   connectorRestartState: Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>
@@ -87,6 +90,8 @@ const ds: {
   openclawDownAgentIds: Set<string>
   /** Per-agent auto-repair state for OpenClaw gateways. */
   openclawRepairState: Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>
+  lastIntegrityCheckAt: number | null
+  lastIntegrityDriftCount: number
   manualStopRequested: boolean
   running: boolean
   lastProcessedAt: number | null
@@ -97,10 +102,13 @@ const ds: {
   healthIntervalId: null,
   memoryConsolidationTimeoutId: null,
   memoryConsolidationIntervalId: null,
+  evalSchedulerIntervalId: null,
   staleSessionIds: new Set<string>(),
   connectorRestartState: new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>(),
   openclawDownAgentIds: new Set<string>(),
   openclawRepairState: new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>(),
+  lastIntegrityCheckAt: null,
+  lastIntegrityDriftCount: 0,
   manualStopRequested: false,
   running: false,
   lastProcessedAt: null,
@@ -111,6 +119,8 @@ if (!ds.staleSessionIds) ds.staleSessionIds = new Set<string>()
 if (!ds.connectorRestartState) ds.connectorRestartState = new Map<string, { lastAttemptAt: number; failCount: number; wakeAttempts: number }>()
 if (!ds.openclawDownAgentIds) ds.openclawDownAgentIds = new Set<string>()
 if (!ds.openclawRepairState) ds.openclawRepairState = new Map<string, { attempts: number; lastAttemptAt: number; cooldownUntil: number }>()
+if (ds.lastIntegrityCheckAt === undefined) ds.lastIntegrityCheckAt = null
+if (ds.lastIntegrityDriftCount === undefined) ds.lastIntegrityDriftCount = 0
 // Migrate from old issueLastAlertAt map if present (HMR across code versions)
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 if ((ds as any).issueLastAlertAt) delete (ds as any).issueLastAlertAt
@@ -118,6 +128,7 @@ if (ds.healthIntervalId === undefined) ds.healthIntervalId = null
 if (ds.manualStopRequested === undefined) ds.manualStopRequested = false
 if (ds.memoryConsolidationTimeoutId === undefined) ds.memoryConsolidationTimeoutId = null
 if (ds.memoryConsolidationIntervalId === undefined) ds.memoryConsolidationIntervalId = null
+if (ds.evalSchedulerIntervalId === undefined) ds.evalSchedulerIntervalId = null
 
 export function ensureDaemonStarted(source = 'unknown'): boolean {
   if (ds.running) return false
@@ -140,6 +151,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
     startHealthMonitor()
     startHeartbeatService()
     startMemoryConsolidation()
+    startEvalScheduler()
     return
   }
   ds.running = true
@@ -155,6 +167,7 @@ export function startDaemon(options?: { source?: string; manualStart?: boolean }
     startHealthMonitor()
     startHeartbeatService()
     startMemoryConsolidation()
+    startEvalScheduler()
   } catch (err: unknown) {
     ds.running = false
     notify('daemon')
@@ -182,6 +195,7 @@ export function stopDaemon(options?: { source?: string; manualStop?: boolean })
   stopHealthMonitor()
   stopHeartbeatService()
   stopMemoryConsolidation()
+  stopEvalScheduler()
   stopAllConnectors().catch(() => {})
 }
 
@@ -725,6 +739,35 @@ async function runHealthChecks() {
     console.error('[daemon] OpenClaw gateway health check failed:', err instanceof Error ? err.message : String(err))
   }
 
+  // Integrity drift monitoring for identity/config/plugin files.
+  try {
+    const integrity = runIntegrityMonitor(loadSettings())
+    ds.lastIntegrityCheckAt = integrity.checkedAt
+    ds.lastIntegrityDriftCount = integrity.drifts.length
+    if (integrity.drifts.length > 0) {
+      for (const drift of integrity.drifts) {
+        const rel = path.relative(process.cwd(), drift.filePath)
+        const shortPath = rel && !rel.startsWith('..') ? rel : drift.filePath
+        const action = drift.type === 'created'
+          ? 'created'
+          : drift.type === 'deleted'
+            ? 'deleted'
+            : 'modified'
+        createNotification({
+          type: drift.type === 'deleted' ? 'error' : 'warning',
+          title: `Integrity drift detected (${drift.kind})`,
+          message: `${shortPath} was ${action}.`,
+          dedupKey: `integrity:${drift.id}:${drift.nextHash || 'missing'}`,
+          entityType: 'session',
+          entityId: drift.id,
+        })
+      }
+      await sendHealthAlert(`Integrity monitor detected ${integrity.drifts.length} file drift event(s).`)
+    }
+  } catch (err: unknown) {
+    console.error('[daemon] Integrity monitor check failed:', err instanceof Error ? err.message : String(err))
+  }
+
   // Process webhook retry queue
   try {
     await processWebhookRetries()
@@ -785,6 +828,69 @@ function stopMemoryConsolidation() {
   }
 }
 
+// --- Eval scheduler ---
+
+const EVAL_DEFAULT_INTERVAL_MS = 24 * 3600_000 // 24 hours
+
+function parseCronToMs(cron: string | null | undefined): number | null {
+  if (!cron || typeof cron !== 'string') return null
+  // Simple heuristic: extract hours from common cron patterns like "0 */6 * * *"
+  const hourMatch = cron.match(/\*\/(\d+)/)
+  if (hourMatch) return parseInt(hourMatch[1], 10) * 3600_000
+  return EVAL_DEFAULT_INTERVAL_MS
+}
+
+async function runEvalSchedulerTick() {
+  try {
+    const settings = loadSettings()
+    if (!settings.autonomyEvalEnabled) return
+
+    const { runEvalSuite } = await import('./eval/runner')
+    const agents = loadAgents()
+    const heartbeatAgentIds = Object.keys(agents).filter(
+      (id) => agents[id].heartbeatEnabled === true,
+    )
+
+    for (const agentId of heartbeatAgentIds) {
+      try {
+        const result = await runEvalSuite(agentId)
+        console.log(
+          `[daemon:eval] Agent ${agents[agentId].name}: ${result.percentage}% (${result.totalScore}/${result.maxScore})`,
+        )
+        createNotification({
+          title: `Eval: ${agents[agentId].name} scored ${result.percentage}%`,
+          message: `${result.runs.length} scenarios, ${result.totalScore}/${result.maxScore} points`,
+          type: result.percentage >= 60 ? 'info' : 'warning',
+        })
+      } catch (err: unknown) {
+        console.error(`[daemon:eval] Failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err))
+      }
+    }
+  } catch (err: unknown) {
+    console.error('[daemon:eval] Scheduler tick error:', err instanceof Error ? err.message : String(err))
+  }
+}
+
+function startEvalScheduler() {
+  if (ds.evalSchedulerIntervalId) return
+  try {
+    const settings = loadSettings()
+    if (!settings.autonomyEvalEnabled) return
+    const intervalMs = parseCronToMs(settings.autonomyEvalCron) || EVAL_DEFAULT_INTERVAL_MS
+    ds.evalSchedulerIntervalId = setInterval(runEvalSchedulerTick, intervalMs)
+    console.log(`[daemon:eval] Eval scheduler started (interval=${Math.round(intervalMs / 3600_000)}h)`)
+  } catch {
+    // Eval scheduling is optional — don't block daemon start
+  }
+}
+
+function stopEvalScheduler() {
+  if (ds.evalSchedulerIntervalId) {
+    clearInterval(ds.evalSchedulerIntervalId)
+    ds.evalSchedulerIntervalId = null
+  }
+}
+
 export async function runDaemonHealthCheckNow() {
   await runHealthChecks()
 }
@@ -823,6 +929,11 @@ export function getDaemonStatus() {
       staleSessions: ds.staleSessionIds.size,
       connectorsInBackoff: ds.connectorRestartState.size,
       checkIntervalSec: Math.trunc(HEALTH_CHECK_INTERVAL / 1000),
+      integrity: {
+        enabled: loadSettings().integrityMonitorEnabled !== false,
+        lastCheckedAt: ds.lastIntegrityCheckAt,
+        lastDriftCount: ds.lastIntegrityDriftCount,
+      },
     },
     webhookRetry: {
       pendingRetries,

package/src/lib/server/data-dir.ts

@@ -1,4 +1,21 @@
 import path from 'path'
+import os from 'os'
+import fs from 'fs'
 
 export const DATA_DIR = process.env.DATA_DIR || path.join(process.cwd(), 'data')
-export const WORKSPACE_DIR = path.join(DATA_DIR, 'workspace')
+
+// Workspace lives outside the project directory to avoid triggering Next.js HMR
+// when agents create/modify files. Falls back to data/workspace for Docker/CI.
+function resolveWorkspaceDir(): string {
+  if (process.env.WORKSPACE_DIR) return process.env.WORKSPACE_DIR
+  const external = path.join(os.homedir(), '.swarmclaw', 'workspace')
+  try {
+    fs.mkdirSync(external, { recursive: true })
+    return external
+  } catch {
+    // If we can't create the external dir (permissions, etc.), fall back to in-project
+    return path.join(DATA_DIR, 'workspace')
+  }
+}
+
+export const WORKSPACE_DIR = resolveWorkspaceDir()

package/src/lib/server/eval/runner.ts

@@ -0,0 +1,126 @@
+import { genId } from '@/lib/id'
+import type { EvalScenario, EvalRun, EvalSuiteResult } from './types'
+import { getScenario, EVAL_SCENARIOS } from './scenarios'
+import { scoreCriteria } from './scorer'
+import { saveEvalRun } from './store'
+import { loadSessions, saveSessions, loadAgents, loadCredentials, decryptKey } from '../storage'
+import { executeSessionChatTurn } from '../chat-execution'
+import type { Session } from '@/types'
+
+export async function runEvalScenario(scenarioId: string, agentId: string): Promise<EvalRun> {
+  const scenario = getScenario(scenarioId)
+  if (!scenario) throw new Error(`Unknown eval scenario: ${scenarioId}`)
+
+  const agents = loadAgents() as Record<string, Record<string, unknown>>
+  const agent = agents[agentId]
+  if (!agent) throw new Error(`Unknown agent: ${agentId}`)
+
+  const runId = genId()
+  const sessionId = `eval-${runId}`
+  const now = Date.now()
+
+  const run: EvalRun = {
+    id: runId,
+    scenarioId,
+    agentId,
+    status: 'running',
+    startedAt: now,
+    score: 0,
+    maxScore: scenario.scoringCriteria.reduce((sum, c) => sum + c.weight, 0),
+    details: [],
+    sessionId,
+  }
+
+  // Create temporary eval session
+  const sessions = loadSessions() as Record<string, Session>
+  const evalSession: Session = {
+    id: sessionId,
+    name: `Eval: ${scenario.name}`,
+    cwd: process.cwd(),
+    user: 'eval-runner',
+    provider: (agent.provider as Session['provider']) ?? 'anthropic',
+    model: (agent.model as string) ?? '',
+    credentialId: (agent.credentialId as string | null) ?? null,
+    apiEndpoint: (agent.apiEndpoint as string | null) ?? null,
+    claudeSessionId: null,
+    agentId,
+    tools: scenario.tools,
+    messages: [],
+    createdAt: now,
+    lastActiveAt: now,
+  }
+  sessions[sessionId] = evalSession
+  saveSessions(sessions)
+
+  try {
+    const result = await executeSessionChatTurn({
+      sessionId,
+      message: scenario.userMessage,
+      internal: true,
+      source: 'eval',
+    })
+
+    const judgeProvider = typeof agent.provider === 'string' ? agent.provider : undefined
+    const judgeModel = typeof agent.model === 'string' ? agent.model : undefined
+    let judgeApiKey: string | null = null
+    if (typeof agent.credentialId === 'string' && agent.credentialId) {
+      const creds = loadCredentials()
+      const cred = creds[agent.credentialId]
+      if (cred) {
+        try { judgeApiKey = decryptKey(cred.encryptedKey) } catch { /* skip undecryptable */ }
+      }
+    }
+    const judgeOpts = judgeProvider && judgeModel ? {
+      provider: judgeProvider,
+      model: judgeModel,
+      apiKey: judgeApiKey,
+      apiEndpoint: typeof agent.apiEndpoint === 'string' ? agent.apiEndpoint : undefined,
+    } : undefined
+
+    run.details = await scoreCriteria(
+      scenario.scoringCriteria,
+      result.text,
+      result.toolEvents || [],
+      judgeOpts,
+    )
+    run.score = run.details.reduce((sum, d) => sum + d.score, 0)
+    run.status = 'completed'
+    run.endedAt = Date.now()
+  } catch (err: unknown) {
+    run.status = 'failed'
+    run.error = err instanceof Error ? err.message : String(err)
+    run.endedAt = Date.now()
+  } finally {
+    // Clean up eval session
+    const currentSessions = loadSessions() as Record<string, Session>
+    delete currentSessions[sessionId]
+    saveSessions(currentSessions)
+  }
+
+  saveEvalRun(run)
+  return run
+}
+
+export async function runEvalSuite(agentId: string, categories?: string[]): Promise<EvalSuiteResult> {
+  const scenarios: EvalScenario[] = categories
+    ? EVAL_SCENARIOS.filter(s => categories.includes(s.category))
+    : EVAL_SCENARIOS
+
+  const runs: EvalRun[] = []
+  for (const scenario of scenarios) {
+    const evalRun = await runEvalScenario(scenario.id, agentId)
+    runs.push(evalRun)
+  }
+
+  const totalScore = runs.reduce((sum, r) => sum + r.score, 0)
+  const maxScore = runs.reduce((sum, r) => sum + r.maxScore, 0)
+
+  return {
+    agentId,
+    totalScore,
+    maxScore,
+    percentage: maxScore > 0 ? Math.round((totalScore / maxScore) * 100) : 0,
+    runs,
+    completedAt: Date.now(),
+  }
+}