@winspan/claude-forge 8.13.1 → 8.16.0

package/dist/web/server.js

@@ -7,8 +7,12 @@ import express from 'express';
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { homedir } from 'os';
+import yaml from 'js-yaml';
+import { Recommender } from '../engine/recommender.js';
 import { logger } from '../core/utils/logger.js';
 import { ErrorHandler } from '../core/utils/error-handler.js';
+import { ConfigManager } from '../core/config.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 export class WebServer {
     options;
@@ -364,6 +368,553 @@ export class WebServer {
                 dailyActivity,
             });
         });
+        // ── Agent Routing API ─────────────────────────────────────────────────
+        // Overview stats for the Agent Routing page
+        this.app.get('/api/routing/stats', (req, res) => {
+            const windowHours = parseInt(req.query.window || '168'); // default 7d
+            const since = Date.now() - windowHours * 3600 * 1000;
+            const events = storage.queryRoutingEvents({ since_ts: since, limit: 5000 });
+            const total = events.length;
+            const byAgent = {};
+            let forced = 0;
+            let obeyedCount = 0;
+            let refusedCount = 0;
+            let unknownCount = 0;
+            let fallbackUsedCount = 0;
+            const latencies = [];
+            const byVersion = {};
+            for (const e of events) {
+                if (e.is_forced)
+                    forced++;
+                if (e.fallback_used)
+                    fallbackUsedCount++;
+                if (typeof e.classification_ms === 'number')
+                    latencies.push(e.classification_ms);
+                const key = e.routed_to_name ?? '—';
+                const bucket = (byAgent[key] ||= { total: 0, obeyed: 0, refused: 0, unknown: 0 });
+                bucket.total++;
+                if (e.obeyed === 1) {
+                    bucket.obeyed++;
+                    obeyedCount++;
+                }
+                else if (e.obeyed === 0) {
+                    bucket.refused++;
+                    refusedCount++;
+                }
+                else {
+                    bucket.unknown++;
+                    unknownCount++;
+                }
+                const v = e.injection_version ?? '—';
+                const vb = (byVersion[v] ||= { total: 0, obeyed: 0 });
+                vb.total++;
+                if (e.obeyed === 1)
+                    vb.obeyed++;
+            }
+            latencies.sort((a, b) => a - b);
+            const p = (pct) => latencies.length === 0 ? null : latencies[Math.min(latencies.length - 1, Math.floor(latencies.length * pct))];
+            res.json({
+                windowHours,
+                total,
+                forced,
+                obeyedCount,
+                refusedCount,
+                unknownCount,
+                obedienceRate: forced === 0 ? null : obeyedCount / forced,
+                refusalRate: forced === 0 ? null : refusedCount / forced,
+                fallbackUsedCount,
+                fallbackRate: total === 0 ? null : fallbackUsedCount / total,
+                latency: {
+                    p50: p(0.5),
+                    p95: p(0.95),
+                    p99: p(0.99),
+                    count: latencies.length,
+                },
+                byAgent,
+                byVersion,
+            });
+        });
+        // Recent routing events (timeline / detail)
+        this.app.get('/api/routing/events', (req, res) => {
+            const limit = parseInt(req.query.limit || '50');
+            const sessionId = req.query.session;
+            const projectPath = req.query.project;
+            const agent = req.query.agent;
+            const obeyedParam = req.query.obeyed;
+            const filter = { limit };
+            if (sessionId)
+                filter.session_id = sessionId;
+            if (projectPath)
+                filter.project_path = projectPath;
+            if (agent)
+                filter.routed_to_name = agent;
+            if (obeyedParam === 'null')
+                filter.obeyed = null;
+            else if (obeyedParam === '0')
+                filter.obeyed = 0;
+            else if (obeyedParam === '1')
+                filter.obeyed = 1;
+            const rows = storage.queryRoutingEvents(filter);
+            res.json(rows);
+        });
+        // Refusal clustering (Plan A: SQL GROUP BY by taskType × agent)
+        this.app.get('/api/routing/refusals', (req, res) => {
+            const windowHours = parseInt(req.query.window || '168');
+            const since = Date.now() - windowHours * 3600 * 1000;
+            const events = storage.queryRoutingEvents({
+                since_ts: since,
+                obeyed: 0,
+                limit: 1000,
+            });
+            // Group by (taskType, routed_to_name)
+            const groups = new Map();
+            for (const e of events) {
+                let taskType = 'unknown';
+                try {
+                    const parsed = JSON.parse(e.intent_json ?? '{}');
+                    if (typeof parsed.taskType === 'string')
+                        taskType = parsed.taskType;
+                }
+                catch { /* ignore */ }
+                const key = `${taskType}__${e.routed_to_name ?? '—'}`;
+                const g = groups.get(key) ?? {
+                    taskType,
+                    agent: e.routed_to_name ?? '—',
+                    count: 0,
+                    samples: [],
+                };
+                g.count++;
+                if (g.samples.length < 5) {
+                    g.samples.push({
+                        prompt: e.prompt.slice(0, 200),
+                        refusal_reason: e.refusal_reason ?? null,
+                        ts: e.ts,
+                    });
+                }
+                groups.set(key, g);
+            }
+            const sorted = Array.from(groups.values()).sort((a, b) => b.count - a.count);
+            res.json({ windowHours, groups: sorted });
+        });
+        // Routing violations analysis (Phase 3 Feature 3)
+        this.app.get('/api/routing/violations', (req, res) => {
+            const windowHours = parseInt(req.query.window || '168'); // default 7d
+            const since = Date.now() - windowHours * 3600 * 1000;
+            const events = storage.queryRoutingEvents({ since_ts: since, limit: 5000 });
+            // Analyze violation patterns: consecutive refusals for same (taskType, agent)
+            const patterns = new Map();
+            for (const e of events) {
+                if (!e.is_forced || e.obeyed === null)
+                    continue; // only analyze forced routes with judgement
+                let taskType = 'unknown';
+                try {
+                    const parsed = JSON.parse(e.intent_json ?? '{}');
+                    if (typeof parsed.taskType === 'string')
+                        taskType = parsed.taskType;
+                }
+                catch { /* ignore */ }
+                const key = `${taskType}__${e.routed_to_name ?? '—'}`;
+                const p = patterns.get(key) ?? {
+                    taskType,
+                    agent: e.routed_to_name ?? '—',
+                    totalAttempts: 0,
+                    refusals: 0,
+                    refusalRate: 0,
+                    recentRefusals: 0,
+                    severity: 'low',
+                    samples: [],
+                };
+                p.totalAttempts++;
+                if (e.obeyed === 0) {
+                    p.refusals++;
+                    if (p.samples.length < 5) {
+                        p.samples.push({
+                            prompt: e.prompt.slice(0, 200),
+                            refusal_reason: e.refusal_reason ?? null,
+                            ts: e.ts,
+                        });
+                    }
+                }
+                patterns.set(key, p);
+            }
+            // Calculate metrics and severity
+            const violations = Array.from(patterns.values())
+                .map(p => {
+                p.refusalRate = p.totalAttempts === 0 ? 0 : p.refusals / p.totalAttempts;
+                // Calculate recent refusals (last 5 attempts for this pattern)
+                const recentEvents = events
+                    .filter(e => {
+                    let taskType = 'unknown';
+                    try {
+                        const parsed = JSON.parse(e.intent_json ?? '{}');
+                        if (typeof parsed.taskType === 'string')
+                            taskType = parsed.taskType;
+                    }
+                    catch { /* ignore */ }
+                    return taskType === p.taskType && e.routed_to_name === p.agent;
+                })
+                    .slice(0, 5);
+                p.recentRefusals = recentEvents.filter(e => e.obeyed === 0).length;
+                // Determine severity
+                if (p.refusalRate >= 0.8 && p.totalAttempts >= 5)
+                    p.severity = 'critical';
+                else if (p.refusalRate >= 0.6 && p.totalAttempts >= 3)
+                    p.severity = 'high';
+                else if (p.refusalRate >= 0.4)
+                    p.severity = 'medium';
+                else
+                    p.severity = 'low';
+                return p;
+            })
+                .filter(p => p.refusals > 0) // only show patterns with at least 1 refusal
+                .sort((a, b) => {
+                // Sort by severity, then refusal rate
+                const severityOrder = { critical: 4, high: 3, medium: 2, low: 1 };
+                if (severityOrder[a.severity] !== severityOrder[b.severity]) {
+                    return severityOrder[b.severity] - severityOrder[a.severity];
+                }
+                return b.refusalRate - a.refusalRate;
+            });
+            res.json({ windowHours, violations });
+        });
+        // Routing config editor API (Phase 3 Feature 2)
+        this.app.get('/api/routing/config', (_req, res) => {
+            const userPath = path.join(homedir(), '.claude-forge', 'routing.yaml');
+            const defaultPath = path.join(__dirname, 'engine', 'conventions', 'routing.yaml');
+            let content = '';
+            let source = 'none';
+            if (fs.existsSync(userPath)) {
+                content = fs.readFileSync(userPath, 'utf-8');
+                source = 'user';
+            }
+            else if (fs.existsSync(defaultPath)) {
+                content = fs.readFileSync(defaultPath, 'utf-8');
+                source = 'default';
+            }
+            res.json({ content, source, userPath, defaultPath });
+        });
+        this.app.put('/api/routing/config', (req, res) => {
+            const { content } = req.body;
+            if (typeof content !== 'string') {
+                res.status(400).json({ error: 'content must be a string' });
+                return;
+            }
+            const userPath = path.join(homedir(), '.claude-forge', 'routing.yaml');
+            const dir = path.dirname(userPath);
+            try {
+                // Validate YAML syntax before saving
+                yaml.load(content);
+                // Ensure directory exists
+                if (!fs.existsSync(dir)) {
+                    fs.mkdirSync(dir, { recursive: true });
+                }
+                // Write to user override path
+                fs.writeFileSync(userPath, content, 'utf-8');
+                logger.info(`[Web] Routing config updated: ${userPath}`);
+                res.json({ success: true, path: userPath });
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to save routing config: ${err}`);
+                res.status(400).json({ error: String(err) });
+            }
+        });
+        // ── Phase 5: A/B Testing APIs ────────────────────────────────────────
+        const experimentsPath = path.join(homedir(), '.claude-forge', 'routing-experiments.yaml');
+        this.app.get('/api/routing/experiments/config', (_req, res) => {
+            let content = '';
+            let source = 'none';
+            if (fs.existsSync(experimentsPath)) {
+                content = fs.readFileSync(experimentsPath, 'utf-8');
+                source = 'user';
+            }
+            res.json({ content, source, path: experimentsPath });
+        });
+        this.app.put('/api/routing/experiments/config', async (req, res) => {
+            const { content } = req.body;
+            if (typeof content !== 'string') {
+                res.status(400).json({ error: 'content must be a string' });
+                return;
+            }
+            try {
+                const parsed = yaml.load(content, { schema: yaml.CORE_SCHEMA });
+                // Reuse validateConfig for structural enforcement (weights, groups, etc.).
+                const { validateConfig } = await import('../engine/experiment-router.js');
+                const cfg = validateConfig(parsed);
+                if (!cfg) {
+                    res.status(400).json({ error: 'experiments YAML failed validation (see daemon logs)' });
+                    return;
+                }
+                const dir = path.dirname(experimentsPath);
+                if (!fs.existsSync(dir))
+                    fs.mkdirSync(dir, { recursive: true });
+                fs.writeFileSync(experimentsPath, content, 'utf-8');
+                logger.info(`[Web] Experiments config updated: ${experimentsPath}`);
+                res.json({ success: true, path: experimentsPath });
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to save experiments config: ${err}`);
+                res.status(400).json({ error: String(err) });
+            }
+        });
+        this.app.get('/api/routing/experiments/analysis', async (_req, res) => {
+            try {
+                if (!fs.existsSync(experimentsPath)) {
+                    res.json({ enabled: false, experimentId: null, groups: [] });
+                    return;
+                }
+                const raw = fs.readFileSync(experimentsPath, 'utf-8');
+                const parsed = yaml.load(raw, { schema: yaml.CORE_SCHEMA });
+                const { validateConfig } = await import('../engine/experiment-router.js');
+                const cfg = validateConfig(parsed);
+                if (!cfg || !cfg.experiment) {
+                    res.json({ enabled: cfg?.enabled ?? false, experimentId: null, groups: [] });
+                    return;
+                }
+                const stats = storage.queryExperimentStats(cfg.experiment.id);
+                const groups = cfg.experiment.groups.map(g => {
+                    const s = stats.find(row => row.group_id === g.id);
+                    const total = s?.total ?? 0;
+                    const obeyed = s?.obeyed ?? 0;
+                    const refused = s?.refused ?? 0;
+                    return {
+                        id: g.id,
+                        name: g.name,
+                        weight: g.weight,
+                        total,
+                        obeyed,
+                        refused,
+                        unknown: s?.unknown ?? 0,
+                        obeyedRate: total > 0 ? obeyed / total : null,
+                        avgClassificationMs: s?.avg_classification_ms ?? null,
+                    };
+                });
+                // Simple z-test between the two groups with the largest samples.
+                let zScore = null;
+                let sampleAdequate = false;
+                let suggestedWinner = null;
+                if (groups.length >= 2) {
+                    const sorted = [...groups].sort((a, b) => b.total - a.total).slice(0, 2);
+                    const [g1, g2] = sorted;
+                    sampleAdequate = g1.total >= 50 && g2.total >= 50;
+                    if (sampleAdequate) {
+                        const p1 = g1.obeyed / g1.total;
+                        const p2 = g2.obeyed / g2.total;
+                        const pPool = (g1.obeyed + g2.obeyed) / (g1.total + g2.total);
+                        const se = Math.sqrt(pPool * (1 - pPool) * (1 / g1.total + 1 / g2.total));
+                        zScore = se > 0 ? (p1 - p2) / se : 0;
+                        if (Math.abs(zScore) > 1.96) {
+                            suggestedWinner = p1 > p2 ? g1.id : g2.id;
+                        }
+                    }
+                }
+                res.json({
+                    enabled: cfg.enabled,
+                    experimentId: cfg.experiment.id,
+                    experimentName: cfg.experiment.name,
+                    startedAt: cfg.experiment.startedAt,
+                    endedAt: cfg.experiment.endedAt,
+                    groups,
+                    zScore,
+                    sampleAdequate,
+                    suggestedWinner,
+                });
+            }
+            catch (err) {
+                logger.warn(`[Web] Experiments analysis failed: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
+        this.app.post('/api/routing/experiments/promote', async (req, res) => {
+            const { groupId } = req.body;
+            if (typeof groupId !== 'string' || groupId.length === 0) {
+                res.status(400).json({ error: 'groupId is required' });
+                return;
+            }
+            try {
+                if (!fs.existsSync(experimentsPath)) {
+                    res.status(400).json({ error: 'experiments config does not exist' });
+                    return;
+                }
+                const raw = fs.readFileSync(experimentsPath, 'utf-8');
+                const parsed = yaml.load(raw, { schema: yaml.CORE_SCHEMA });
+                const { validateConfig } = await import('../engine/experiment-router.js');
+                const cfg = validateConfig(parsed);
+                if (!cfg || !cfg.experiment) {
+                    res.status(400).json({ error: 'experiments config has no active experiment' });
+                    return;
+                }
+                const group = cfg.experiment.groups.find(g => g.id === groupId);
+                if (!group) {
+                    res.status(404).json({ error: `group '${groupId}' not found` });
+                    return;
+                }
+                // Step 1: backup existing routing.yaml if present
+                const routingPath = path.join(homedir(), '.claude-forge', 'routing.yaml');
+                const ts = new Date().toISOString().replace(/[:.]/g, '-');
+                let backupPath = null;
+                if (fs.existsSync(routingPath)) {
+                    backupPath = `${routingPath}.bak-${ts}`;
+                    fs.copyFileSync(routingPath, backupPath);
+                }
+                else {
+                    const dir = path.dirname(routingPath);
+                    if (!fs.existsSync(dir))
+                        fs.mkdirSync(dir, { recursive: true });
+                }
+                // Step 2: write the winner's rules as the new routing.yaml
+                const newRouting = yaml.dump({ schemaVersion: '1.0', rules: group.rules });
+                fs.writeFileSync(routingPath, newRouting, 'utf-8');
+                // Step 3: mark experiment as ended (enabled=false, endedAt=now)
+                const endedAt = new Date().toISOString();
+                const updated = yaml.dump({
+                    schemaVersion: '1.0',
+                    enabled: false,
+                    experiment: {
+                        ...cfg.experiment,
+                        endedAt,
+                    },
+                });
+                fs.writeFileSync(experimentsPath, updated, 'utf-8');
+                logger.info(`[Web] Promoted group '${groupId}' from experiment '${cfg.experiment.id}'; backup: ${backupPath}`);
+                res.json({ promoted: groupId, routingPath, backupPath, endedAt });
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to promote experiment group: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
+        // ── Phase 5 Feature 2: Rule States (auto-disable) ────────────────────
+        this.app.get('/api/routing/rule-states', (req, res) => {
+            const disabledOnly = req.query.disabled === '1' || req.query.disabled === 'true';
+            const rows = storage.listRuleStates({ disabledOnly });
+            res.json({ ruleStates: rows });
+        });
+        this.app.put('/api/routing/rule-states', (req, res) => {
+            const { taskType, agent, disabled, reason } = req.body ?? {};
+            if (typeof taskType !== 'string' || taskType.length === 0) {
+                res.status(400).json({ error: 'taskType is required' });
+                return;
+            }
+            if (typeof agent !== 'string' || agent.length === 0) {
+                res.status(400).json({ error: 'agent is required' });
+                return;
+            }
+            if (typeof disabled !== 'boolean') {
+                res.status(400).json({ error: 'disabled must be boolean' });
+                return;
+            }
+            try {
+                storage.setRuleState({
+                    taskType,
+                    agent,
+                    disabled,
+                    reason: typeof reason === 'string' ? reason : null,
+                    autoDisabled: false, // manual toggle
+                });
+                logger.info(`[Web] Rule state updated: ${taskType}__${agent} disabled=${disabled}`);
+                res.json({ success: true });
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to set rule state: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
+        // ── Phase 5 Feature 3: Rule Recommendations ──────────────────────────
+        this.app.get('/api/routing/recommendations', (req, res) => {
+            const { router, agents } = this.options;
+            if (!router || !agents) {
+                res.json({ recommendations: [], reason: 'router/agents not injected' });
+                return;
+            }
+            const windowDays = Math.max(1, Math.min(90, parseInt(req.query.days || '7')));
+            try {
+                const recommender = new Recommender(storage, router, agents, {
+                    windowMs: windowDays * 24 * 3600 * 1000,
+                });
+                const recommendations = recommender.analyze();
+                res.json({ windowDays, recommendations });
+            }
+            catch (err) {
+                logger.warn(`[Web] Recommender failed: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
+        // ── AI Configuration APIs ─────────────────────────────────────────────
+        // GET /api/config/ai — read current AI config (mask apiKey)
+        this.app.get('/api/config/ai', (_req, res) => {
+            const configManager = new ConfigManager();
+            const config = configManager.get();
+            const maskApiKey = (key) => {
+                if (!key || key.length < 10)
+                    return '***';
+                return key.slice(0, 6) + '***' + key.slice(-4);
+            };
+            res.json({
+                api_key: maskApiKey(config.distill.api_key),
+                base_url: config.distill.base_url || '',
+                model: config.distill.model,
+                provider: config.distill.provider,
+            });
+        });
+        // PUT /api/config/ai — update AI config
+        this.app.put('/api/config/ai', (req, res) => {
+            const { api_key, base_url, model, provider } = req.body ?? {};
+            try {
+                const configManager = new ConfigManager();
+                const config = configManager.get();
+                // Update only provided fields
+                if (typeof api_key === 'string')
+                    config.distill.api_key = api_key;
+                if (typeof base_url === 'string')
+                    config.distill.base_url = base_url;
+                if (typeof model === 'string')
+                    config.distill.model = model;
+                if (typeof provider === 'string')
+                    config.distill.provider = provider;
+                configManager.save();
+                logger.info('[Web] AI config updated');
+                res.json({ success: true });
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to update AI config: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
+        // GET /api/ai/models — proxy to upstream /v1/models
+        this.app.get('/api/ai/models', async (_req, res) => {
+            try {
+                const configManager = new ConfigManager();
+                const config = configManager.get();
+                const apiKey = config.distill.api_key;
+                const baseUrl = config.distill.base_url || 'https://api.anthropic.com';
+                if (!apiKey) {
+                    res.status(400).json({ error: 'API key not configured' });
+                    return;
+                }
+                // Construct models endpoint URL
+                const modelsUrl = baseUrl.endsWith('/v1')
+                    ? `${baseUrl}/models`
+                    : `${baseUrl}/v1/models`;
+                const response = await fetch(modelsUrl, {
+                    headers: {
+                        'Authorization': `Bearer ${apiKey}`,
+                        'x-api-key': apiKey,
+                    },
+                });
+                if (!response.ok) {
+                    const text = await response.text();
+                    logger.warn(`[Web] Upstream /v1/models failed: ${response.status} ${text}`);
+                    res.status(response.status).json({ error: text });
+                    return;
+                }
+                const data = await response.json();
+                res.json(data);
+            }
+            catch (err) {
+                logger.warn(`[Web] Failed to fetch models: ${err}`);
+                res.status(500).json({ error: String(err) });
+            }
+        });
         // SSE: real-time event stream
         this.app.get('/api/events/stream', (req, res) => {
             res.writeHead(200, {