@rulemetric/cli 0.2.0 → 0.2.2

package/dist/lib/eval-benchmark.js

@@ -1,92 +1,12 @@
-import { apiGet, apiPost } from './api-client.js';
-function computeStats(values) {
-    if (values.length === 0)
-        return { mean: 0, stddev: 0, min: 0, max: 0 };
-    const mean = values.reduce((a, b) => a + b, 0) / values.length;
-    const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
-    return {
-        mean: Math.round(mean * 1000) / 1000,
-        stddev: Math.round(Math.sqrt(variance) * 1000) / 1000,
-        min: Math.min(...values),
-        max: Math.max(...values),
-    };
-}
-export async function aggregateBenchmark(targetId, iteration) {
-    const query = iteration ? `?iteration=${iteration}` : '';
-    const runs = await apiGet(`/api/eval-targets/${targetId}/runs${query}`);
-    const completedRuns = runs.filter((r) => r.status === 'completed');
-    if (completedRuns.length === 0) {
-        return { runSummary: {}, notes: ['No completed runs found'], totalRuns: 0 };
-    }
-    // Group by configuration
-    const byConfig = {};
-    for (const run of completedRuns) {
-        if (!byConfig[run.configuration])
-            byConfig[run.configuration] = [];
-        byConfig[run.configuration].push(run);
-    }
-    const runSummary = {};
-    const notes = [];
-    for (const [config, configRuns] of Object.entries(byConfig)) {
-        const passRates = configRuns
-            .filter((r) => Object.keys(r.grading).length > 0)
-            .map((r) => r.grading.summary?.pass_rate ?? 0);
-        const durations = configRuns
-            .map((r) => r.timing.duration_ms / 1000)
-            .filter((v) => !isNaN(v));
-        const tokens = configRuns
-            .map((r) => r.timing.total_tokens ?? 0)
-            .filter((v) => v > 0);
-        runSummary[config] = {};
-        if (passRates.length > 0)
-            runSummary[config].pass_rate = computeStats(passRates);
-        if (durations.length > 0)
-            runSummary[config].time_seconds = computeStats(durations);
-        if (tokens.length > 0)
-            runSummary[config].tokens = computeStats(tokens);
-    }
-    // Compute delta if both configurations exist
-    if (runSummary.with_target && runSummary.without_target) {
-        const delta = {};
-        for (const metric of ['pass_rate', 'time_seconds', 'tokens']) {
-            const withVal = runSummary.with_target[metric]?.mean ?? 0;
-            const withoutVal = runSummary.without_target[metric]?.mean ?? 0;
-            const diff = withVal - withoutVal;
-            delta[metric] = `${diff >= 0 ? '+' : ''}${Math.round(diff * 1000) / 1000}`;
-        }
-        runSummary.delta = delta;
-        // Generate analysis notes
-        const passRateDelta = (runSummary.with_target.pass_rate?.mean ?? 0) - (runSummary.without_target.pass_rate?.mean ?? 0);
-        if (passRateDelta > 0.1) {
-            notes.push(`Target improves pass rate by ${(passRateDelta * 100).toFixed(0)}%`);
-        }
-        else if (passRateDelta < -0.1) {
-            notes.push(`Target decreases pass rate by ${(Math.abs(passRateDelta) * 100).toFixed(0)}%`);
-        }
-        else {
-            notes.push('Target has minimal impact on pass rate');
-        }
-        const timeDelta = (runSummary.with_target.time_seconds?.mean ?? 0) - (runSummary.without_target.time_seconds?.mean ?? 0);
-        if (Math.abs(timeDelta) > 5) {
-            notes.push(`Target ${timeDelta > 0 ? 'adds' : 'saves'} ${Math.abs(timeDelta).toFixed(1)}s average execution time`);
-        }
-    }
-    // Check for high variance
-    for (const [config, stats] of Object.entries(runSummary)) {
-        if (config === 'delta')
-            continue;
-        const passRate = stats.pass_rate;
-        if (passRate && passRate.stddev > 0.2) {
-            notes.push(`${config} shows high pass rate variance (stddev=${passRate.stddev.toFixed(2)}) — results may be unreliable`);
-        }
-    }
-    return { runSummary, notes, totalRuns: completedRuns.length };
-}
-export async function postBenchmark(targetId, iteration, result) {
-    return apiPost(`/api/eval-targets/${targetId}/benchmarks`, {
-        iteration,
-        runSummary: result.runSummary,
-        notes: result.notes,
-    });
-}
-//# sourceMappingURL=eval-benchmark.js.map
+import {
+  aggregateBenchmark,
+  postBenchmark
+} from "../chunk-PQU7GZ7A.js";
+import "../chunk-EJOI2ZGU.js";
+import "../chunk-KZRFUXGF.js";
+import "../chunk-NSBPE2FW.js";
+export {
+  aggregateBenchmark,
+  postBenchmark
+};
+//# sourceMappingURL=eval-benchmark.js.map

package/dist/lib/eval-benchmark.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"eval-benchmark.js","sourceRoot":"","sources":["../../src/lib/eval-benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAwBlD,SAAS,YAAY,CAAC,MAAgB;IACpC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACrF,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;QACpC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QACrD,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QACxB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;KACzB,CAAC;AACJ,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAkB;IAElB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAY,qBAAqB,QAAQ,QAAQ,KAAK,EAAE,CAAC,CAAC;IAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;IAEnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAC9E,CAAC;IAED,yBAAyB;IACzB,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;QACnE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,UAAU,GAA0C,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,OAAe,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,UAAU;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,MAAc,CAAC,WAAW,GAAG,IAAI,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,MAAc,CAAC,YAAY,IAAI,CAAC,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACjF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACpF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;QACxD,MAAM,KAAK,GAA2B,EAAE,CAAC;QACzC,KAAK,MAAM,MAAM,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,OAAO,GAAG,UAAU,CAAC;YAClC,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QAC7E,CAAC;QACA,UAAkB,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,0BAA0B;QAC1B,MAAM,aAAa,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACvH,IAAI,aAAa,GAAG,GAAG,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IAAI,aAAa,GAAG,CAAC,GAAG,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACzH,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;QACrH,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACzD,IAAI,MAAM,KAAK,OAAO;YAAE,SAAS;QACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,0CAA0C,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC;QAC3H,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,SAAiB,EACjB,MAAuB;IAEvB,OAAO,OAAO,CAAiB,qBAAqB,QAAQ,aAAa,EAAE;QACzE,SAAS;QACT,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CAAC;AACL,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/eval-conversation-handler.js

@@ -1,200 +1,13 @@
-import { apiGet, apiPost } from './api-client.js';
-import { executeEval } from './eval-executor.js';
-import { gradeEvalOutput } from './eval-grader.js';
-import { analyzeEvalResults } from './eval-analyzer.js';
-export async function processConversationJob(jobConfig, evalTargetId, logger) {
-    const { conversationId, messageId, messageContent } = jobConfig;
-    // Fetch context
-    const target = await apiGet(`/api/eval-targets/${evalTargetId}`);
-    const evalCases = await apiGet(`/api/eval-targets/${evalTargetId}/evals`);
-    const messages = await apiGet(`/api/eval-conversations/${conversationId}/messages`);
-    // Idempotency: check if this message was already processed (has assistant responses after it)
-    const msgIdx = messages.findIndex((m) => m.id === messageId);
-    if (msgIdx >= 0) {
-        const hasResponse = messages.slice(msgIdx + 1).some((m) => m.role === 'assistant' && (m.type === 'results' || m.type === 'proposal' || m.type === 'generated' || m.type === 'text'));
-        if (hasResponse) {
-            logger(`Message ${messageId} already processed — skipping (idempotency guard)`);
-            return;
-        }
-    }
-    // Helper to post assistant messages
-    const postMessage = async (content, type, data) => {
-        await apiPost(`/api/eval-conversations/${conversationId}/messages/assistant`, {
-            content,
-            type,
-            data,
-        });
-    };
-    // Decide what action to take
-    const decision = await decideAction(target, evalCases, messages, messageContent, logger);
-    logger(`Action: ${decision.action}`);
-    switch (decision.action) {
-        case 'run_evals':
-            await handleRunEvals(target, evalCases, conversationId, postMessage, logger);
-            break;
-        case 'generate_cases':
-            await handleGenerateCases(target, evalTargetId, conversationId, postMessage, logger);
-            break;
-        case 'analyze_results':
-        case 'propose_changes':
-            await handleAnalyzeAndPropose(target, evalCases, messages, messageContent, conversationId, postMessage, logger);
-            break;
-        case 'respond':
-            await postMessage(decision.response, 'text');
-            break;
-    }
-}
-async function decideAction(target, evalCases, messages, latestMessage, logger) {
-    // Build conversation summary for context
-    const recentMessages = messages.slice(-10).map((m) => `${m.role}: ${m.content.slice(0, 200)}`).join('\n');
-    const prompt = `You are an eval assistant. Decide what action to take based on the user's message.
-
-## Current State
-- Eval target: "${target.name}" v${target.version} (${target.type})
-- Eval cases: ${evalCases.length} defined${evalCases.length > 0 ? ` (${evalCases.map((e) => e.name).join(', ')})` : ''}
-- Content length: ${target.content.length} chars
-
-## Recent Conversation
-${recentMessages}
-
-## User's Latest Message
-${latestMessage}
-
-## Available Actions
-- run_evals: Run all eval cases against current version (with + without target). Choose this when the user wants to test, evaluate, or see how the instruction performs.
-- generate_cases: Auto-generate eval test cases from the instruction content. Choose this when there are no eval cases (${evalCases.length} currently) or user asks to create/generate tests.
-- analyze_results: Analyze recent eval failures and propose improvements based on user feedback. Choose this when user gives qualitative feedback like "this is wrong", "missing X", etc.
-- propose_changes: Propose specific content edits based on conversation. Choose this when user asks for improvements or changes.
-- respond: Just answer a question without running any evals. Choose this for informational questions.
-
-Respond with ONLY valid JSON:
-{"action": "run_evals", "response": "What you want to say to the user", "params": {}}`;
-    const result = await executeEval({ prompt, timeout: 30_000 });
-    try {
-        let jsonStr = result.output.trim();
-        const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
-        if (jsonMatch)
-            jsonStr = jsonMatch[0];
-        return JSON.parse(jsonStr);
-    }
-    catch {
-        logger('Failed to parse action decision, defaulting to respond');
-        // Fallback: simple heuristics
-        const lower = latestMessage.toLowerCase();
-        if (lower.includes('run') || lower.includes('test') || lower.includes('eval')) {
-            return { action: 'run_evals', response: 'Running evals now...' };
-        }
-        if (lower.includes('generate') || lower.includes('create') || (evalCases.length === 0 && lower.includes('start'))) {
-            return { action: 'generate_cases', response: 'Generating test cases...' };
-        }
-        return { action: 'respond', response: 'I can help you run evals, generate test cases, or analyze results. What would you like to do?' };
-    }
-}
-async function handleRunEvals(target, evalCases, _conversationId, postMessage, logger) {
-    if (evalCases.length === 0) {
-        await postMessage('No eval cases exist yet. Would you like me to generate some from the instruction content?', 'text');
-        return;
-    }
-    await postMessage(`Running ${evalCases.length} evals against "${target.name}" v${target.version}...`, 'status');
-    const results = [];
-    // Run with_target
-    for (const eval_ of evalCases) {
-        logger(`Running: ${eval_.name} (with_target)`);
-        const execResult = await executeEval({
-            prompt: eval_.prompt,
-            targetContent: target.content,
-            timeout: 120_000,
-        });
-        let passed = false;
-        let evidence = '';
-        if (!execResult.error && eval_.expectations.length > 0) {
-            const grading = await gradeEvalOutput({
-                prompt: eval_.prompt,
-                output: execResult.output,
-                expectations: eval_.expectations,
-                timeout: 120_000,
-            });
-            const summary = grading?.summary;
-            passed = summary?.pass_rate >= 1.0;
-            evidence = grading?.expectations
-                ?.map((e) => `${e.passed ? 'PASS' : 'FAIL'}: ${e.text} — ${e.evidence}`)
-                .join('\n') ?? '';
-        }
-        results.push({
-            evalName: eval_.name,
-            passed,
-            output: execResult.error ?? execResult.output.slice(0, 2000),
-            expectations: eval_.expectations,
-            evidence,
-            configuration: 'with_target',
-        });
-    }
-    const passedCount = results.filter((r) => r.passed).length;
-    const total = results.length;
-    await postMessage(`${passedCount} of ${total} evals passed (${Math.round((passedCount / total) * 100)}%)`, 'results', {
-        summary: { passed: passedCount, failed: total - passedCount, total, passRate: passedCount / total },
-        results: results.map((r) => ({
-            evalName: r.evalName,
-            passed: r.passed,
-            output: r.output,
-            expectations: r.expectations,
-            evidence: r.evidence,
-        })),
-    });
-}
-async function handleGenerateCases(target, _evalTargetId, _conversationId, postMessage, logger) {
-    await postMessage('Generating eval cases from instruction content...', 'status');
-    logger('Calling eval-from-context endpoint');
-    // Use eval-from-context endpoint
-    const result = await apiPost('/api/eval-from-context', {
-        contentHash: '',
-        name: target.name,
-        type: target.type,
-        content: target.content,
-        config: { skipBaseline: false, skipGrading: false },
-    });
-    const cases = result.evals ?? [];
-    await postMessage(`Generated ${cases.length} eval cases. Review them below:`, 'generated', {
-        cases: cases.map((c) => ({
-            id: c.id,
-            name: c.name,
-            prompt: c.prompt,
-            expectations: c.expectations,
-        })),
-    });
-}
-async function handleAnalyzeAndPropose(target, evalCases, messages, userFeedback, _conversationId, postMessage, logger) {
-    await postMessage('Analyzing results and preparing improvements...', 'status');
-    // Find the most recent results message for context
-    const lastResults = [...messages]
-        .reverse()
-        .find((m) => m.type === 'results');
-    const evalResults = lastResults?.data?.results?.map((r) => ({
-        evalName: r.evalName,
-        prompt: '',
-        expectations: r.expectations ?? [],
-        grading: {
-            summary: { pass_rate: r.passed ? 1 : 0 },
-            expectations: r.expectations?.map((e) => ({
-                text: e,
-                passed: r.passed,
-                evidence: r.evidence ?? '',
-            })),
-        },
-    })) ?? [];
-    logger(`Analyzing ${evalResults.length} eval results with user feedback`);
-    // Include user feedback in the analysis
-    const analysis = await analyzeEvalResults({
-        content: target.content,
-        evalResults,
-        model: undefined,
-        timeout: 120_000,
-        userFeedback,
-    });
-    await postMessage(analysis.overallAssessment, 'proposal', {
-        suggestions: analysis.suggestions,
-        proposedContent: analysis.proposedContent,
-        fromVersion: target.version,
-    });
-}
-//# sourceMappingURL=eval-conversation-handler.js.map
+import {
+  processConversationJob
+} from "../chunk-IWYCRLEK.js";
+import "../chunk-3TIMQ3O6.js";
+import "../chunk-YNIH3KMU.js";
+import "../chunk-QN34TGD3.js";
+import "../chunk-EJOI2ZGU.js";
+import "../chunk-KZRFUXGF.js";
+import "../chunk-NSBPE2FW.js";
+export {
+  processConversationJob
+};
+//# sourceMappingURL=eval-conversation-handler.js.map

package/dist/lib/eval-conversation-handler.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"eval-conversation-handler.js","sourceRoot":"","sources":["../../src/lib/eval-conversation-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAyCxD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAgC,EAChC,YAAoB,EACpB,MAA6B;IAE7B,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC;IAEhE,gBAAgB;IAChB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,QAAQ,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAC3B,2BAA2B,cAAc,WAAW,CACrD,CAAC;IAEF,8FAA8F;IAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;IAC7D,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAChI,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,CAAC,WAAW,SAAS,mDAAmD,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,MAAM,WAAW,GAAG,KAAK,EAAE,OAAe,EAAE,IAAY,EAAE,IAA8B,EAAE,EAAE;QAC1F,MAAM,OAAO,CAAC,2BAA2B,cAAc,qBAAqB,EAAE;YAC5E,OAAO;YACP,IAAI;YACJ,IAAI;SACL,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;IAEzF,MAAM,CAAC,WAAW,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAErC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,WAAW;YACd,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAC7E,MAAM;QAER,KAAK,gBAAgB;YACnB,MAAM,mBAAmB,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YACrF,MAAM;QAER,KAAK,iBAAiB,CAAC;QACvB,KAAK,iBAAiB;YACpB,MAAM,uBAAuB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAChH,MAAM;QAER,KAAK,SAAS;YACZ,MAAM,WAAW,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM;IACV,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAkB,EAClB,SAAqB,EACrB,QAA+B,EAC/B,aAAqB,EACrB,MAA6B;IAE7B,yCAAyC;IACzC,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnD,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACxC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,MAAM,GAAG;;;kBAGC,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,IAAI;gBACjD,SAAS,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAClG,MAAM,CAAC,OAAO,CAAC,MAAM;;;EAGvC,cAAc;;;EAGd,aAAa;;;;0HAI2G,SAAS,CAAC,MAAM;;;;;;sFAMpD,CAAC;IAErF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,wDAAwD,CAAC,CAAC;QACjE,8BAA8B;QAC9B,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9E,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC;QACnE,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAClH,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,0BAA0B,EAAE,CAAC;QAC5E,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,+FAA+F,EAAE,CAAC;IAC1I,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAkB,EAClB,SAAqB,EACrB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,WAAW,CACf,2FAA2F,EAC3F,MAAM,CACP,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,WAAW,CAAC,WAAW,SAAS,CAAC,MAAM,mBAAmB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEhH,MAAM,OAAO,GAOR,EAAE,CAAC;IAER,kBAAkB;IAClB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,YAAY,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;YACnC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,aAAa,EAAE,MAAM,CAAC,OAAO;YAC7B,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;gBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;YACH,MAAM,OAAO,GAAI,OAAe,EAAE,OAAO,CAAC;YAC1C,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;YACnC,QAAQ,GAAI,OAAe,EAAE,YAAY;gBACvC,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,QAAQ,EAAE,KAAK,CAAC,IAAI;YACpB,MAAM;YACN,MAAM,EAAE,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;YAC5D,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,QAAQ;YACR,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,MAAM,WAAW,CACf,GAAG,WAAW,OAAO,KAAK,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,EACvF,SAAS,EACT;QACE,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,GAAG,KAAK,EAAE;QACnG,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC;KACJ,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,MAAkB,EAClB,aAAqB,EACrB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,MAAM,WAAW,CAAC,mDAAmD,EAAE,QAAQ,CAAC,CAAC;IAEjF,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAE7C,iCAAiC;IACjC,MAAM,MAAM,GAAG,MAAM,OAAO,CAIzB,wBAAwB,EAAE;QAC3B,WAAW,EAAE,EAAE;QACf,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE;KACpD,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAEjC,MAAM,WAAW,CACf,aAAa,KAAK,CAAC,MAAM,iCAAiC,EAC1D,WAAW,EACX;QACE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;SAC7B,CAAC,CAAC;KACJ,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,MAAkB,EAClB,SAAqB,EACrB,QAA+B,EAC/B,YAAoB,EACpB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,MAAM,WAAW,CAAC,iDAAiD,EAAE,QAAQ,CAAC,CAAC;IAE/E,mDAAmD;IACnD,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC;SAC9B,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAErC,MAAM,WAAW,GAAI,WAAW,EAAE,IAAY,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACxE,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,EAAE;QACV,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE;QAClC,OAAO,EAAE;YACP,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;YACxC,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC;gBAChD,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE;aAC3B,CAAC,CAAC;SACJ;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IAEV,MAAM,CAAC,aAAa,WAAW,CAAC,MAAM,kCAAkC,CAAC,CAAC;IAE1E,wCAAwC;IACxC,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW;QACX,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,OAAO;QAChB,YAAY;KACb,CAAC,CAAC;IAEH,MAAM,WAAW,CACf,QAAQ,CAAC,iBAAiB,EAC1B,UAAU,EACV;QACE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,WAAW,EAAE,MAAM,CAAC,OAAO;KAC5B,CACF,CAAC;AACJ,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/eval-executor.js

@@ -1,83 +1,8 @@
-import { spawn } from 'node:child_process';
-/**
- * Executes a prompt via `claude -p` and captures the output.
- * Optionally injects target content as a system prompt.
- */
-export async function executeEval(options) {
-    const { prompt, targetContent, systemPrompt, workDir = process.cwd(), timeout = 120_000, model, } = options;
-    const args = ['-p', prompt, '--output-format', 'json'];
-    const sysPrompt = targetContent ?? systemPrompt;
-    if (sysPrompt) {
-        args.push('--system-prompt', sysPrompt);
-    }
-    if (model) {
-        args.push('--model', model);
-    }
-    const startTime = Date.now();
-    return new Promise((resolve) => {
-        // Remove CLAUDECODE flag so spawned claude doesn't think it's a nested session.
-        // Remove HTTPS_PROXY so eval runs bypass the capture proxy — these are internal
-        // infrastructure calls, not user sessions. Routing through the proxy creates
-        // phantom sessions that pollute insights.
-        const env = { ...process.env };
-        delete env.CLAUDECODE;
-        delete env.HTTPS_PROXY;
-        const proc = spawn('claude', args, {
-            cwd: workDir,
-            env,
-            stdio: ['ignore', 'pipe', 'pipe'],
-        });
-        let stdout = '';
-        let stderr = '';
-        proc.stdout.on('data', (data) => {
-            stdout += data.toString();
-        });
-        proc.stderr.on('data', (data) => {
-            stderr += data.toString();
-        });
-        const timer = setTimeout(() => {
-            proc.kill('SIGTERM');
-            resolve({
-                output: stdout,
-                exitCode: null,
-                durationMs: Date.now() - startTime,
-                totalTokens: 0,
-                error: `Timed out after ${timeout}ms`,
-            });
-        }, timeout);
-        proc.on('close', (code) => {
-            clearTimeout(timer);
-            const durationMs = Date.now() - startTime;
-            // Try to parse JSON output for token usage
-            let totalTokens = 0;
-            let output = stdout;
-            try {
-                const parsed = JSON.parse(stdout);
-                output = parsed.result ?? parsed.content ?? stdout;
-                totalTokens = parsed.usage?.total_tokens
-                    ?? ((parsed.usage?.input_tokens ?? 0) + (parsed.usage?.output_tokens ?? 0));
-            }
-            catch {
-                // stdout wasn't JSON, use raw text
-            }
-            resolve({
-                output,
-                exitCode: code,
-                durationMs,
-                totalTokens,
-                error: code !== 0 ? stderr || `Exit code ${code}` : undefined,
-            });
-        });
-        proc.on('error', (err) => {
-            clearTimeout(timer);
-            resolve({
-                output: '',
-                exitCode: null,
-                durationMs: Date.now() - startTime,
-                totalTokens: 0,
-                error: `Failed to spawn claude: ${err.message}`,
-            });
-        });
-    });
-}
-//# sourceMappingURL=eval-executor.js.map
+import {
+  executeEval
+} from "../chunk-QN34TGD3.js";
+import "../chunk-NSBPE2FW.js";
+export {
+  executeEval
+};
+//# sourceMappingURL=eval-executor.js.map

package/dist/lib/eval-executor.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"eval-executor.js","sourceRoot":"","sources":["../../src/lib/eval-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAmB3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA4B;IAC5D,MAAM,EACJ,MAAM,EACN,aAAa,EACb,YAAY,EACZ,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,EACvB,OAAO,GAAG,OAAO,EACjB,KAAK,GACN,GAAG,OAAO,CAAC;IAEZ,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,aAAa,IAAI,YAAY,CAAC;IAChD,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,EAAE;QACjD,gFAAgF;QAChF,gFAAgF;QAChF,6EAA6E;QAC7E,0CAA0C;QAC1C,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC,UAAU,CAAC;QACtB,OAAO,GAAG,CAAC,WAAW,CAAC;QAEvB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,OAAO;YACZ,GAAG;YACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACtC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACtC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,OAAO,CAAC;gBACN,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,WAAW,EAAE,CAAC;gBACd,KAAK,EAAE,mBAAmB,OAAO,IAAI;aACtC,CAAC,CAAC;QACL,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,2CAA2C;YAC3C,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,IAAI,MAAM,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;gBACnD,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,YAAY;uBACnC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;YAAC,MAAM,CAAC;gBACP,mCAAmC;YACrC,CAAC;YAED,OAAO,CAAC;gBACN,MAAM;gBACN,QAAQ,EAAE,IAAI;gBACd,UAAU;gBACV,WAAW;gBACX,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;aAC9D,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC;gBACN,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,WAAW,EAAE,CAAC;gBACd,KAAK,EAAE,2BAA2B,GAAG,CAAC,OAAO,EAAE;aAChD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/eval-grader.js

@@ -1,123 +1,9 @@
-import { executeEval } from './eval-executor.js';
-const GRADER_SYSTEM_PROMPT = `You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
-
-You will receive:
-1. The original prompt that was given to the AI
-2. The AI's output/response
-3. A list of expectations (assertions) to evaluate
-
-For each expectation:
-- Search the output for evidence that the expectation is met
-- Determine PASS or FAIL (no partial credit)
-- Cite specific evidence (quote or description)
-
-PASS criteria:
-- Clear evidence the assertion is true
-- Evidence reflects genuine completion, not surface compliance
-
-FAIL criteria:
-- No evidence found
-- Evidence contradicts the expectation
-- Cannot verify from available information
-
-Beyond the predefined expectations, extract implicit claims from the output:
-- Factual claims ("uses TypeScript", "has 3 parameters")
-- Process claims ("used async/await pattern")
-- Quality claims ("handles edge cases")
-
-Verify each claim against the actual output.
-
-Finally, critique the eval itself — identify assertions that are too weak (pass for wrong outputs) or important outcomes that no assertion covers.
-
-You MUST respond with ONLY a valid JSON object matching this exact structure (no markdown, no explanation, just JSON):
-
-{
-  "expectations": [
-    { "text": "assertion text", "passed": true, "evidence": "specific quote or description" }
-  ],
-  "summary": { "passed": 2, "failed": 1, "total": 3, "pass_rate": 0.67 },
-  "claims": [
-    { "claim": "statement", "type": "factual", "verified": true, "evidence": "..." }
-  ],
-  "eval_feedback": {
-    "suggestions": [{ "assertion": "suggested new assertion", "reason": "why it matters" }],
-    "overall": "brief summary assessment"
-  }
-}`;
-/**
- * Grades an eval output against expectations using an LLM grader.
- */
-export async function gradeEvalOutput(options) {
-    const { prompt, output, expectations, model, timeout = 120_000 } = options;
-    const graderPrompt = `## Original Prompt
-
-${prompt}
-
-## AI Output
-
-${output}
-
-## Expectations to Evaluate
-
-${expectations.map((e, i) => `${i + 1}. ${e}`).join('\n')}
-
-Grade the output against each expectation. Respond with ONLY valid JSON.`;
-    const result = await executeEval({
-        prompt: graderPrompt,
-        targetContent: GRADER_SYSTEM_PROMPT,
-        timeout,
-        model,
-    });
-    if (result.error) {
-        // Return a failed grading if the grader itself errors
-        return {
-            expectations: expectations.map((e) => ({
-                text: e,
-                passed: false,
-                evidence: `Grader error: ${result.error}`,
-            })),
-            summary: { passed: 0, failed: expectations.length, total: expectations.length, pass_rate: 0 },
-            claims: [],
-            eval_feedback: { suggestions: [], overall: `Grading failed: ${result.error}` },
-        };
-    }
-    // Parse the grader's JSON response
-    try {
-        // Extract JSON from the output (handle potential markdown wrapping)
-        let jsonStr = result.output.trim();
-        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-        if (jsonMatch) {
-            jsonStr = jsonMatch[1].trim();
-        }
-        // Also try to find JSON object directly
-        const objMatch = jsonStr.match(/\{[\s\S]*\}/);
-        if (objMatch) {
-            jsonStr = objMatch[0];
-        }
-        const parsed = JSON.parse(jsonStr);
-        // Validate and ensure summary is correct
-        const passed = parsed.expectations.filter((e) => e.passed).length;
-        const total = parsed.expectations.length;
-        parsed.summary = {
-            passed,
-            failed: total - passed,
-            total,
-            pass_rate: total > 0 ? passed / total : 0,
-        };
-        return parsed;
-    }
-    catch {
-        // If we can't parse, return a best-effort result
-        return {
-            expectations: expectations.map((e) => ({
-                text: e,
-                passed: false,
-                evidence: 'Grader output was not valid JSON',
-            })),
-            summary: { passed: 0, failed: expectations.length, total: expectations.length, pass_rate: 0 },
-            claims: [],
-            eval_feedback: { suggestions: [], overall: `Grader returned unparseable output: ${result.output.slice(0, 200)}` },
-        };
-    }
-}
-//# sourceMappingURL=eval-grader.js.map
+import {
+  gradeEvalOutput
+} from "../chunk-YNIH3KMU.js";
+import "../chunk-QN34TGD3.js";
+import "../chunk-NSBPE2FW.js";
+export {
+  gradeEvalOutput
+};
+//# sourceMappingURL=eval-grader.js.map

package/dist/lib/eval-grader.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/lib/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA4BjD,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4C3B,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAMrC;IACC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC;IAE3E,MAAM,YAAY,GAAG;;EAErB,MAAM;;;;EAIN,MAAM;;;;EAIN,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;yEAEgB,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,oBAAoB;QACnC,OAAO;QACP,KAAK;KACN,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,sDAAsD;QACtD,OAAO;YACL,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,iBAAiB,MAAM,CAAC,KAAK,EAAE;aAC1C,CAAC,CAAC;YACH,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE;YAC7F,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,mBAAmB,MAAM,CAAC,KAAK,EAAE,EAAE;SAC/E,CAAC;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,CAAC;QACH,oEAAoE;QACpE,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAChE,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,wCAAwC;QACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;QAEpD,yCAAyC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;QACzC,MAAM,CAAC,OAAO,GAAG;YACf,MAAM;YACN,MAAM,EAAE,KAAK,GAAG,MAAM;YACtB,KAAK;YACL,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;SAC1C,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;QACjD,OAAO;YACL,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,kCAAkC;aAC7C,CAAC,CAAC;YACH,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE;YAC7F,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,uCAAuC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;SAClH,CAAC;IACJ,CAAC;AACH,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}