@rulemetric/cli 0.1.1

package/dist/lib/eval-conversation-handler.js

@@ -0,0 +1,200 @@
+import { apiGet, apiPost } from './api-client.js';
+import { executeEval } from './eval-executor.js';
+import { gradeEvalOutput } from './eval-grader.js';
+import { analyzeEvalResults } from './eval-analyzer.js';
+export async function processConversationJob(jobConfig, evalTargetId, logger) {
+    const { conversationId, messageId, messageContent } = jobConfig;
+    // Fetch context
+    const target = await apiGet(`/api/eval-targets/${evalTargetId}`);
+    const evalCases = await apiGet(`/api/eval-targets/${evalTargetId}/evals`);
+    const messages = await apiGet(`/api/eval-conversations/${conversationId}/messages`);
+    // Idempotency: check if this message was already processed (has assistant responses after it)
+    const msgIdx = messages.findIndex((m) => m.id === messageId);
+    if (msgIdx >= 0) {
+        const hasResponse = messages.slice(msgIdx + 1).some((m) => m.role === 'assistant' && (m.type === 'results' || m.type === 'proposal' || m.type === 'generated' || m.type === 'text'));
+        if (hasResponse) {
+            logger(`Message ${messageId} already processed — skipping (idempotency guard)`);
+            return;
+        }
+    }
+    // Helper to post assistant messages
+    const postMessage = async (content, type, data) => {
+        await apiPost(`/api/eval-conversations/${conversationId}/messages/assistant`, {
+            content,
+            type,
+            data,
+        });
+    };
+    // Decide what action to take
+    const decision = await decideAction(target, evalCases, messages, messageContent, logger);
+    logger(`Action: ${decision.action}`);
+    switch (decision.action) {
+        case 'run_evals':
+            await handleRunEvals(target, evalCases, conversationId, postMessage, logger);
+            break;
+        case 'generate_cases':
+            await handleGenerateCases(target, evalTargetId, conversationId, postMessage, logger);
+            break;
+        case 'analyze_results':
+        case 'propose_changes':
+            await handleAnalyzeAndPropose(target, evalCases, messages, messageContent, conversationId, postMessage, logger);
+            break;
+        case 'respond':
+            await postMessage(decision.response, 'text');
+            break;
+    }
+}
+async function decideAction(target, evalCases, messages, latestMessage, logger) {
+    // Build conversation summary for context
+    const recentMessages = messages.slice(-10).map((m) => `${m.role}: ${m.content.slice(0, 200)}`).join('\n');
+    const prompt = `You are an eval assistant. Decide what action to take based on the user's message.
+
+## Current State
+- Eval target: "${target.name}" v${target.version} (${target.type})
+- Eval cases: ${evalCases.length} defined${evalCases.length > 0 ? ` (${evalCases.map((e) => e.name).join(', ')})` : ''}
+- Content length: ${target.content.length} chars
+
+## Recent Conversation
+${recentMessages}
+
+## User's Latest Message
+${latestMessage}
+
+## Available Actions
+- run_evals: Run all eval cases against current version (with + without target). Choose this when the user wants to test, evaluate, or see how the instruction performs.
+- generate_cases: Auto-generate eval test cases from the instruction content. Choose this when there are no eval cases (${evalCases.length} currently) or user asks to create/generate tests.
+- analyze_results: Analyze recent eval failures and propose improvements based on user feedback. Choose this when user gives qualitative feedback like "this is wrong", "missing X", etc.
+- propose_changes: Propose specific content edits based on conversation. Choose this when user asks for improvements or changes.
+- respond: Just answer a question without running any evals. Choose this for informational questions.
+
+Respond with ONLY valid JSON:
+{"action": "run_evals", "response": "What you want to say to the user", "params": {}}`;
+    const result = await executeEval({ prompt, timeout: 30_000 });
+    try {
+        let jsonStr = result.output.trim();
+        const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
+        if (jsonMatch)
+            jsonStr = jsonMatch[0];
+        return JSON.parse(jsonStr);
+    }
+    catch {
+        logger('Failed to parse action decision, defaulting to respond');
+        // Fallback: simple heuristics
+        const lower = latestMessage.toLowerCase();
+        if (lower.includes('run') || lower.includes('test') || lower.includes('eval')) {
+            return { action: 'run_evals', response: 'Running evals now...' };
+        }
+        if (lower.includes('generate') || lower.includes('create') || (evalCases.length === 0 && lower.includes('start'))) {
+            return { action: 'generate_cases', response: 'Generating test cases...' };
+        }
+        return { action: 'respond', response: 'I can help you run evals, generate test cases, or analyze results. What would you like to do?' };
+    }
+}
+async function handleRunEvals(target, evalCases, _conversationId, postMessage, logger) {
+    if (evalCases.length === 0) {
+        await postMessage('No eval cases exist yet. Would you like me to generate some from the instruction content?', 'text');
+        return;
+    }
+    await postMessage(`Running ${evalCases.length} evals against "${target.name}" v${target.version}...`, 'status');
+    const results = [];
+    // Run with_target
+    for (const eval_ of evalCases) {
+        logger(`Running: ${eval_.name} (with_target)`);
+        const execResult = await executeEval({
+            prompt: eval_.prompt,
+            targetContent: target.content,
+            timeout: 120_000,
+        });
+        let passed = false;
+        let evidence = '';
+        if (!execResult.error && eval_.expectations.length > 0) {
+            const grading = await gradeEvalOutput({
+                prompt: eval_.prompt,
+                output: execResult.output,
+                expectations: eval_.expectations,
+                timeout: 120_000,
+            });
+            const summary = grading?.summary;
+            passed = summary?.pass_rate >= 1.0;
+            evidence = grading?.expectations
+                ?.map((e) => `${e.passed ? 'PASS' : 'FAIL'}: ${e.text} — ${e.evidence}`)
+                .join('\n') ?? '';
+        }
+        results.push({
+            evalName: eval_.name,
+            passed,
+            output: execResult.error ?? execResult.output.slice(0, 2000),
+            expectations: eval_.expectations,
+            evidence,
+            configuration: 'with_target',
+        });
+    }
+    const passedCount = results.filter((r) => r.passed).length;
+    const total = results.length;
+    await postMessage(`${passedCount} of ${total} evals passed (${Math.round((passedCount / total) * 100)}%)`, 'results', {
+        summary: { passed: passedCount, failed: total - passedCount, total, passRate: passedCount / total },
+        results: results.map((r) => ({
+            evalName: r.evalName,
+            passed: r.passed,
+            output: r.output,
+            expectations: r.expectations,
+            evidence: r.evidence,
+        })),
+    });
+}
+async function handleGenerateCases(target, _evalTargetId, _conversationId, postMessage, logger) {
+    await postMessage('Generating eval cases from instruction content...', 'status');
+    logger('Calling eval-from-context endpoint');
+    // Use eval-from-context endpoint
+    const result = await apiPost('/api/eval-from-context', {
+        contentHash: '',
+        name: target.name,
+        type: target.type,
+        content: target.content,
+        config: { skipBaseline: false, skipGrading: false },
+    });
+    const cases = result.evals ?? [];
+    await postMessage(`Generated ${cases.length} eval cases. Review them below:`, 'generated', {
+        cases: cases.map((c) => ({
+            id: c.id,
+            name: c.name,
+            prompt: c.prompt,
+            expectations: c.expectations,
+        })),
+    });
+}
+async function handleAnalyzeAndPropose(target, evalCases, messages, userFeedback, _conversationId, postMessage, logger) {
+    await postMessage('Analyzing results and preparing improvements...', 'status');
+    // Find the most recent results message for context
+    const lastResults = [...messages]
+        .reverse()
+        .find((m) => m.type === 'results');
+    const evalResults = lastResults?.data?.results?.map((r) => ({
+        evalName: r.evalName,
+        prompt: '',
+        expectations: r.expectations ?? [],
+        grading: {
+            summary: { pass_rate: r.passed ? 1 : 0 },
+            expectations: r.expectations?.map((e) => ({
+                text: e,
+                passed: r.passed,
+                evidence: r.evidence ?? '',
+            })),
+        },
+    })) ?? [];
+    logger(`Analyzing ${evalResults.length} eval results with user feedback`);
+    // Include user feedback in the analysis
+    const analysis = await analyzeEvalResults({
+        content: target.content,
+        evalResults,
+        model: undefined,
+        timeout: 120_000,
+        userFeedback,
+    });
+    await postMessage(analysis.overallAssessment, 'proposal', {
+        suggestions: analysis.suggestions,
+        proposedContent: analysis.proposedContent,
+        fromVersion: target.version,
+    });
+}
+//# sourceMappingURL=eval-conversation-handler.js.map

package/dist/lib/eval-conversation-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-conversation-handler.js","sourceRoot":"","sources":["../../src/lib/eval-conversation-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAyCxD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,SAAgC,EAChC,YAAoB,EACpB,MAA6B;IAE7B,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC;IAEhE,gBAAgB;IAChB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,QAAQ,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAC3B,2BAA2B,cAAc,WAAW,CACrD,CAAC;IAEF,8FAA8F;IAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;IAC7D,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAChI,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,CAAC,WAAW,SAAS,mDAAmD,CAAC,CAAC;YAChF,OAAO;QACT,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,MAAM,WAAW,GAAG,KAAK,EAAE,OAAe,EAAE,IAAY,EAAE,IAA8B,EAAE,EAAE;QAC1F,MAAM,OAAO,CAAC,2BAA2B,cAAc,qBAAqB,EAAE;YAC5E,OAAO;YACP,IAAI;YACJ,IAAI;SACL,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;IAEzF,MAAM,CAAC,WAAW,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAErC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,WAAW;YACd,MAAM,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAC7E,MAAM;QAER,KAAK,gBAAgB;YACnB,MAAM,mBAAmB,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YACrF,MAAM;QAER,KAAK,iBAAiB,CAAC;QACvB,KAAK,iBAAiB;YACpB,MAAM,uBAAuB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YAChH,MAAM;QAER,KAAK,SAAS;YACZ,MAAM,WAAW,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM;IACV,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAkB,EAClB,SAAqB,EACrB,QAA+B,EAC/B,aAAqB,EACrB,MAA6B;IAE7B,yCAAyC;IACzC,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnD,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACxC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,MAAM,GAAG;;;kBAGC,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,IAAI;gBACjD,SAAS,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;oBAClG,MAAM,CAAC,OAAO,CAAC,MAAM;;;EAGvC,cAAc;;;EAGd,aAAa;;;;0HAI2G,SAAS,CAAC,MAAM;;;;;;sFAMpD,CAAC;IAErF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,wDAAwD,CAAC,CAAC;QACjE,8BAA8B;QAC9B,MAAM,KAAK,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9E,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC;QACnE,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAClH,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,0BAA0B,EAAE,CAAC;QAC5E,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,+FAA+F,EAAE,CAAC;IAC1I,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAkB,EAClB,SAAqB,EACrB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,WAAW,CACf,2FAA2F,EAC3F,MAAM,CACP,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,WAAW,CAAC,WAAW,SAAS,CAAC,MAAM,mBAAmB,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEhH,MAAM,OAAO,GAOR,EAAE,CAAC;IAER,kBAAkB;IAClB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,CAAC,YAAY,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;YACnC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,aAAa,EAAE,MAAM,CAAC,OAAO;YAC7B,OAAO,EAAE,OAAO;SACjB,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;gBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;YACH,MAAM,OAAO,GAAI,OAAe,EAAE,OAAO,CAAC;YAC1C,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;YACnC,QAAQ,GAAI,OAAe,EAAE,YAAY;gBACvC,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;iBAC5E,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,QAAQ,EAAE,KAAK,CAAC,IAAI;YACpB,MAAM;YACN,MAAM,EAAE,UAAU,CAAC,KAAK,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;YAC5D,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,QAAQ;YACR,aAAa,EAAE,aAAa;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAE7B,MAAM,WAAW,CACf,GAAG,WAAW,OAAO,KAAK,kBAAkB,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,EACvF,SAAS,EACT;QACE,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,GAAG,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,GAAG,KAAK,EAAE;QACnG,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC;KACJ,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,MAAkB,EAClB,aAAqB,EACrB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,MAAM,WAAW,CAAC,mDAAmD,EAAE,QAAQ,CAAC,CAAC;IAEjF,MAAM,CAAC,oCAAoC,CAAC,CAAC;IAE7C,iCAAiC;IACjC,MAAM,MAAM,GAAG,MAAM,OAAO,CAIzB,wBAAwB,EAAE;QAC3B,WAAW,EAAE,EAAE;QACf,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE;KACpD,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAEjC,MAAM,WAAW,CACf,aAAa,KAAK,CAAC,MAAM,iCAAiC,EAC1D,WAAW,EACX;QACE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;SAC7B,CAAC,CAAC;KACJ,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,MAAkB,EAClB,SAAqB,EACrB,QAA+B,EAC/B,YAAoB,EACpB,eAAuB,EACvB,WAA6F,EAC7F,MAA6B;IAE7B,MAAM,WAAW,CAAC,iDAAiD,EAAE,QAAQ,CAAC,CAAC;IAE/E,mDAAmD;IACnD,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC;SAC9B,OAAO,EAAE;SACT,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAErC,MAAM,WAAW,GAAI,WAAW,EAAE,IAAY,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACxE,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,EAAE;QACV,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,EAAE;QAClC,OAAO,EAAE;YACP,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;YACxC,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC;gBAChD,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,EAAE;aAC3B,CAAC,CAAC;SACJ;KACF,CAAC,CAAC,IAAI,EAAE,CAAC;IAEV,MAAM,CAAC,aAAa,WAAW,CAAC,MAAM,kCAAkC,CAAC,CAAC;IAE1E,wCAAwC;IACxC,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW;QACX,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,OAAO;QAChB,YAAY;KACb,CAAC,CAAC;IAEH,MAAM,WAAW,CACf,QAAQ,CAAC,iBAAiB,EAC1B,UAAU,EACV;QACE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,WAAW,EAAE,MAAM,CAAC,OAAO;KAC5B,CACF,CAAC;AACJ,CAAC"}

package/dist/lib/eval-executor.d.ts

@@ -0,0 +1,21 @@
+export interface EvalExecutorOptions {
+    prompt: string;
+    targetContent?: string;
+    systemPrompt?: string;
+    workDir?: string;
+    timeout?: number;
+    model?: string;
+}
+export interface EvalExecutorResult {
+    output: string;
+    exitCode: number | null;
+    durationMs: number;
+    totalTokens: number;
+    error?: string;
+}
+/**
+ * Executes a prompt via `claude -p` and captures the output.
+ * Optionally injects target content as a system prompt.
+ */
+export declare function executeEval(options: EvalExecutorOptions): Promise<EvalExecutorResult>;
+//# sourceMappingURL=eval-executor.d.ts.map

package/dist/lib/eval-executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.d.ts","sourceRoot":"","sources":["../../src/lib/eval-executor.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA2F3F"}

package/dist/lib/eval-executor.js

@@ -0,0 +1,80 @@
+import { spawn } from 'node:child_process';
+/**
+ * Executes a prompt via `claude -p` and captures the output.
+ * Optionally injects target content as a system prompt.
+ */
+export async function executeEval(options) {
+    const { prompt, targetContent, systemPrompt, workDir = process.cwd(), timeout = 120_000, model, } = options;
+    const args = ['-p', prompt, '--output-format', 'json'];
+    const sysPrompt = targetContent ?? systemPrompt;
+    if (sysPrompt) {
+        args.push('--system-prompt', sysPrompt);
+    }
+    if (model) {
+        args.push('--model', model);
+    }
+    const startTime = Date.now();
+    return new Promise((resolve) => {
+        // Remove CLAUDECODE flag so spawned claude doesn't think it's a nested session,
+        // but keep CLAUDE_CODE_SSE_PORT etc. so it can still authenticate
+        const env = { ...process.env };
+        delete env.CLAUDECODE;
+        const proc = spawn('claude', args, {
+            cwd: workDir,
+            env,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let stdout = '';
+        let stderr = '';
+        proc.stdout.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr.on('data', (data) => {
+            stderr += data.toString();
+        });
+        const timer = setTimeout(() => {
+            proc.kill('SIGTERM');
+            resolve({
+                output: stdout,
+                exitCode: null,
+                durationMs: Date.now() - startTime,
+                totalTokens: 0,
+                error: `Timed out after ${timeout}ms`,
+            });
+        }, timeout);
+        proc.on('close', (code) => {
+            clearTimeout(timer);
+            const durationMs = Date.now() - startTime;
+            // Try to parse JSON output for token usage
+            let totalTokens = 0;
+            let output = stdout;
+            try {
+                const parsed = JSON.parse(stdout);
+                output = parsed.result ?? parsed.content ?? stdout;
+                totalTokens = parsed.usage?.total_tokens
+                    ?? ((parsed.usage?.input_tokens ?? 0) + (parsed.usage?.output_tokens ?? 0));
+            }
+            catch {
+                // stdout wasn't JSON, use raw text
+            }
+            resolve({
+                output,
+                exitCode: code,
+                durationMs,
+                totalTokens,
+                error: code !== 0 ? stderr || `Exit code ${code}` : undefined,
+            });
+        });
+        proc.on('error', (err) => {
+            clearTimeout(timer);
+            resolve({
+                output: '',
+                exitCode: null,
+                durationMs: Date.now() - startTime,
+                totalTokens: 0,
+                error: `Failed to spawn claude: ${err.message}`,
+            });
+        });
+    });
+}
+//# sourceMappingURL=eval-executor.js.map

package/dist/lib/eval-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.js","sourceRoot":"","sources":["../../src/lib/eval-executor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAmB3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA4B;IAC5D,MAAM,EACJ,MAAM,EACN,aAAa,EACb,YAAY,EACZ,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,EACvB,OAAO,GAAG,OAAO,EACjB,KAAK,GACN,GAAG,OAAO,CAAC;IAEZ,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,aAAa,IAAI,YAAY,CAAC;IAChD,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,EAAE;QACjD,gFAAgF;QAChF,kEAAkE;QAClE,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC/B,OAAO,GAAG,CAAC,UAAU,CAAC;QAEtB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,OAAO;YACZ,GAAG;YACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACtC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACtC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,OAAO,CAAC;gBACN,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,WAAW,EAAE,CAAC;gBACd,KAAK,EAAE,mBAAmB,OAAO,IAAI;aACtC,CAAC,CAAC;QACL,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,2CAA2C;YAC3C,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,IAAI,MAAM,GAAG,MAAM,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;gBACnD,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,YAAY;uBACnC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;YAAC,MAAM,CAAC;gBACP,mCAAmC;YACrC,CAAC;YAED,OAAO,CAAC;gBACN,MAAM;gBACN,QAAQ,EAAE,IAAI;gBACd,UAAU;gBACV,WAAW;gBACX,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;aAC9D,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC;gBACN,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,WAAW,EAAE,CAAC;gBACd,KAAK,EAAE,2BAA2B,GAAG,CAAC,OAAO,EAAE;aAChD,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/lib/eval-executor.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=eval-executor.test.d.ts.map

package/dist/lib/eval-executor.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.test.d.ts","sourceRoot":"","sources":["../../src/lib/eval-executor.test.ts"],"names":[],"mappings":""}

package/dist/lib/eval-executor.test.js

@@ -0,0 +1,8 @@
+import { describe, it, expect } from 'vitest';
+describe('eval-executor', () => {
+    it('module exports executeEval function', async () => {
+        const mod = await import('./eval-executor.js');
+        expect(typeof mod.executeEval).toBe('function');
+    });
+});
+//# sourceMappingURL=eval-executor.test.js.map

package/dist/lib/eval-executor.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-executor.test.js","sourceRoot":"","sources":["../../src/lib/eval-executor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE9C,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,OAAO,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/eval-grader.d.ts

@@ -0,0 +1,38 @@
+export interface GradingExpectation {
+    text: string;
+    passed: boolean;
+    evidence: string;
+}
+export interface GradingResult {
+    expectations: GradingExpectation[];
+    summary: {
+        passed: number;
+        failed: number;
+        total: number;
+        pass_rate: number;
+    };
+    claims: Array<{
+        claim: string;
+        type: 'factual' | 'process' | 'quality';
+        verified: boolean;
+        evidence: string;
+    }>;
+    eval_feedback: {
+        suggestions: Array<{
+            assertion?: string;
+            reason: string;
+        }>;
+        overall: string;
+    };
+}
+/**
+ * Grades an eval output against expectations using an LLM grader.
+ */
+export declare function gradeEvalOutput(options: {
+    prompt: string;
+    output: string;
+    expectations: string[];
+    model?: string;
+    timeout?: number;
+}): Promise<GradingResult>;
+//# sourceMappingURL=eval-grader.d.ts.map

package/dist/lib/eval-grader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.d.ts","sourceRoot":"","sources":["../../src/lib/eval-grader.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,YAAY,EAAE,kBAAkB,EAAE,CAAC;IACnC,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,MAAM,EAAE,KAAK,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;QACxC,QAAQ,EAAE,OAAO,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,aAAa,EAAE;QACb,WAAW,EAAE,KAAK,CAAC;YAAE,SAAS,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAgDD;;GAEG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,aAAa,CAAC,CA8EzB"}

package/dist/lib/eval-grader.js

@@ -0,0 +1,123 @@
+import { executeEval } from './eval-executor.js';
+const GRADER_SYSTEM_PROMPT = `You are a rigorous eval grader. Your job is to evaluate whether an AI assistant's output meets a set of expectations.
+
+You will receive:
+1. The original prompt that was given to the AI
+2. The AI's output/response
+3. A list of expectations (assertions) to evaluate
+
+For each expectation:
+- Search the output for evidence that the expectation is met
+- Determine PASS or FAIL (no partial credit)
+- Cite specific evidence (quote or description)
+
+PASS criteria:
+- Clear evidence the assertion is true
+- Evidence reflects genuine completion, not surface compliance
+
+FAIL criteria:
+- No evidence found
+- Evidence contradicts the expectation
+- Cannot verify from available information
+
+Beyond the predefined expectations, extract implicit claims from the output:
+- Factual claims ("uses TypeScript", "has 3 parameters")
+- Process claims ("used async/await pattern")
+- Quality claims ("handles edge cases")
+
+Verify each claim against the actual output.
+
+Finally, critique the eval itself — identify assertions that are too weak (pass for wrong outputs) or important outcomes that no assertion covers.
+
+You MUST respond with ONLY a valid JSON object matching this exact structure (no markdown, no explanation, just JSON):
+
+{
+  "expectations": [
+    { "text": "assertion text", "passed": true, "evidence": "specific quote or description" }
+  ],
+  "summary": { "passed": 2, "failed": 1, "total": 3, "pass_rate": 0.67 },
+  "claims": [
+    { "claim": "statement", "type": "factual", "verified": true, "evidence": "..." }
+  ],
+  "eval_feedback": {
+    "suggestions": [{ "assertion": "suggested new assertion", "reason": "why it matters" }],
+    "overall": "brief summary assessment"
+  }
+}`;
+/**
+ * Grades an eval output against expectations using an LLM grader.
+ */
+export async function gradeEvalOutput(options) {
+    const { prompt, output, expectations, model, timeout = 120_000 } = options;
+    const graderPrompt = `## Original Prompt
+
+${prompt}
+
+## AI Output
+
+${output}
+
+## Expectations to Evaluate
+
+${expectations.map((e, i) => `${i + 1}. ${e}`).join('\n')}
+
+Grade the output against each expectation. Respond with ONLY valid JSON.`;
+    const result = await executeEval({
+        prompt: graderPrompt,
+        targetContent: GRADER_SYSTEM_PROMPT,
+        timeout,
+        model,
+    });
+    if (result.error) {
+        // Return a failed grading if the grader itself errors
+        return {
+            expectations: expectations.map((e) => ({
+                text: e,
+                passed: false,
+                evidence: `Grader error: ${result.error}`,
+            })),
+            summary: { passed: 0, failed: expectations.length, total: expectations.length, pass_rate: 0 },
+            claims: [],
+            eval_feedback: { suggestions: [], overall: `Grading failed: ${result.error}` },
+        };
+    }
+    // Parse the grader's JSON response
+    try {
+        // Extract JSON from the output (handle potential markdown wrapping)
+        let jsonStr = result.output.trim();
+        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+        if (jsonMatch) {
+            jsonStr = jsonMatch[1].trim();
+        }
+        // Also try to find JSON object directly
+        const objMatch = jsonStr.match(/\{[\s\S]*\}/);
+        if (objMatch) {
+            jsonStr = objMatch[0];
+        }
+        const parsed = JSON.parse(jsonStr);
+        // Validate and ensure summary is correct
+        const passed = parsed.expectations.filter((e) => e.passed).length;
+        const total = parsed.expectations.length;
+        parsed.summary = {
+            passed,
+            failed: total - passed,
+            total,
+            pass_rate: total > 0 ? passed / total : 0,
+        };
+        return parsed;
+    }
+    catch {
+        // If we can't parse, return a best-effort result
+        return {
+            expectations: expectations.map((e) => ({
+                text: e,
+                passed: false,
+                evidence: 'Grader output was not valid JSON',
+            })),
+            summary: { passed: 0, failed: expectations.length, total: expectations.length, pass_rate: 0 },
+            claims: [],
+            eval_feedback: { suggestions: [], overall: `Grader returned unparseable output: ${result.output.slice(0, 200)}` },
+        };
+    }
+}
+//# sourceMappingURL=eval-grader.js.map

package/dist/lib/eval-grader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/lib/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA4BjD,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4C3B,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAMrC;IACC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC;IAE3E,MAAM,YAAY,GAAG;;EAErB,MAAM;;;;EAIN,MAAM;;;;EAIN,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;yEAEgB,CAAC;IAExE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,MAAM,EAAE,YAAY;QACpB,aAAa,EAAE,oBAAoB;QACnC,OAAO;QACP,KAAK;KACN,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,sDAAsD;QACtD,OAAO;YACL,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,iBAAiB,MAAM,CAAC,KAAK,EAAE;aAC1C,CAAC,CAAC;YACH,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE;YAC7F,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,mBAAmB,MAAM,CAAC,KAAK,EAAE,EAAE;SAC/E,CAAC;IACJ,CAAC;IAED,mCAAmC;IACnC,IAAI,CAAC;QACH,oEAAoE;QACpE,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAChE,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,wCAAwC;QACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;QAEpD,yCAAyC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;QACzC,MAAM,CAAC,OAAO,GAAG;YACf,MAAM;YACN,MAAM,EAAE,KAAK,GAAG,MAAM;YACtB,KAAK;YACL,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;SAC1C,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,iDAAiD;QACjD,OAAO;YACL,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,kCAAkC;aAC7C,CAAC,CAAC;YACH,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE;YAC7F,MAAM,EAAE,EAAE;YACV,aAAa,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,uCAAuC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;SAClH,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/lib/eval-grader.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=eval-grader.test.d.ts.map

package/dist/lib/eval-grader.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.test.d.ts","sourceRoot":"","sources":["../../src/lib/eval-grader.test.ts"],"names":[],"mappings":""}

package/dist/lib/eval-grader.test.js

@@ -0,0 +1,8 @@
+import { describe, it, expect } from 'vitest';
+describe('eval-grader', () => {
+    it('module exports gradeEvalOutput function', async () => {
+        const mod = await import('./eval-grader.js');
+        expect(typeof mod.gradeEvalOutput).toBe('function');
+    });
+});
+//# sourceMappingURL=eval-grader.test.js.map

package/dist/lib/eval-grader.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-grader.test.js","sourceRoot":"","sources":["../../src/lib/eval-grader.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE9C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAC7C,MAAM,CAAC,OAAO,GAAG,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/eval-meta-judge.d.ts

@@ -0,0 +1,28 @@
+import type { GradingResult } from './eval-grader.js';
+export interface MetaJudgement {
+    quality_score: number;
+    issues: MetaIssue[];
+    recommendations: string[];
+    summary: string;
+}
+export interface MetaIssue {
+    type: 'weak_expectation' | 'inconsistent_grading' | 'trivial_test' | 'untestable_expectation' | 'all_pass_suspicious';
+    severity: 'low' | 'medium' | 'high';
+    description: string;
+    affected_eval?: string;
+}
+interface MetaJudgeInput {
+    targetName: string;
+    evalResults: Array<{
+        evalName: string;
+        prompt: string;
+        expectations: string[];
+        grading: GradingResult | null;
+        output?: string;
+    }>;
+    model?: string;
+    timeout?: number;
+}
+export declare function judgeEvalQuality(input: MetaJudgeInput): Promise<MetaJudgement>;
+export {};
+//# sourceMappingURL=eval-meta-judge.d.ts.map

package/dist/lib/eval-meta-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-meta-judge.d.ts","sourceRoot":"","sources":["../../src/lib/eval-meta-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,kBAAkB,GAAG,sBAAsB,GAAG,cAAc,GAAG,wBAAwB,GAAG,qBAAqB,CAAC;IACtH,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAmCD,UAAU,cAAc;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,KAAK,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAuEpF"}