@rulemetric/cli 0.1.1

package/dist/commands/evals/optimize-description.js

@@ -0,0 +1,269 @@
+import { Args, Flags } from '@oclif/core';
+import { BaseCommand } from '../../base-command.js';
+import { apiGet, apiPost, apiPut } from '../../lib/api-client.js';
+import { executeEval } from '../../lib/eval-executor.js';
+import { testTrigger, computeTriggerScore } from '../../lib/eval-trigger-tester.js';
+export default class EvalOptimizeDescriptionCommand extends BaseCommand {
+    static description = 'Optimize eval target description for better triggering accuracy';
+    static examples = [
+        '<%= config.bin %> evals optimize-description <target-id>',
+        '<%= config.bin %> evals optimize-description <target-id> --queries 10',
+    ];
+    static args = {
+        targetId: Args.string({ description: 'Eval target ID', required: true }),
+    };
+    static flags = {
+        'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),
+        model: Flags.string({ description: 'Model for trigger testing and improvement' }),
+        queries: Flags.integer({ description: 'Number of trigger queries to generate', default: 20 }),
+        'runs-per-query': Flags.integer({ description: 'Runs per query for reliability', default: 3 }),
+        'poll-interval': Flags.integer({ description: 'Ms between approval polls', default: 5000 }),
+        timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(EvalOptimizeDescriptionCommand);
+        this.requireAuth();
+        const target = await apiGet(`/api/eval-targets/${args.targetId}`);
+        this.log(`Optimizing description for "${target.name}" v${target.version}`);
+        this.log(`Current description: "${target.description}"`);
+        this.log('');
+        // Step 1: Generate trigger queries
+        this.log('Generating trigger queries...');
+        const queries = await this.generateTriggerQueries(target, flags.queries, flags);
+        // Step 2: Post to API for review
+        const triggerEval = await apiPost(`/api/eval-targets/${args.targetId}/trigger-evals`, {
+            queries,
+        });
+        this.log(`\nGenerated ${queries.length} trigger queries.`);
+        this.log(`Review and approve them at:`);
+        this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);
+        this.log(`  (Triggers tab → click to review)`);
+        this.log('\nWaiting for approval...');
+        // Step 3: Poll for approval
+        let approved = false;
+        while (!approved) {
+            await new Promise((resolve) => setTimeout(resolve, flags['poll-interval']));
+            const sets = await apiGet(`/api/eval-targets/${args.targetId}/trigger-evals?status=approved`);
+            const current = sets.find((s) => s.id === triggerEval.id);
+            if (current) {
+                approved = true;
+                // Use potentially edited queries
+                queries.length = 0;
+                queries.push(...current.queries);
+                this.log('Trigger queries approved!');
+            }
+        }
+        // Step 4: Split train/test (60/40)
+        const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);
+        const splitIdx = Math.ceil(indices.length * 0.6);
+        const trainIndices = indices.slice(0, splitIdx);
+        const testIndices = indices.slice(splitIdx);
+        // Save split to API
+        await apiPut(`/api/trigger-evals/${triggerEval.id}`, {
+            trainIndices,
+            testIndices,
+        });
+        const trainQueries = trainIndices.map((i) => queries[i]);
+        const testQueries = testIndices.map((i) => queries[i]);
+        this.log(`\nSplit: ${trainQueries.length} train, ${testQueries.length} test`);
+        // Step 5: Optimization loop
+        let currentDescription = target.description;
+        let bestDescription = currentDescription;
+        let bestTestScore = 0;
+        for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {
+            this.log(`\n=== Iteration ${iteration} ===`);
+            this.log(`Description: "${currentDescription}"`);
+            // Test on train set (multiple runs per query)
+            this.log('Testing train set...');
+            const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);
+            const trainScore = computeTriggerScore(trainResults);
+            this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);
+            this.printTriggerResults(trainResults);
+            // Test on test set
+            this.log('Testing test set...');
+            const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);
+            const testScore = computeTriggerScore(testResults);
+            this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);
+            // Record optimization run
+            await apiPost('/api/optimization-runs', {
+                evalTargetId: target.id,
+                type: 'description',
+                iteration,
+                inputVersion: target.version,
+                trainScore,
+                testScore,
+                analysis: {
+                    description: currentDescription,
+                    trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
+                    testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
+                },
+                status: 'completed',
+            });
+            // Track best by test score
+            if (testScore > bestTestScore) {
+                bestTestScore = testScore;
+                bestDescription = currentDescription;
+            }
+            // Stop if perfect
+            if (trainScore >= 1.0) {
+                this.log('Perfect train score. Stopping.');
+                break;
+            }
+            // Propose improved description
+            this.log('Generating improved description...');
+            const failures = trainResults.filter((r) => !r.correct);
+            currentDescription = await this.improveDescription(target, currentDescription, failures, flags);
+        }
+        // Step 6: Apply best description
+        this.log(`\n=== Optimization Complete ===`);
+        this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);
+        this.log(`\nBefore: "${target.description}"`);
+        this.log(`After:  "${bestDescription}"`);
+        if (bestDescription !== target.description) {
+            await apiPut(`/api/eval-targets/${target.id}`, {
+                description: bestDescription,
+            });
+            this.log('\nDescription updated.');
+        }
+        else {
+            this.log('\nNo improvement found. Description unchanged.');
+        }
+    }
+    async generateTriggerQueries(target, count, flags) {
+        // Try to seed with real session data
+        let sessionSeed = '';
+        try {
+            const sessions = await apiGet('/api/sessions?limit=10');
+            if (sessions.length > 0) {
+                sessionSeed = '\n\nReal user prompts from recent sessions (use as inspiration for realistic queries):\n' +
+                    sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join('\n');
+            }
+        }
+        catch {
+            // No sessions available, that's fine
+        }
+        const shouldTriggerCount = Math.ceil(count / 2);
+        const shouldNotTriggerCount = count - shouldTriggerCount;
+        const prompt = `Generate ${count} trigger eval queries for a skill/instruction.
+
+## Skill Name
+${target.name}
+
+## Skill Description
+${target.description}
+
+## Skill Content (first 1000 chars)
+${target.content.slice(0, 1000)}
+${sessionSeed}
+
+Generate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.
+
+Rules:
+- Queries must be realistic — what a real user would actually type
+- Include concrete details (file paths, project names, specific problems)
+- Vary length and formality
+- Should-not-trigger queries should be near-misses (share keywords but need something different)
+- Do NOT make should-not-trigger queries obviously irrelevant
+
+Respond with ONLY a valid JSON array:
+[
+  {"query": "realistic user prompt...", "should_trigger": true},
+  {"query": "another prompt...", "should_trigger": false}
+]`;
+        const result = await executeEval({
+            prompt,
+            timeout: flags.timeout,
+            model: flags.model,
+        });
+        try {
+            let jsonStr = result.output.trim();
+            const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+            if (jsonMatch)
+                jsonStr = jsonMatch[1].trim();
+            const arrMatch = jsonStr.match(/\[[\s\S]*\]/);
+            if (arrMatch)
+                jsonStr = arrMatch[0];
+            return JSON.parse(jsonStr);
+        }
+        catch {
+            // Fallback: generate minimal set
+            return [
+                { query: `Help me use ${target.name}`, should_trigger: true },
+                { query: 'Write a hello world program', should_trigger: false },
+            ];
+        }
+    }
+    async testQuerySet(queries, target, description, flags) {
+        const results = [];
+        for (const q of queries) {
+            // Run multiple times and take majority vote
+            const runs = [];
+            for (let run = 0; run < flags['runs-per-query']; run++) {
+                const result = await testTrigger({
+                    query: q.query,
+                    shouldTrigger: q.should_trigger,
+                    skillName: target.name,
+                    skillDescription: description,
+                    skillContent: target.content,
+                    model: flags.model,
+                    timeout: flags.timeout,
+                });
+                runs.push(result);
+            }
+            // Majority vote
+            const triggerCount = runs.filter((r) => r.didTrigger).length;
+            const didTrigger = triggerCount > flags['runs-per-query'] / 2;
+            results.push({
+                query: q.query,
+                shouldTrigger: q.should_trigger,
+                didTrigger,
+                correct: didTrigger === q.should_trigger,
+                confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,
+                evidence: runs[0].evidence,
+            });
+        }
+        return results;
+    }
+    printTriggerResults(results) {
+        for (const r of results) {
+            const status = r.correct ? 'OK' : 'FAIL';
+            const expected = r.shouldTrigger ? 'should trigger' : 'should NOT trigger';
+            const actual = r.didTrigger ? 'triggered' : 'did NOT trigger';
+            this.log(`  [${status}] ${expected} / ${actual}: "${r.query.slice(0, 60)}..."`);
+        }
+    }
+    async improveDescription(target, currentDescription, failures, flags) {
+        const failureText = failures.map((f) => {
+            const expected = f.shouldTrigger ? 'SHOULD trigger but DID NOT' : 'SHOULD NOT trigger but DID';
+            return `- ${expected}: "${f.query}"`;
+        }).join('\n');
+        const prompt = `You are optimizing a skill description for better triggering accuracy.
+
+## Skill Name
+${target.name}
+
+## Current Description
+${currentDescription}
+
+## Skill Content (first 500 chars)
+${target.content.slice(0, 500)}
+
+## Failures
+${failureText}
+
+Propose an improved description that:
+- Fixes the trigger failures above
+- Keeps the description concise but comprehensive
+- Uses specific keywords and contexts for when to trigger
+- Includes "Use PROACTIVELY" phrasing for important trigger contexts
+
+Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;
+        const result = await executeEval({
+            prompt,
+            timeout: flags.timeout,
+            model: flags.model,
+        });
+        return result.output.trim().replace(/^["']|["']$/g, '');
+    }
+}
+//# sourceMappingURL=optimize-description.js.map

package/dist/commands/evals/optimize-description.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize-description.js","sourceRoot":"","sources":["../../../src/commands/evals/optimize-description.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAyBpF,MAAM,CAAC,OAAO,OAAO,8BAA+B,SAAQ,WAAW;IACrE,MAAM,CAAU,WAAW,GAAG,iEAAiE,CAAC;IAEhG,MAAM,CAAU,QAAQ,GAAG;QACzB,0DAA0D;QAC1D,uEAAuE;KACxE,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,iCAAiC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/F,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC;QACjF,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,uCAAuC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC7F,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9F,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,2BAA2B,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;KACpF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACzE,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,GAAG,CAAC,+BAA+B,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEb,mCAAmC;QACnC,IAAI,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEhF,iCAAiC;QACjC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAiB,qBAAqB,IAAI,CAAC,QAAQ,gBAAgB,EAAE;YACpG,OAAO;SACR,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,wCAAwC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QAEtC,4BAA4B;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC5E,MAAM,IAAI,GAAG,MAAM,MAAM,CACvB,qBAAqB,IAAI,CAAC,QAAQ,gCAAgC,CACnE,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,WAAW,CAAC,EAAE,CAAC,CAAC;YAC1D,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,GAAG,IAAI,CAAC;gBAChB,iCAAiC;gBACjC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBACjC,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE5C,oBAAoB;QACpB,MAAM,MAAM,CAAC,sBAAsB,WAAW,CAAC,EAAE,EAAE,EAAE;YACnD,YAAY;YACZ,WAAW;SACZ,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,IAAI,CAAC,GAAG,CAAC,YAAY,YAAY,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,OAAO,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,kBAAkB,GAAG,MAAM,CAAC,WAAW,CAAC;QAC5C,IAAI,eAAe,GAAG,kBAAkB,CAAC;QACzC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,mBAAmB,SAAS,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,iBAAiB,kBAAkB,GAAG,CAAC,CAAC;YAEjD,8CAA8C;YAC9C,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACjC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;YAC9F,MAAM,UAAU,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE9D,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAEvC,mBAAmB;YACnB,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;YAC5F,MAAM,SAAS,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE5D,0BAA0B;YAC1B,MAAM,OAAO,CAAC,wBAAwB,EAAE;gBACtC,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,IAAI,EAAE,aAAa;gBACnB,SAAS;gBACT,YAAY,EAAE,MAAM,CAAC,OAAO;gBAC5B,UAAU;gBACV,SAAS;gBACT,QAAQ,EAAE;oBACR,WAAW,EAAE,kBAAkB;oBAC/B,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACzI,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;iBACxI;gBACD,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,2BAA2B;YAC3B,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;gBAC9B,aAAa,GAAG,SAAS,CAAC;gBAC1B,eAAe,GAAG,kBAAkB,CAAC;YACvC,CAAC;YAED,kBAAkB;YAClB,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;gBAC3C,MAAM;YACR,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClG,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,uBAAuB,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,IAAI,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,YAAY,eAAe,GAAG,CAAC,CAAC;QAEzC,IAAI,eAAe,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,MAAM,CAAC,qBAAqB,MAAM,CAAC,EAAE,EAAE,EAAE;gBAC7C,WAAW,EAAE,eAAe;aAC7B,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,MAAkB,EAClB,KAAa,EACb,KAA0C;QAE1C,qCAAqC;QACrC,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAwB,wBAAwB,CAAC,CAAC;YAC/E,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,WAAW,GAAG,0FAA0F;oBACtG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,qBAAqB,GAAG,KAAK,GAAG,kBAAkB,CAAC;QAEzD,MAAM,MAAM,GAAG,YAAY,KAAK;;;EAGlC,MAAM,CAAC,IAAI;;;EAGX,MAAM,CAAC,WAAW;;;EAGlB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;EAC7B,WAAW;;WAEF,kBAAkB,+BAA+B,qBAAqB;;;;;;;;;;;;;EAa/E,CAAC;QAEC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAChE,IAAI,SAAS;gBAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC9C,IAAI,QAAQ;gBAAE,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAEpC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;YACjC,OAAO;gBACL,EAAE,KAAK,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;gBAC7D,EAAE,KAAK,EAAE,6BAA6B,EAAE,cAAc,EAAE,KAAK,EAAE;aAChE,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAuB,EACvB,MAAkB,EAClB,WAAmB,EACnB,KAAoE;QAEpE,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,4CAA4C;YAC5C,MAAM,IAAI,GAAwB,EAAE,CAAC;YACrC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;oBAC/B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,aAAa,EAAE,CAAC,CAAC,cAAc;oBAC/B,SAAS,EAAE,MAAM,CAAC,IAAI;oBACtB,gBAAgB,EAAE,WAAW;oBAC7B,YAAY,EAAE,MAAM,CAAC,OAAO;oBAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YAED,gBAAgB;YAChB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YAC7D,MAAM,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAE9D,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,aAAa,EAAE,CAAC,CAAC,cAAc;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,cAAc;gBACxC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;gBACxE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,mBAAmB,CAAC,OAA4B;QACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,MAAM,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,oBAAoB,CAAC;YAC3E,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,MAAkB,EAClB,kBAA0B,EAC1B,QAA6B,EAC7B,KAA0C;QAE1C,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,4BAA4B,CAAC;YAC/F,OAAO,KAAK,QAAQ,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC;QACvC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,MAAM,GAAG;;;EAGjB,MAAM,CAAC,IAAI;;;EAGX,kBAAkB;;;EAGlB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;;;EAG5B,WAAW;;;;;;;;mGAQsF,CAAC;QAEhG,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC"}

package/dist/commands/evals/optimize.d.ts

@@ -0,0 +1,22 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalOptimizeCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        auto: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'max-iterations': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        threshold: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'analyzer-model': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'runs-per-eval': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        parallel: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        timeout: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+    private runEvalsOnTarget;
+    private computePassRate;
+}
+//# sourceMappingURL=optimize.d.ts.map

package/dist/commands/evals/optimize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/optimize.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAuBpD,MAAM,CAAC,OAAO,OAAO,mBAAoB,SAAQ,WAAW;IAC1D,OAAgB,WAAW,SAA6E;IAExG,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,IAAI;;MAElB;IAEF,OAAgB,KAAK;;;;;;;;;MASnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YA6JZ,gBAAgB;IAqC9B,OAAO,CAAC,eAAe;CAMxB"}

package/dist/commands/evals/optimize.js

@@ -0,0 +1,200 @@
+import { Args, Flags } from '@oclif/core';
+import { BaseCommand } from '../../base-command.js';
+import { apiGet, apiPost, apiPut } from '../../lib/api-client.js';
+import { executeEval } from '../../lib/eval-executor.js';
+import { gradeEvalOutput } from '../../lib/eval-grader.js';
+import { analyzeEvalResults } from '../../lib/eval-analyzer.js';
+export default class EvalOptimizeCommand extends BaseCommand {
+    static description = 'Iteratively optimize eval target content using eval results as feedback';
+    static examples = [
+        '<%= config.bin %> evals optimize <target-id>',
+        '<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3',
+    ];
+    static args = {
+        targetId: Args.string({ description: 'Eval target ID', required: true }),
+    };
+    static flags = {
+        auto: Flags.boolean({ description: 'Skip interactive review, apply suggestions automatically', default: false }),
+        'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),
+        threshold: Flags.string({ description: 'Stop if pass rate reaches this threshold (0-1)', default: '0.95' }),
+        model: Flags.string({ description: 'Model for eval execution' }),
+        'analyzer-model': Flags.string({ description: 'Model for analysis/suggestions' }),
+        'runs-per-eval': Flags.integer({ description: 'Runs per eval per iteration', default: 1 }),
+        parallel: Flags.integer({ description: 'Number of parallel eval executions', default: 1 }),
+        timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(EvalOptimizeCommand);
+        this.requireAuth();
+        let target = await apiGet(`/api/eval-targets/${args.targetId}`);
+        const allEvals = await apiGet(`/api/eval-targets/${args.targetId}/evals`);
+        if (allEvals.length === 0) {
+            this.error('No evals found. Create some first with `rulemetric evals create`.');
+        }
+        // Split evals into train/test (60/40)
+        const shuffled = [...allEvals].sort(() => Math.random() - 0.5);
+        const splitIdx = Math.ceil(shuffled.length * 0.6);
+        const trainEvals = shuffled.slice(0, splitIdx);
+        const testEvals = shuffled.slice(splitIdx);
+        this.log(`Optimizing "${target.name}" v${target.version}`);
+        this.log(`Evals: ${trainEvals.length} train, ${testEvals.length} test`);
+        const threshold = parseFloat(flags.threshold);
+        this.log(`Config: max ${flags['max-iterations']} iterations, threshold ${(threshold * 100).toFixed(0)}%`);
+        if (flags.auto)
+            this.log('Mode: auto (no interactive review)');
+        this.log('');
+        let bestVersion = target.version;
+        let bestTestScore = 0;
+        let noImprovementCount = 0;
+        for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {
+            this.log(`\n=== Iteration ${iteration} ===`);
+            // Record optimization run as running
+            const optRun = await apiPost('/api/optimization-runs', {
+                evalTargetId: target.id,
+                type: 'content',
+                iteration,
+                inputVersion: target.version,
+                status: 'running',
+            });
+            // Run train evals
+            this.log('Running train evals...');
+            const trainResults = await this.runEvalsOnTarget(target, trainEvals, flags);
+            const trainScore = this.computePassRate(trainResults);
+            this.log(`Train pass rate: ${(trainScore * 100).toFixed(1)}%`);
+            // Check if we've hit the threshold
+            if (trainScore >= threshold) {
+                this.log(`Train score ${(trainScore * 100).toFixed(1)}% >= threshold ${(threshold * 100).toFixed(0)}%. Stopping.`);
+                // Run test evals for final score
+                const testResults = await this.runEvalsOnTarget(target, testEvals, flags);
+                const testScore = this.computePassRate(testResults);
+                this.log(`Final test score: ${(testScore * 100).toFixed(1)}%`);
+                await apiPost('/api/optimization-runs', {
+                    evalTargetId: target.id,
+                    type: 'content',
+                    iteration,
+                    inputVersion: target.version,
+                    trainScore,
+                    testScore,
+                    analysis: { reason: 'threshold_reached' },
+                    status: 'completed',
+                });
+                break;
+            }
+            // Analyze and propose improvements
+            this.log('Analyzing results...');
+            const evalResults = trainResults.map((r) => ({
+                evalName: r.evalName,
+                prompt: r.prompt,
+                expectations: r.expectations,
+                grading: r.grading,
+            }));
+            const analysis = await analyzeEvalResults({
+                content: target.content,
+                evalResults,
+                model: flags['analyzer-model'] ?? flags.model,
+                timeout: flags.timeout,
+            });
+            this.log(`\nAssessment: ${analysis.overallAssessment}`);
+            this.log(`Suggestions: ${analysis.suggestions.length}`);
+            for (const s of analysis.suggestions) {
+                this.log(`  - [${s.section}] ${s.reasoning}`);
+            }
+            // In interactive mode, ask for approval
+            if (!flags.auto) {
+                this.log('\nProposed changes above. Applying in auto mode would proceed automatically.');
+                this.log('(Interactive approval not yet implemented — use --auto for now)');
+            }
+            if (analysis.proposedContent === target.content) {
+                this.log('No changes proposed. Stopping.');
+                await apiPost('/api/optimization-runs', {
+                    evalTargetId: target.id,
+                    type: 'content',
+                    iteration,
+                    inputVersion: target.version,
+                    trainScore,
+                    analysis: { reason: 'no_changes' },
+                    status: 'completed',
+                });
+                break;
+            }
+            // Create new version
+            const newTarget = await apiPut(`/api/eval-targets/${target.id}`, {
+                content: analysis.proposedContent,
+            });
+            this.log(`Created v${newTarget.version}`);
+            // Run test evals on new version
+            this.log('Running test evals...');
+            const testResults = await this.runEvalsOnTarget(newTarget, testEvals, flags);
+            const testScore = this.computePassRate(testResults);
+            this.log(`Test pass rate: ${(testScore * 100).toFixed(1)}%`);
+            // Update optimization run
+            await apiPost('/api/optimization-runs', {
+                evalTargetId: newTarget.id,
+                type: 'content',
+                iteration,
+                inputVersion: target.version,
+                outputVersion: newTarget.version,
+                trainScore,
+                testScore,
+                analysis: {
+                    suggestions: analysis.suggestions,
+                    overallAssessment: analysis.overallAssessment,
+                },
+                status: 'completed',
+            });
+            // Track best
+            if (testScore > bestTestScore) {
+                bestTestScore = testScore;
+                bestVersion = newTarget.version;
+                noImprovementCount = 0;
+            }
+            else {
+                noImprovementCount++;
+                if (noImprovementCount >= 2) {
+                    this.log('No improvement for 2 consecutive iterations. Stopping.');
+                    break;
+                }
+            }
+            target = newTarget;
+        }
+        this.log(`\n=== Optimization Complete ===`);
+        this.log(`Best version: v${bestVersion} (test score: ${(bestTestScore * 100).toFixed(1)}%)`);
+    }
+    async runEvalsOnTarget(target, evalSet, flags) {
+        const results = [];
+        for (const eval_ of evalSet) {
+            const execResult = await executeEval({
+                prompt: eval_.prompt,
+                targetContent: target.content,
+                timeout: flags.timeout,
+                model: flags.model,
+            });
+            if (execResult.error) {
+                results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
+                continue;
+            }
+            if (eval_.expectations.length > 0) {
+                const grading = await gradeEvalOutput({
+                    prompt: eval_.prompt,
+                    output: execResult.output,
+                    expectations: eval_.expectations,
+                    model: flags.model,
+                    timeout: flags.timeout,
+                });
+                results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: grading });
+            }
+            else {
+                results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
+            }
+        }
+        return results;
+    }
+    computePassRate(results) {
+        const graded = results.filter((r) => Object.keys(r.grading).length > 0);
+        if (graded.length === 0)
+            return 0;
+        const rates = graded.map((r) => r.grading.summary?.pass_rate ?? 0);
+        return rates.reduce((a, b) => a + b, 0) / rates.length;
+    }
+}
+//# sourceMappingURL=optimize.js.map

package/dist/commands/evals/optimize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize.js","sourceRoot":"","sources":["../../../src/commands/evals/optimize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAmBhE,MAAM,CAAC,OAAO,OAAO,mBAAoB,SAAQ,WAAW;IAC1D,MAAM,CAAU,WAAW,GAAG,yEAAyE,CAAC;IAExG,MAAM,CAAU,QAAQ,GAAG;QACzB,8CAA8C;QAC9C,wEAAwE;KACzE,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,0DAA0D,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAChH,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,iCAAiC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/F,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gDAAgD,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAC3G,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,0BAA0B,EAAE,CAAC;QAChE,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,CAAC;QACjF,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,6BAA6B,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1F,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,oCAAoC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC1F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;KACpF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC9D,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,IAAI,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAS,qBAAqB,IAAI,CAAC,QAAQ,QAAQ,CAAC,CAAC;QAElF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAClF,CAAC;QAED,sCAAsC;QACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE3C,IAAI,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,CAAC,UAAU,UAAU,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,OAAO,CAAC,CAAC;QACxE,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,eAAe,KAAK,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1G,IAAI,KAAK,CAAC,IAAI;YAAE,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC/D,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEb,IAAI,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC;QACjC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAE3B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,mBAAmB,SAAS,MAAM,CAAC,CAAC;YAE7C,qCAAqC;YACrC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAiB,wBAAwB,EAAE;gBACrE,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,IAAI,EAAE,SAAS;gBACf,SAAS;gBACT,YAAY,EAAE,MAAM,CAAC,OAAO;gBAC5B,MAAM,EAAE,SAAS;aAClB,CAAC,CAAC;YAEH,kBAAkB;YAClB,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACnC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YACtD,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE/D,mCAAmC;YACnC,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;gBAC5B,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;gBACnH,iCAAiC;gBACjC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;gBAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;gBACpD,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAE/D,MAAM,OAAO,CAAC,wBAAwB,EAAE;oBACtC,YAAY,EAAE,MAAM,CAAC,EAAE;oBACvB,IAAI,EAAE,SAAS;oBACf,SAAS;oBACT,YAAY,EAAE,MAAM,CAAC,OAAO;oBAC5B,UAAU;oBACV,SAAS;oBACT,QAAQ,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE;oBACzC,MAAM,EAAE,WAAW;iBACpB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,mCAAmC;YACnC,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACjC,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC,CAAC;YAEJ,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC;gBACxC,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,WAAW;gBACX,KAAK,EAAE,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,KAAK;gBAC7C,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,iBAAiB,EAAE,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YAChD,CAAC;YAED,wCAAwC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,IAAI,CAAC,GAAG,CAAC,8EAA8E,CAAC,CAAC;gBACzF,IAAI,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,QAAQ,CAAC,eAAe,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC;gBAChD,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;gBAC3C,MAAM,OAAO,CAAC,wBAAwB,EAAE;oBACtC,YAAY,EAAE,MAAM,CAAC,EAAE;oBACvB,IAAI,EAAE,SAAS;oBACf,SAAS;oBACT,YAAY,EAAE,MAAM,CAAC,OAAO;oBAC5B,UAAU;oBACV,QAAQ,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE;oBAClC,MAAM,EAAE,WAAW;iBACpB,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,qBAAqB;YACrB,MAAM,SAAS,GAAG,MAAM,MAAM,CAAa,qBAAqB,MAAM,CAAC,EAAE,EAAE,EAAE;gBAC3E,OAAO,EAAE,QAAQ,CAAC,eAAe;aAClC,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,YAAY,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAE1C,gCAAgC;YAChC,IAAI,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YACpD,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE7D,0BAA0B;YAC1B,MAAM,OAAO,CAAC,wBAAwB,EAAE;gBACtC,YAAY,EAAE,SAAS,CAAC,EAAE;gBAC1B,IAAI,EAAE,SAAS;gBACf,SAAS;gBACT,YAAY,EAAE,MAAM,CAAC,OAAO;gBAC5B,aAAa,EAAE,SAAS,CAAC,OAAO;gBAChC,UAAU;gBACV,SAAS;gBACT,QAAQ,EAAE;oBACR,WAAW,EAAE,QAAQ,CAAC,WAAW;oBACjC,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;iBAC9C;gBACD,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,aAAa;YACb,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;gBAC9B,aAAa,GAAG,SAAS,CAAC;gBAC1B,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC;gBAChC,kBAAkB,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,kBAAkB,EAAE,CAAC;gBACrB,IAAI,kBAAkB,IAAI,CAAC,EAAE,CAAC;oBAC5B,IAAI,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;oBACnE,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,GAAG,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,kBAAkB,WAAW,iBAAiB,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/F,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAkB,EAClB,OAAe,EACf,KAAqF;QAErF,MAAM,OAAO,GAA0G,EAAE,CAAC;QAE1H,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;gBACnC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,aAAa,EAAE,MAAM,CAAC,OAAO;gBAC7B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YAEH,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC5G,SAAS;YACX,CAAC;YAED,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC;oBACpC,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,EAAE,OAA6C,EAAE,CAAC,CAAC;YACzJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9G,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,eAAe,CAAC,OAAoD;QAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACxE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,OAAe,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;QAC5E,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACzE,CAAC"}

package/dist/commands/evals/run.d.ts

@@ -0,0 +1,21 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalRunCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        parallel: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'runs-per-eval': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        iteration: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'skip-baseline': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'skip-grading': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'grader-model': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        timeout: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+    private executeAndGrade;
+}
+//# sourceMappingURL=run.d.ts.map

package/dist/commands/evals/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAkCpD,MAAM,CAAC,OAAO,OAAO,cAAe,SAAQ,WAAW;IACrD,OAAgB,WAAW,SAA2E;IAEtG,OAAgB,QAAQ,WAItB;IAEF,OAAgB,IAAI;;MAElB;IAEF,OAAgB,KAAK;;;;;;;;;MASnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YA4GZ,eAAe;CAuE9B"}