@rulemetric/cli 0.2.1 → 0.2.2

package/dist/commands/evals/optimize-description.js

@@ -1,150 +1,156 @@
-import { Args, Flags } from '@oclif/core';
-import { BaseCommand } from '../../base-command.js';
-import { apiGet, apiPost, apiPut } from '../../lib/api-client.js';
-import { executeEval } from '../../lib/eval-executor.js';
-import { testTrigger, computeTriggerScore } from '../../lib/eval-trigger-tester.js';
-export default class EvalOptimizeDescriptionCommand extends BaseCommand {
-    static description = 'Optimize eval target description for better triggering accuracy';
-    static examples = [
-        '<%= config.bin %> evals optimize-description <target-id>',
-        '<%= config.bin %> evals optimize-description <target-id> --queries 10',
-    ];
-    static args = {
-        targetId: Args.string({ description: 'Eval target ID', required: true }),
-    };
-    static flags = {
-        'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),
-        model: Flags.string({ description: 'Model for trigger testing and improvement' }),
-        queries: Flags.integer({ description: 'Number of trigger queries to generate', default: 20 }),
-        'runs-per-query': Flags.integer({ description: 'Runs per query for reliability', default: 3 }),
-        'poll-interval': Flags.integer({ description: 'Ms between approval polls', default: 5000 }),
-        timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),
-    };
-    async run() {
-        const { args, flags } = await this.parse(EvalOptimizeDescriptionCommand);
-        this.requireAuth();
-        const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-        this.log(`Optimizing description for "${target.name}" v${target.version}`);
-        this.log(`Current description: "${target.description}"`);
-        this.log('');
-        // Step 1: Generate trigger queries
-        this.log('Generating trigger queries...');
-        const queries = await this.generateTriggerQueries(target, flags.queries, flags);
-        // Step 2: Post to API for review
-        const triggerEval = await apiPost(`/api/eval-targets/${args.targetId}/trigger-evals`, {
-            queries,
-        });
-        this.log(`\nGenerated ${queries.length} trigger queries.`);
-        this.log(`Review and approve them at:`);
-        this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);
-        this.log(`  (Triggers tab → click to review)`);
-        this.log('\nWaiting for approval...');
-        // Step 3: Poll for approval
-        let approved = false;
-        while (!approved) {
-            await new Promise((resolve) => setTimeout(resolve, flags['poll-interval']));
-            const sets = await apiGet(`/api/eval-targets/${args.targetId}/trigger-evals?status=approved`);
-            const current = sets.find((s) => s.id === triggerEval.id);
-            if (current) {
-                approved = true;
-                // Use potentially edited queries
-                queries.length = 0;
-                queries.push(...current.queries);
-                this.log('Trigger queries approved!');
-            }
-        }
-        // Step 4: Split train/test (60/40)
-        const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);
-        const splitIdx = Math.ceil(indices.length * 0.6);
-        const trainIndices = indices.slice(0, splitIdx);
-        const testIndices = indices.slice(splitIdx);
-        // Save split to API
-        await apiPut(`/api/trigger-evals/${triggerEval.id}`, {
-            trainIndices,
-            testIndices,
-        });
-        const trainQueries = trainIndices.map((i) => queries[i]);
-        const testQueries = testIndices.map((i) => queries[i]);
-        this.log(`\nSplit: ${trainQueries.length} train, ${testQueries.length} test`);
-        // Step 5: Optimization loop
-        let currentDescription = target.description;
-        let bestDescription = currentDescription;
-        let bestTestScore = 0;
-        for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {
-            this.log(`\n=== Iteration ${iteration} ===`);
-            this.log(`Description: "${currentDescription}"`);
-            // Test on train set (multiple runs per query)
-            this.log('Testing train set...');
-            const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);
-            const trainScore = computeTriggerScore(trainResults);
-            this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);
-            this.printTriggerResults(trainResults);
-            // Test on test set
-            this.log('Testing test set...');
-            const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);
-            const testScore = computeTriggerScore(testResults);
-            this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);
-            // Record optimization run
-            await apiPost('/api/optimization-runs', {
-                evalTargetId: target.id,
-                type: 'description',
-                iteration,
-                inputVersion: target.version,
-                trainScore,
-                testScore,
-                analysis: {
-                    description: currentDescription,
-                    trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
-                    testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
-                },
-                status: 'completed',
-            });
-            // Track best by test score
-            if (testScore > bestTestScore) {
-                bestTestScore = testScore;
-                bestDescription = currentDescription;
-            }
-            // Stop if perfect
-            if (trainScore >= 1.0) {
-                this.log('Perfect train score. Stopping.');
-                break;
-            }
-            // Propose improved description
-            this.log('Generating improved description...');
-            const failures = trainResults.filter((r) => !r.correct);
-            currentDescription = await this.improveDescription(target, currentDescription, failures, flags);
-        }
-        // Step 6: Apply best description
-        this.log(`\n=== Optimization Complete ===`);
-        this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);
-        this.log(`\nBefore: "${target.description}"`);
-        this.log(`After:  "${bestDescription}"`);
-        if (bestDescription !== target.description) {
-            await apiPut(`/api/eval-targets/${target.id}`, {
-                description: bestDescription,
-            });
-            this.log('\nDescription updated.');
-        }
-        else {
-            this.log('\nNo improvement found. Description unchanged.');
-        }
+import {
+  computeTriggerScore,
+  testTrigger
+} from "../../chunk-4D537GXE.js";
+import {
+  executeEval
+} from "../../chunk-QN34TGD3.js";
+import {
+  BaseCommand
+} from "../../chunk-S56LOTGQ.js";
+import "../../chunk-OGVORIFM.js";
+import "../../chunk-2AM54TUK.js";
+import {
+  apiGet,
+  apiPost,
+  apiPut
+} from "../../chunk-EJOI2ZGU.js";
+import "../../chunk-KZRFUXGF.js";
+import "../../chunk-EKJ2CABV.js";
+import "../../chunk-NSBPE2FW.js";
+
+// src/commands/evals/optimize-description.ts
+import { Args, Flags } from "@oclif/core";
+var EvalOptimizeDescriptionCommand = class _EvalOptimizeDescriptionCommand extends BaseCommand {
+  static description = "Optimize eval target description for better triggering accuracy";
+  static examples = [
+    "<%= config.bin %> evals optimize-description <target-id>",
+    "<%= config.bin %> evals optimize-description <target-id> --queries 10"
+  ];
+  static args = {
+    targetId: Args.string({ description: "Eval target ID", required: true })
+  };
+  static flags = {
+    "max-iterations": Flags.integer({ description: "Maximum optimization iterations", default: 5 }),
+    model: Flags.string({ description: "Model for trigger testing and improvement" }),
+    queries: Flags.integer({ description: "Number of trigger queries to generate", default: 20 }),
+    "runs-per-query": Flags.integer({ description: "Runs per query for reliability", default: 3 }),
+    "poll-interval": Flags.integer({ description: "Ms between approval polls", default: 5e3 }),
+    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
+  };
+  async run() {
+    const { args, flags } = await this.parse(_EvalOptimizeDescriptionCommand);
+    this.requireAuth();
+    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
+    this.log(`Optimizing description for "${target.name}" v${target.version}`);
+    this.log(`Current description: "${target.description}"`);
+    this.log("");
+    this.log("Generating trigger queries...");
+    const queries = await this.generateTriggerQueries(target, flags.queries, flags);
+    const triggerEval = await apiPost(`/api/eval-targets/${args.targetId}/trigger-evals`, {
+      queries
+    });
+    this.log(`
+Generated ${queries.length} trigger queries.`);
+    this.log(`Review and approve them at:`);
+    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);
+    this.log(`  (Triggers tab \u2192 click to review)`);
+    this.log("\nWaiting for approval...");
+    let approved = false;
+    while (!approved) {
+      await new Promise((resolve) => setTimeout(resolve, flags["poll-interval"]));
+      const sets = await apiGet(
+        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`
+      );
+      const current = sets.find((s) => s.id === triggerEval.id);
+      if (current) {
+        approved = true;
+        queries.length = 0;
+        queries.push(...current.queries);
+        this.log("Trigger queries approved!");
+      }
+    }
+    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);
+    const splitIdx = Math.ceil(indices.length * 0.6);
+    const trainIndices = indices.slice(0, splitIdx);
+    const testIndices = indices.slice(splitIdx);
+    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {
+      trainIndices,
+      testIndices
+    });
+    const trainQueries = trainIndices.map((i) => queries[i]);
+    const testQueries = testIndices.map((i) => queries[i]);
+    this.log(`
+Split: ${trainQueries.length} train, ${testQueries.length} test`);
+    let currentDescription = target.description;
+    let bestDescription = currentDescription;
+    let bestTestScore = 0;
+    for (let iteration = 1; iteration <= flags["max-iterations"]; iteration++) {
+      this.log(`
+=== Iteration ${iteration} ===`);
+      this.log(`Description: "${currentDescription}"`);
+      this.log("Testing train set...");
+      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);
+      const trainScore = computeTriggerScore(trainResults);
+      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);
+      this.printTriggerResults(trainResults);
+      this.log("Testing test set...");
+      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);
+      const testScore = computeTriggerScore(testResults);
+      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);
+      await apiPost("/api/optimization-runs", {
+        evalTargetId: target.id,
+        type: "description",
+        iteration,
+        inputVersion: target.version,
+        trainScore,
+        testScore,
+        analysis: {
+          description: currentDescription,
+          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
+          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct }))
+        },
+        status: "completed"
+      });
+      if (testScore > bestTestScore) {
+        bestTestScore = testScore;
+        bestDescription = currentDescription;
+      }
+      if (trainScore >= 1) {
+        this.log("Perfect train score. Stopping.");
+        break;
+      }
+      this.log("Generating improved description...");
+      const failures = trainResults.filter((r) => !r.correct);
+      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);
+    }
+    this.log(`
+=== Optimization Complete ===`);
+    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);
+    this.log(`
+Before: "${target.description}"`);
+    this.log(`After:  "${bestDescription}"`);
+    if (bestDescription !== target.description) {
+      await apiPut(`/api/eval-targets/${target.id}`, {
+        description: bestDescription
+      });
+      this.log("\nDescription updated.");
+    } else {
+      this.log("\nNo improvement found. Description unchanged.");
     }
-    async generateTriggerQueries(target, count, flags) {
-        // Try to seed with real session data
-        let sessionSeed = '';
-        try {
-            const sessions = await apiGet('/api/sessions?limit=10');
-            if (sessions.length > 0) {
-                sessionSeed = '\n\nReal user prompts from recent sessions (use as inspiration for realistic queries):\n' +
-                    sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join('\n');
-            }
-        }
-        catch {
-            // No sessions available, that's fine
-        }
-        const shouldTriggerCount = Math.ceil(count / 2);
-        const shouldNotTriggerCount = count - shouldTriggerCount;
-        const prompt = `Generate ${count} trigger eval queries for a skill/instruction.
+  }
+  async generateTriggerQueries(target, count, flags) {
+    let sessionSeed = "";
+    try {
+      const sessions = await apiGet("/api/sessions?limit=10");
+      if (sessions.length > 0) {
+        sessionSeed = "\n\nReal user prompts from recent sessions (use as inspiration for realistic queries):\n" + sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join("\n");
+      }
+    } catch {
+    }
+    const shouldTriggerCount = Math.ceil(count / 2);
+    const shouldNotTriggerCount = count - shouldTriggerCount;
+    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.
 
 ## Skill Name
 ${target.name}
@@ -153,13 +159,13 @@ ${target.name}
 ${target.description}
 
 ## Skill Content (first 1000 chars)
-${target.content.slice(0, 1000)}
+${target.content.slice(0, 1e3)}
 ${sessionSeed}
 
 Generate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.
 
 Rules:
-- Queries must be realistic — what a real user would actually type
+- Queries must be realistic \u2014 what a real user would actually type
 - Include concrete details (file paths, project names, specific problems)
 - Vary length and formality
 - Should-not-trigger queries should be near-misses (share keywords but need something different)
@@ -170,74 +176,68 @@ Respond with ONLY a valid JSON array:
   {"query": "realistic user prompt...", "should_trigger": true},
   {"query": "another prompt...", "should_trigger": false}
 ]`;
-        const result = await executeEval({
-            prompt,
-            timeout: flags.timeout,
-            model: flags.model,
-        });
-        try {
-            let jsonStr = result.output.trim();
-            const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-            if (jsonMatch)
-                jsonStr = jsonMatch[1].trim();
-            const arrMatch = jsonStr.match(/\[[\s\S]*\]/);
-            if (arrMatch)
-                jsonStr = arrMatch[0];
-            return JSON.parse(jsonStr);
-        }
-        catch {
-            // Fallback: generate minimal set
-            return [
-                { query: `Help me use ${target.name}`, should_trigger: true },
-                { query: 'Write a hello world program', should_trigger: false },
-            ];
-        }
+    const result = await executeEval({
+      prompt,
+      timeout: flags.timeout,
+      model: flags.model
+    });
+    try {
+      let jsonStr = result.output.trim();
+      const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+      if (jsonMatch) jsonStr = jsonMatch[1].trim();
+      const arrMatch = jsonStr.match(/\[[\s\S]*\]/);
+      if (arrMatch) jsonStr = arrMatch[0];
+      return JSON.parse(jsonStr);
+    } catch {
+      return [
+        { query: `Help me use ${target.name}`, should_trigger: true },
+        { query: "Write a hello world program", should_trigger: false }
+      ];
     }
-    async testQuerySet(queries, target, description, flags) {
-        const results = [];
-        for (const q of queries) {
-            // Run multiple times and take majority vote
-            const runs = [];
-            for (let run = 0; run < flags['runs-per-query']; run++) {
-                const result = await testTrigger({
-                    query: q.query,
-                    shouldTrigger: q.should_trigger,
-                    skillName: target.name,
-                    skillDescription: description,
-                    skillContent: target.content,
-                    model: flags.model,
-                    timeout: flags.timeout,
-                });
-                runs.push(result);
-            }
-            // Majority vote
-            const triggerCount = runs.filter((r) => r.didTrigger).length;
-            const didTrigger = triggerCount > flags['runs-per-query'] / 2;
-            results.push({
-                query: q.query,
-                shouldTrigger: q.should_trigger,
-                didTrigger,
-                correct: didTrigger === q.should_trigger,
-                confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,
-                evidence: runs[0].evidence,
-            });
-        }
-        return results;
+  }
+  async testQuerySet(queries, target, description, flags) {
+    const results = [];
+    for (const q of queries) {
+      const runs = [];
+      for (let run = 0; run < flags["runs-per-query"]; run++) {
+        const result = await testTrigger({
+          query: q.query,
+          shouldTrigger: q.should_trigger,
+          skillName: target.name,
+          skillDescription: description,
+          skillContent: target.content,
+          model: flags.model,
+          timeout: flags.timeout
+        });
+        runs.push(result);
+      }
+      const triggerCount = runs.filter((r) => r.didTrigger).length;
+      const didTrigger = triggerCount > flags["runs-per-query"] / 2;
+      results.push({
+        query: q.query,
+        shouldTrigger: q.should_trigger,
+        didTrigger,
+        correct: didTrigger === q.should_trigger,
+        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,
+        evidence: runs[0].evidence
+      });
     }
-    printTriggerResults(results) {
-        for (const r of results) {
-            const status = r.correct ? 'OK' : 'FAIL';
-            const expected = r.shouldTrigger ? 'should trigger' : 'should NOT trigger';
-            const actual = r.didTrigger ? 'triggered' : 'did NOT trigger';
-            this.log(`  [${status}] ${expected} / ${actual}: "${r.query.slice(0, 60)}..."`);
-        }
+    return results;
+  }
+  printTriggerResults(results) {
+    for (const r of results) {
+      const status = r.correct ? "OK" : "FAIL";
+      const expected = r.shouldTrigger ? "should trigger" : "should NOT trigger";
+      const actual = r.didTrigger ? "triggered" : "did NOT trigger";
+      this.log(`  [${status}] ${expected} / ${actual}: "${r.query.slice(0, 60)}..."`);
     }
-    async improveDescription(target, currentDescription, failures, flags) {
-        const failureText = failures.map((f) => {
-            const expected = f.shouldTrigger ? 'SHOULD trigger but DID NOT' : 'SHOULD NOT trigger but DID';
-            return `- ${expected}: "${f.query}"`;
-        }).join('\n');
-        const prompt = `You are optimizing a skill description for better triggering accuracy.
+  }
+  async improveDescription(target, currentDescription, failures, flags) {
+    const failureText = failures.map((f) => {
+      const expected = f.shouldTrigger ? "SHOULD trigger but DID NOT" : "SHOULD NOT trigger but DID";
+      return `- ${expected}: "${f.query}"`;
+    }).join("\n");
+    const prompt = `You are optimizing a skill description for better triggering accuracy.
 
 ## Skill Name
 ${target.name}
@@ -258,12 +258,15 @@ Propose an improved description that:
 - Includes "Use PROACTIVELY" phrasing for important trigger contexts
 
 Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;
-        const result = await executeEval({
-            prompt,
-            timeout: flags.timeout,
-            model: flags.model,
-        });
-        return result.output.trim().replace(/^["']|["']$/g, '');
-    }
-}
-//# sourceMappingURL=optimize-description.js.map
+    const result = await executeEval({
+      prompt,
+      timeout: flags.timeout,
+      model: flags.model
+    });
+    return result.output.trim().replace(/^["']|["']$/g, "");
+  }
+};
+export {
+  EvalOptimizeDescriptionCommand as default
+};
+//# sourceMappingURL=optimize-description.js.map

package/dist/commands/evals/optimize-description.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"optimize-description.js","sourceRoot":"","sources":["../../../src/commands/evals/optimize-description.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAyBpF,MAAM,CAAC,OAAO,OAAO,8BAA+B,SAAQ,WAAW;IACrE,MAAM,CAAU,WAAW,GAAG,iEAAiE,CAAC;IAEhG,MAAM,CAAU,QAAQ,GAAG;QACzB,0DAA0D;QAC1D,uEAAuE;KACxE,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,iCAAiC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/F,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC;QACjF,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,uCAAuC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC7F,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,gCAAgC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAC9F,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,2BAA2B,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3F,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;KACpF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QACzE,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,GAAG,CAAC,+BAA+B,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QACzD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEb,mCAAmC;QACnC,IAAI,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEhF,iCAAiC;QACjC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAiB,qBAAqB,IAAI,CAAC,QAAQ,gBAAgB,EAAE;YACpG,OAAO;SACR,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAC3D,IAAI,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,wCAAwC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QAEtC,4BAA4B;QAC5B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC5E,MAAM,IAAI,GAAG,MAAM,MAAM,CACvB,qBAAqB,IAAI,CAAC,QAAQ,gCAAgC,CACnE,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,WAAW,CAAC,EAAE,CAAC,CAAC;YAC1D,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,GAAG,IAAI,CAAC;gBAChB,iCAAiC;gBACjC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBACjC,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE5C,oBAAoB;QACpB,MAAM,MAAM,CAAC,sBAAsB,WAAW,CAAC,EAAE,EAAE,EAAE;YACnD,YAAY;YACZ,WAAW;SACZ,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvD,IAAI,CAAC,GAAG,CAAC,YAAY,YAAY,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,OAAO,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,kBAAkB,GAAG,MAAM,CAAC,WAAW,CAAC;QAC5C,IAAI,eAAe,GAAG,kBAAkB,CAAC;QACzC,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC,GAAG,CAAC,mBAAmB,SAAS,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,iBAAiB,kBAAkB,GAAG,CAAC,CAAC;YAEjD,8CAA8C;YAC9C,IAAI,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACjC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;YAC9F,MAAM,UAAU,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAC;YACrD,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE9D,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAEvC,mBAAmB;YACnB,IAAI,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;YAC5F,MAAM,SAAS,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAE5D,0BAA0B;YAC1B,MAAM,OAAO,CAAC,wBAAwB,EAAE;gBACtC,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,IAAI,EAAE,aAAa;gBACnB,SAAS;gBACT,YAAY,EAAE,MAAM,CAAC,OAAO;gBAC5B,UAAU;gBACV,SAAS;gBACT,QAAQ,EAAE;oBACR,WAAW,EAAE,kBAAkB;oBAC/B,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;oBACzI,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;iBACxI;gBACD,MAAM,EAAE,WAAW;aACpB,CAAC,CAAC;YAEH,2BAA2B;YAC3B,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;gBAC9B,aAAa,GAAG,SAAS,CAAC;gBAC1B,eAAe,GAAG,kBAAkB,CAAC;YACvC,CAAC;YAED,kBAAkB;YAClB,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;gBAC3C,MAAM;YACR,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxD,kBAAkB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAClG,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,uBAAuB,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,IAAI,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,YAAY,eAAe,GAAG,CAAC,CAAC;QAEzC,IAAI,eAAe,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,MAAM,CAAC,qBAAqB,MAAM,CAAC,EAAE,EAAE,EAAE;gBAC7C,WAAW,EAAE,eAAe;aAC7B,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,MAAkB,EAClB,KAAa,EACb,KAA0C;QAE1C,qCAAqC;QACrC,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAwB,wBAAwB,CAAC,CAAC;YAC/E,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,WAAW,GAAG,0FAA0F;oBACtG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,qBAAqB,GAAG,KAAK,GAAG,kBAAkB,CAAC;QAEzD,MAAM,MAAM,GAAG,YAAY,KAAK;;;EAGlC,MAAM,CAAC,IAAI;;;EAGX,MAAM,CAAC,WAAW;;;EAGlB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;EAC7B,WAAW;;WAEF,kBAAkB,+BAA+B,qBAAqB;;;;;;;;;;;;;EAa/E,CAAC;QAEC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAChE,IAAI,SAAS;gBAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC9C,IAAI,QAAQ;gBAAE,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YAEpC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;YACjC,OAAO;gBACL,EAAE,KAAK,EAAE,eAAe,MAAM,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;gBAC7D,EAAE,KAAK,EAAE,6BAA6B,EAAE,cAAc,EAAE,KAAK,EAAE;aAChE,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAuB,EACvB,MAAkB,EAClB,WAAmB,EACnB,KAAoE;QAEpE,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,4CAA4C;YAC5C,MAAM,IAAI,GAAwB,EAAE,CAAC;YACrC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;oBAC/B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,aAAa,EAAE,CAAC,CAAC,cAAc;oBAC/B,SAAS,EAAE,MAAM,CAAC,IAAI;oBACtB,gBAAgB,EAAE,WAAW;oBAC7B,YAAY,EAAE,MAAM,CAAC,OAAO;oBAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YAED,gBAAgB;YAChB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;YAC7D,MAAM,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAE9D,OAAO,CAAC,IAAI,CAAC;gBACX,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,aAAa,EAAE,CAAC,CAAC,cAAc;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,cAAc;gBACxC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;gBACxE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,mBAAmB,CAAC,OAA4B;QACtD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,MAAM,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,oBAAoB,CAAC;YAC3E,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,MAAkB,EAClB,kBAA0B,EAC1B,QAA6B,EAC7B,KAA0C;QAE1C,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,4BAA4B,CAAC;YAC/F,OAAO,KAAK,QAAQ,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC;QACvC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,MAAM,GAAG;;;EAGjB,MAAM,CAAC,IAAI;;;EAGX,kBAAkB;;;EAGlB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;;;EAG5B,WAAW;;;;;;;;mGAQsF,CAAC;QAEhG,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC"}
+{
+  "version": 3,
+  "sources": ["../../../src/commands/evals/optimize-description.ts"],
+  "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost, apiPut } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { testTrigger, computeTriggerScore } from '../../lib/eval-trigger-tester.js';\nimport type { TriggerTestResult } from '../../lib/eval-trigger-tester.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  description: string;\n  version: number;\n}\n\ninterface TriggerQuery {\n  query: string;\n  should_trigger: boolean;\n}\n\ninterface TriggerEvalSet {\n  id: string;\n  queries: TriggerQuery[];\n  status: string;\n  trainIndices: number[];\n  testIndices: number[];\n}\n\nexport default class EvalOptimizeDescriptionCommand extends BaseCommand {\n  static override description = 'Optimize eval target description for better triggering accuracy';\n\n  static override examples = [\n    '<%= config.bin %> evals optimize-description <target-id>',\n    '<%= config.bin %> evals optimize-description <target-id> --queries 10',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),\n    model: Flags.string({ description: 'Model for trigger testing and improvement' }),\n    queries: Flags.integer({ description: 'Number of trigger queries to generate', default: 20 }),\n    'runs-per-query': Flags.integer({ description: 'Runs per query for reliability', default: 3 }),\n    'poll-interval': Flags.integer({ description: 'Ms between approval polls', default: 5000 }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalOptimizeDescriptionCommand);\n    this.requireAuth();\n\n    const target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n\n    this.log(`Optimizing description for \"${target.name}\" v${target.version}`);\n    this.log(`Current description: \"${target.description}\"`);\n    this.log('');\n\n    // Step 1: Generate trigger queries\n    this.log('Generating trigger queries...');\n    const queries = await this.generateTriggerQueries(target, flags.queries, flags);\n\n    // Step 2: Post to API for review\n    const triggerEval = await apiPost<TriggerEvalSet>(`/api/eval-targets/${args.targetId}/trigger-evals`, {\n      queries,\n    });\n\n    this.log(`\\nGenerated ${queries.length} trigger queries.`);\n    this.log(`Review and approve them at:`);\n    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);\n    this.log(`  (Triggers tab \u2192 click to review)`);\n    this.log('\\nWaiting for approval...');\n\n    // Step 3: Poll for approval\n    let approved = false;\n    while (!approved) {\n      await new Promise((resolve) => setTimeout(resolve, flags['poll-interval']));\n      const sets = await apiGet<TriggerEvalSet[]>(\n        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`,\n      );\n      const current = sets.find((s) => s.id === triggerEval.id);\n      if (current) {\n        approved = true;\n        // Use potentially edited queries\n        queries.length = 0;\n        queries.push(...current.queries);\n        this.log('Trigger queries approved!');\n      }\n    }\n\n    // Step 4: Split train/test (60/40)\n    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);\n    const splitIdx = Math.ceil(indices.length * 0.6);\n    const trainIndices = indices.slice(0, splitIdx);\n    const testIndices = indices.slice(splitIdx);\n\n    // Save split to API\n    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {\n      trainIndices,\n      testIndices,\n    });\n\n    const trainQueries = trainIndices.map((i) => queries[i]);\n    const testQueries = testIndices.map((i) => queries[i]);\n\n    this.log(`\\nSplit: ${trainQueries.length} train, ${testQueries.length} test`);\n\n    // Step 5: Optimization loop\n    let currentDescription = target.description;\n    let bestDescription = currentDescription;\n    let bestTestScore = 0;\n\n    for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {\n      this.log(`\\n=== Iteration ${iteration} ===`);\n      this.log(`Description: \"${currentDescription}\"`);\n\n      // Test on train set (multiple runs per query)\n      this.log('Testing train set...');\n      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);\n      const trainScore = computeTriggerScore(trainResults);\n      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);\n\n      this.printTriggerResults(trainResults);\n\n      // Test on test set\n      this.log('Testing test set...');\n      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);\n      const testScore = computeTriggerScore(testResults);\n      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);\n\n      // Record optimization run\n      await apiPost('/api/optimization-runs', {\n        evalTargetId: target.id,\n        type: 'description',\n        iteration,\n        inputVersion: target.version,\n        trainScore,\n        testScore,\n        analysis: {\n          description: currentDescription,\n          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n        },\n        status: 'completed',\n      });\n\n      // Track best by test score\n      if (testScore > bestTestScore) {\n        bestTestScore = testScore;\n        bestDescription = currentDescription;\n      }\n\n      // Stop if perfect\n      if (trainScore >= 1.0) {\n        this.log('Perfect train score. Stopping.');\n        break;\n      }\n\n      // Propose improved description\n      this.log('Generating improved description...');\n      const failures = trainResults.filter((r) => !r.correct);\n      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);\n    }\n\n    // Step 6: Apply best description\n    this.log(`\\n=== Optimization Complete ===`);\n    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);\n    this.log(`\\nBefore: \"${target.description}\"`);\n    this.log(`After:  \"${bestDescription}\"`);\n\n    if (bestDescription !== target.description) {\n      await apiPut(`/api/eval-targets/${target.id}`, {\n        description: bestDescription,\n      });\n      this.log('\\nDescription updated.');\n    } else {\n      this.log('\\nNo improvement found. Description unchanged.');\n    }\n  }\n\n  private async generateTriggerQueries(\n    target: EvalTarget,\n    count: number,\n    flags: { model?: string; timeout: number },\n  ): Promise<TriggerQuery[]> {\n    // Try to seed with real session data\n    let sessionSeed = '';\n    try {\n      const sessions = await apiGet<Array<{ id: string }>>('/api/sessions?limit=10');\n      if (sessions.length > 0) {\n        sessionSeed = '\\n\\nReal user prompts from recent sessions (use as inspiration for realistic queries):\\n' +\n          sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join('\\n');\n      }\n    } catch {\n      // No sessions available, that's fine\n    }\n\n    const shouldTriggerCount = Math.ceil(count / 2);\n    const shouldNotTriggerCount = count - shouldTriggerCount;\n\n    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.\n\n## Skill Name\n${target.name}\n\n## Skill Description\n${target.description}\n\n## Skill Content (first 1000 chars)\n${target.content.slice(0, 1000)}\n${sessionSeed}\n\nGenerate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.\n\nRules:\n- Queries must be realistic \u2014 what a real user would actually type\n- Include concrete details (file paths, project names, specific problems)\n- Vary length and formality\n- Should-not-trigger queries should be near-misses (share keywords but need something different)\n- Do NOT make should-not-trigger queries obviously irrelevant\n\nRespond with ONLY a valid JSON array:\n[\n  {\"query\": \"realistic user prompt...\", \"should_trigger\": true},\n  {\"query\": \"another prompt...\", \"should_trigger\": false}\n]`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    try {\n      let jsonStr = result.output.trim();\n      const jsonMatch = jsonStr.match(/```(?:json)?\\s*([\\s\\S]*?)```/);\n      if (jsonMatch) jsonStr = jsonMatch[1].trim();\n      const arrMatch = jsonStr.match(/\\[[\\s\\S]*\\]/);\n      if (arrMatch) jsonStr = arrMatch[0];\n\n      return JSON.parse(jsonStr) as TriggerQuery[];\n    } catch {\n      // Fallback: generate minimal set\n      return [\n        { query: `Help me use ${target.name}`, should_trigger: true },\n        { query: 'Write a hello world program', should_trigger: false },\n      ];\n    }\n  }\n\n  private async testQuerySet(\n    queries: TriggerQuery[],\n    target: EvalTarget,\n    description: string,\n    flags: { model?: string; timeout: number; 'runs-per-query': number },\n  ): Promise<TriggerTestResult[]> {\n    const results: TriggerTestResult[] = [];\n\n    for (const q of queries) {\n      // Run multiple times and take majority vote\n      const runs: TriggerTestResult[] = [];\n      for (let run = 0; run < flags['runs-per-query']; run++) {\n        const result = await testTrigger({\n          query: q.query,\n          shouldTrigger: q.should_trigger,\n          skillName: target.name,\n          skillDescription: description,\n          skillContent: target.content,\n          model: flags.model,\n          timeout: flags.timeout,\n        });\n        runs.push(result);\n      }\n\n      // Majority vote\n      const triggerCount = runs.filter((r) => r.didTrigger).length;\n      const didTrigger = triggerCount > flags['runs-per-query'] / 2;\n\n      results.push({\n        query: q.query,\n        shouldTrigger: q.should_trigger,\n        didTrigger,\n        correct: didTrigger === q.should_trigger,\n        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,\n        evidence: runs[0].evidence,\n      });\n    }\n\n    return results;\n  }\n\n  private printTriggerResults(results: TriggerTestResult[]): void {\n    for (const r of results) {\n      const status = r.correct ? 'OK' : 'FAIL';\n      const expected = r.shouldTrigger ? 'should trigger' : 'should NOT trigger';\n      const actual = r.didTrigger ? 'triggered' : 'did NOT trigger';\n      this.log(`  [${status}] ${expected} / ${actual}: \"${r.query.slice(0, 60)}...\"`);\n    }\n  }\n\n  private async improveDescription(\n    target: EvalTarget,\n    currentDescription: string,\n    failures: TriggerTestResult[],\n    flags: { model?: string; timeout: number },\n  ): Promise<string> {\n    const failureText = failures.map((f) => {\n      const expected = f.shouldTrigger ? 'SHOULD trigger but DID NOT' : 'SHOULD NOT trigger but DID';\n      return `- ${expected}: \"${f.query}\"`;\n    }).join('\\n');\n\n    const prompt = `You are optimizing a skill description for better triggering accuracy.\n\n## Skill Name\n${target.name}\n\n## Current Description\n${currentDescription}\n\n## Skill Content (first 500 chars)\n${target.content.slice(0, 500)}\n\n## Failures\n${failureText}\n\nPropose an improved description that:\n- Fixes the trigger failures above\n- Keeps the description concise but comprehensive\n- Uses specific keywords and contexts for when to trigger\n- Includes \"Use PROACTIVELY\" phrasing for important trigger contexts\n\nRespond with ONLY the new description text (no JSON, no explanation, just the description string).`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    return result.output.trim().replace(/^[\"']|[\"']$/g, '');\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AA6B5B,IAAqB,iCAArB,MAAqB,wCAAuC,YAAY;AAAA,EACtE,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,OAAO,MAAM,OAAO,EAAE,aAAa,4CAA4C,CAAC;AAAA,IAChF,SAAS,MAAM,QAAQ,EAAE,aAAa,yCAAyC,SAAS,GAAG,CAAC;AAAA,IAC5F,kBAAkB,MAAM,QAAQ,EAAE,aAAa,kCAAkC,SAAS,EAAE,CAAC;AAAA,IAC7F,iBAAiB,MAAM,QAAQ,EAAE,aAAa,6BAA6B,SAAS,IAAK,CAAC;AAAA,IAC1F,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,+BAA8B;AACvE,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAE5E,SAAK,IAAI,+BAA+B,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzE,SAAK,IAAI,yBAAyB,OAAO,WAAW,GAAG;AACvD,SAAK,IAAI,EAAE;AAGX,SAAK,IAAI,+BAA+B;AACxC,UAAM,UAAU,MAAM,KAAK,uBAAuB,QAAQ,MAAM,SAAS,KAAK;AAG9E,UAAM,cAAc,MAAM,QAAwB,qBAAqB,KAAK,QAAQ,kBAAkB;AAAA,MACpG;AAAA,IACF,CAAC;AAED,SAAK,IAAI;AAAA,YAAe,QAAQ,MAAM,mBAAmB;AACzD,SAAK,IAAI,6BAA6B;AACtC,SAAK,IAAI,wCAAwC,KAAK,QAAQ,EAAE;AAChE,SAAK,IAAI,yCAAoC;AAC7C,SAAK,IAAI,2BAA2B;AAGpC,QAAI,WAAW;AACf,WAAO,CAAC,UAAU;AAChB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,MAAM,eAAe,CAAC,CAAC;AAC1E,YAAM,OAAO,MAAM;AAAA,QACjB,qBAAqB,KAAK,QAAQ;AAAA,MACpC;AACA,YAAM,UAAU,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE;AACxD,UAAI,SAAS;AACX,mBAAW;AAEX,gBAAQ,SAAS;AACjB,gBAAQ,KAAK,GAAG,QAAQ,OAAO;AAC/B,aAAK,IAAI,2BAA2B;AAAA,MACtC;AAAA,IACF;AAGA,UAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACvE,UAAM,WAAW,KAAK,KAAK,QAAQ,SAAS,GAAG;AAC/C,UAAM,eAAe,QAAQ,MAAM,GAAG,QAAQ;AAC9C,UAAM,cAAc,QAAQ,MAAM,QAAQ;AAG1C,UAAM,OAAO,sBAAsB,YAAY,EAAE,IAAI;AAAA,MACnD;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,eAAe,aAAa,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACvD,UAAM,cAAc,YAAY,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AAErD,SAAK,IAAI;AAAA,SAAY,aAAa,MAAM,WAAW,YAAY,MAAM,OAAO;AAG5E,QAAI,qBAAqB,OAAO;AAChC,QAAI,kBAAkB;AACtB,QAAI,gBAAgB;AAEpB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAC3C,WAAK,IAAI,iBAAiB,kBAAkB,GAAG;AAG/C,WAAK,IAAI,sBAAsB;AAC/B,YAAM,eAAe,MAAM,KAAK,aAAa,cAAc,QAAQ,oBAAoB,KAAK;AAC5F,YAAM,aAAa,oBAAoB,YAAY;AACnD,WAAK,IAAI,oBAAoB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE5D,WAAK,oBAAoB,YAAY;AAGrC,WAAK,IAAI,qBAAqB;AAC9B,YAAM,cAAc,MAAM,KAAK,aAAa,aAAa,QAAQ,oBAAoB,KAAK;AAC1F,YAAM,YAAY,oBAAoB,WAAW;AACjD,WAAK,IAAI,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG1D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa;AAAA,UACb,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,UACxI,aAAa,YAAY,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,QACxI;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,0BAAkB;AAAA,MACpB;AAGA,UAAI,cAAc,GAAK;AACrB,aAAK,IAAI,gCAAgC;AACzC;AAAA,MACF;AAGA,WAAK,IAAI,oCAAoC;AAC7C,YAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AACtD,2BAAqB,MAAM,KAAK,mBAAmB,QAAQ,oBAAoB,UAAU,KAAK;AAAA,IAChG;AAGA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,wBAAwB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,GAAG;AACnE,SAAK,IAAI;AAAA,WAAc,OAAO,WAAW,GAAG;AAC5C,SAAK,IAAI,YAAY,eAAe,GAAG;AAEvC,QAAI,oBAAoB,OAAO,aAAa;AAC1C,YAAM,OAAO,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC7C,aAAa;AAAA,MACf,CAAC;AACD,WAAK,IAAI,wBAAwB;AAAA,IACnC,OAAO;AACL,WAAK,IAAI,gDAAgD;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,QACA,OACA,OACyB;AAEzB,QAAI,cAAc;AAClB,QAAI;AACF,YAAM,WAAW,MAAM,OAA8B,wBAAwB;AAC7E,UAAI,SAAS,SAAS,GAAG;AACvB,sBAAc,6FACZ,SAAS,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,aAAa,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI;AAAA,MAClE;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,qBAAqB,KAAK,KAAK,QAAQ,CAAC;AAC9C,UAAM,wBAAwB,QAAQ;AAEtC,UAAM,SAAS,YAAY,KAAK;AAAA;AAAA;AAAA,EAGlC,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,OAAO,WAAW;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC;AAAA,EAC7B,WAAW;AAAA;AAAA,WAEF,kBAAkB,+BAA+B,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe7E,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI;AACF,UAAI,UAAU,OAAO,OAAO,KAAK;AACjC,YAAM,YAAY,QAAQ,MAAM,8BAA8B;AAC9D,UAAI,UAAW,WAAU,UAAU,CAAC,EAAE,KAAK;AAC3C,YAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAI,SAAU,WAAU,SAAS,CAAC;AAElC,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AAEN,aAAO;AAAA,QACL,EAAE,OAAO,eAAe,OAAO,IAAI,IAAI,gBAAgB,KAAK;AAAA,QAC5D,EAAE,OAAO,+BAA+B,gBAAgB,MAAM;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,aACZ,SACA,QACA,aACA,OAC8B;AAC9B,UAAM,UAA+B,CAAC;AAEtC,eAAW,KAAK,SAAS;AAEvB,YAAM,OAA4B,CAAC;AACnC,eAAS,MAAM,GAAG,MAAM,MAAM,gBAAgB,GAAG,OAAO;AACtD,cAAM,SAAS,MAAM,YAAY;AAAA,UAC/B,OAAO,EAAE;AAAA,UACT,eAAe,EAAE;AAAA,UACjB,WAAW,OAAO;AAAA,UAClB,kBAAkB;AAAA,UAClB,cAAc,OAAO;AAAA,UACrB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,aAAK,KAAK,MAAM;AAAA,MAClB;AAGA,YAAM,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE;AACtD,YAAM,aAAa,eAAe,MAAM,gBAAgB,IAAI;AAE5D,cAAQ,KAAK;AAAA,QACX,OAAO,EAAE;AAAA,QACT,eAAe,EAAE;AAAA,QACjB;AAAA,QACA,SAAS,eAAe,EAAE;AAAA,QAC1B,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,KAAK;AAAA,QAClE,UAAU,KAAK,CAAC,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,SAAoC;AAC9D,eAAW,KAAK,SAAS;AACvB,YAAM,SAAS,EAAE,UAAU,OAAO;AAClC,YAAM,WAAW,EAAE,gBAAgB,mBAAmB;AACtD,YAAM,SAAS,EAAE,aAAa,cAAc;AAC5C,WAAK,IAAI,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAChF;AAAA,EACF;AAAA,EAEA,MAAc,mBACZ,QACA,oBACA,UACA,OACiB;AACjB,UAAM,cAAc,SAAS,IAAI,CAAC,MAAM;AACtC,YAAM,WAAW,EAAE,gBAAgB,+BAA+B;AAClE,aAAO,KAAK,QAAQ,MAAM,EAAE,KAAK;AAAA,IACnC,CAAC,EAAE,KAAK,IAAI;AAEZ,UAAM,SAAS;AAAA;AAAA;AAAA,EAGjB,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,kBAAkB;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA;AAAA,EAG5B,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUT,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO,OAAO,OAAO,KAAK,EAAE,QAAQ,gBAAgB,EAAE;AAAA,EACxD;AACF;",
+  "names": []
+}