@rulemetric/cli 0.7.3 → 0.7.5

package/dist/commands/evals/optimize-description.js

@@ -1,169 +1,28 @@
-import {
-  computeTriggerScore,
-  testTrigger
-} from "../../chunk-4D537GXE.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost,
-  apiPut
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/optimize-description.ts
-import { Args, Flags } from "@oclif/core";
-var EvalOptimizeDescriptionCommand = class _EvalOptimizeDescriptionCommand extends BaseCommand {
-  static description = "Optimize eval target description for better triggering accuracy";
-  static examples = [
-    "<%= config.bin %> evals optimize-description <target-id>",
-    "<%= config.bin %> evals optimize-description <target-id> --queries 10"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    "max-iterations": Flags.integer({ description: "Maximum optimization iterations", default: 5 }),
-    model: Flags.string({ description: "Model for trigger testing and improvement" }),
-    queries: Flags.integer({ description: "Number of trigger queries to generate", default: 20 }),
-    "runs-per-query": Flags.integer({ description: "Runs per query for reliability", default: 3 }),
-    "poll-interval": Flags.integer({ description: "Ms between approval polls", default: 5e3 }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalOptimizeDescriptionCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    this.log(`Optimizing description for "${target.name}" v${target.version}`);
-    this.log(`Current description: "${target.description}"`);
-    this.log("");
-    this.log("Generating trigger queries...");
-    const queries = await this.generateTriggerQueries(target, flags.queries, flags);
-    const triggerEval = await apiPost(`/api/eval-targets/${args.targetId}/trigger-evals`, {
-      queries
-    });
-    this.log(`
-Generated ${queries.length} trigger queries.`);
-    this.log(`Review and approve them at:`);
-    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);
-    this.log(`  (Triggers tab \u2192 click to review)`);
-    this.log("\nWaiting for approval...");
-    let approved = false;
-    while (!approved) {
-      await new Promise((resolve) => setTimeout(resolve, flags["poll-interval"]));
-      const sets = await apiGet(
-        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`
-      );
-      const current = sets.find((s) => s.id === triggerEval.id);
-      if (current) {
-        approved = true;
-        queries.length = 0;
-        queries.push(...current.queries);
-        this.log("Trigger queries approved!");
-      }
-    }
-    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);
-    const splitIdx = Math.ceil(indices.length * 0.6);
-    const trainIndices = indices.slice(0, splitIdx);
-    const testIndices = indices.slice(splitIdx);
-    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {
-      trainIndices,
-      testIndices
-    });
-    const trainQueries = trainIndices.map((i) => queries[i]);
-    const testQueries = testIndices.map((i) => queries[i]);
-    this.log(`
-Split: ${trainQueries.length} train, ${testQueries.length} test`);
-    let currentDescription = target.description;
-    let bestDescription = currentDescription;
-    let bestTestScore = 0;
-    for (let iteration = 1; iteration <= flags["max-iterations"]; iteration++) {
-      this.log(`
-=== Iteration ${iteration} ===`);
-      this.log(`Description: "${currentDescription}"`);
-      this.log("Testing train set...");
-      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);
-      const trainScore = computeTriggerScore(trainResults);
-      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);
-      this.printTriggerResults(trainResults);
-      this.log("Testing test set...");
-      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);
-      const testScore = computeTriggerScore(testResults);
-      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);
-      await apiPost("/api/optimization-runs", {
-        evalTargetId: target.id,
-        type: "description",
-        iteration,
-        inputVersion: target.version,
-        trainScore,
-        testScore,
-        analysis: {
-          description: currentDescription,
-          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
-          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct }))
-        },
-        status: "completed"
-      });
-      if (testScore > bestTestScore) {
-        bestTestScore = testScore;
-        bestDescription = currentDescription;
-      }
-      if (trainScore >= 1) {
-        this.log("Perfect train score. Stopping.");
-        break;
-      }
-      this.log("Generating improved description...");
-      const failures = trainResults.filter((r) => !r.correct);
-      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);
-    }
-    this.log(`
-=== Optimization Complete ===`);
-    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);
-    this.log(`
-Before: "${target.description}"`);
-    this.log(`After:  "${bestDescription}"`);
-    if (bestDescription !== target.description) {
-      await apiPut(`/api/eval-targets/${target.id}`, {
-        description: bestDescription
-      });
-      this.log("\nDescription updated.");
-    } else {
-      this.log("\nNo improvement found. Description unchanged.");
-    }
-  }
-  async generateTriggerQueries(target, count, flags) {
-    let sessionSeed = "";
-    try {
-      const sessions = await apiGet("/api/sessions?limit=10");
-      if (sessions.length > 0) {
-        sessionSeed = "\n\nReal user prompts from recent sessions (use as inspiration for realistic queries):\n" + sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join("\n");
-      }
-    } catch {
-    }
-    const shouldTriggerCount = Math.ceil(count / 2);
-    const shouldNotTriggerCount = count - shouldTriggerCount;
-    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.
+import{a as S,b as P,c as R}from"../../chunk-KV325CXF.js";import{a as b}from"../../chunk-MGHFDHHM.js";import{a as O}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as q,g as w}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as N}from"../../chunk-ZLLER3HP.js";import{Args as C,Flags as T}from"@oclif/core";var A=process.env.RULEMETRIC_WEB_URL??"https://rulemetric.com",k=30*6e4,I=class M extends O{static{N(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:C.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":T.integer({description:"Maximum optimization iterations",default:5}),model:T.string({description:"Model for trigger testing and improvement"}),queries:T.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":T.integer({description:"Runs per query for reliability",default:3}),"poll-interval":T.integer({description:"Ms between approval polls",default:5e3}),timeout:T.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(M);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log(S),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),a=await q(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r}),g=`${A}/eval-targets/${i.targetId}`;this.log(`
+Generated ${r.length} trigger queries.`),this.log("Review and approve them at:"),this.log(`  ${g}`),this.log("  (Triggers tab \u2192 click to review)"),this.log(`
+Waiting for approval...`);let u=Date.now(),d=!1;for(;!d;){Date.now()-u>k&&this.error(`Timed out after ${k/6e4} minutes waiting for approval. Approve the trigger set at ${g} (Triggers tab) and re-run this command.`),await new Promise(h=>setTimeout(h,t["poll-interval"]));let p=(await y(`/api/eval-targets/${i.targetId}/trigger-evals?status=approved`)).find(h=>h.id===a.id);p&&(d=!0,r.length=0,r.push(...p.queries),this.log("Trigger queries approved!"))}let n=r.map((c,p)=>p).sort(()=>Math.random()-.5),o=Math.ceil(n.length*.6),s=n.slice(0,o),x=n.slice(o);await w(`/api/trigger-evals/${a.id}`,{trainIndices:s,testIndices:x});let D=s.map(c=>r[c]),_=x.map(c=>r[c]);this.log(`
+Split: ${D.length} train, ${_.length} test`);let m=e.description,v=m,$=0;for(let c=1;c<=t["max-iterations"];c++){this.log(`
+=== Iteration ${c} ===`),this.log(`Description: "${m}"`),this.log("Testing train set...");let p=await this.testQuerySet(D,e,m,t),h=R(p);this.log(`Train accuracy: ${(h*100).toFixed(1)}%`),this.printTriggerResults(p),this.log("Testing test set...");let E=await this.testQuerySet(_,e,m,t),f=R(E);if(this.log(`Test accuracy: ${(f*100).toFixed(1)}%`),await q("/api/optimization-runs",{evalTargetId:e.id,type:"description",iteration:c,inputVersion:e.version,trainScore:h,testScore:f,analysis:{description:m,trainResults:p.map(l=>({query:l.query,shouldTrigger:l.shouldTrigger,didTrigger:l.didTrigger,correct:l.correct})),testResults:E.map(l=>({query:l.query,shouldTrigger:l.shouldTrigger,didTrigger:l.didTrigger,correct:l.correct}))},status:"completed"}),f>$&&($=f,v=m),h>=1){this.log("Perfect train score. Stopping.");break}this.log("Generating improved description...");let Q=p.filter(l=>!l.correct);m=await this.improveDescription(e,m,Q,t)}this.log(`
+=== Optimization Complete ===`),this.log(`Best test accuracy: ${($*100).toFixed(1)}%`),this.log(S),this.log(`
+Before: "${e.description}"`),this.log(`After:  "${v}"`),v!==e.description?(await w(`/api/eval-targets/${e.id}`,{description:v}),this.log(`
+Description updated.`)):this.log(`
+No improvement found. Description unchanged.`)}async generateTriggerQueries(i,t,e){let r="";try{let o=(await y("/api/sessions?limit=10")).map(s=>s.firstUserPrompt??s.lastMessagePreview??"").filter(s=>s.trim().length>0).slice(0,5);o.length>0&&(r=`
+
+Real user prompts from recent sessions (use as inspiration for realistic queries):
+`+o.map(s=>`- ${s}`).join(`
+`))}catch{}let a=Math.ceil(t/2),g=t-a,u=`Generate ${t} trigger eval queries for a skill/instruction.
 
 ## Skill Name
-${target.name}
+${i.name}
 
 ## Skill Description
-${target.description}
+${i.description}
 
 ## Skill Content (first 1000 chars)
-${target.content.slice(0, 1e3)}
-${sessionSeed}
+${i.content.slice(0,1e3)}
+${r}
 
-Generate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.
+Generate ${a} should-trigger queries and ${g} should-not-trigger queries.
 
 Rules:
 - Queries must be realistic \u2014 what a real user would actually type
@@ -176,81 +35,20 @@ Respond with ONLY a valid JSON array:
 [
   {"query": "realistic user prompt...", "should_trigger": true},
   {"query": "another prompt...", "should_trigger": false}
-]`;
-    const result = await executeEval({
-      prompt,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    try {
-      let jsonStr = result.output.trim();
-      const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-      if (jsonMatch) jsonStr = jsonMatch[1].trim();
-      const arrMatch = jsonStr.match(/\[[\s\S]*\]/);
-      if (arrMatch) jsonStr = arrMatch[0];
-      return JSON.parse(jsonStr);
-    } catch {
-      return [
-        { query: `Help me use ${target.name}`, should_trigger: true },
-        { query: "Write a hello world program", should_trigger: false }
-      ];
-    }
-  }
-  async testQuerySet(queries, target, description, flags) {
-    const results = [];
-    for (const q of queries) {
-      const runs = [];
-      for (let run = 0; run < flags["runs-per-query"]; run++) {
-        const result = await testTrigger({
-          query: q.query,
-          shouldTrigger: q.should_trigger,
-          skillName: target.name,
-          skillDescription: description,
-          skillContent: target.content,
-          model: flags.model,
-          timeout: flags.timeout
-        });
-        runs.push(result);
-      }
-      const triggerCount = runs.filter((r) => r.didTrigger).length;
-      const didTrigger = triggerCount > flags["runs-per-query"] / 2;
-      results.push({
-        query: q.query,
-        shouldTrigger: q.should_trigger,
-        didTrigger,
-        correct: didTrigger === q.should_trigger,
-        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,
-        evidence: runs[0].evidence
-      });
-    }
-    return results;
-  }
-  printTriggerResults(results) {
-    for (const r of results) {
-      const status = r.correct ? "OK" : "FAIL";
-      const expected = r.shouldTrigger ? "should trigger" : "should NOT trigger";
-      const actual = r.didTrigger ? "triggered" : "did NOT trigger";
-      this.log(`  [${status}] ${expected} / ${actual}: "${r.query.slice(0, 60)}..."`);
-    }
-  }
-  async improveDescription(target, currentDescription, failures, flags) {
-    const failureText = failures.map((f) => {
-      const expected = f.shouldTrigger ? "SHOULD trigger but DID NOT" : "SHOULD NOT trigger but DID";
-      return `- ${expected}: "${f.query}"`;
-    }).join("\n");
-    const prompt = `You are optimizing a skill description for better triggering accuracy.
+]`,d=await b({prompt:u,timeout:e.timeout,model:e.model});try{let n=d.output.trim(),o=n.match(/```(?:json)?\s*([\s\S]*?)```/);o&&(n=o[1].trim());let s=n.match(/\[[\s\S]*\]/);return s&&(n=s[0]),JSON.parse(n)}catch{return[{query:`Help me use ${i.name}`,should_trigger:!0},{query:"Write a hello world program",should_trigger:!1}]}}async testQuerySet(i,t,e,r){let a=[];for(let g of i){let u=[];for(let o=0;o<r["runs-per-query"];o++){let s=await P({query:g.query,shouldTrigger:g.should_trigger,skillName:t.name,skillDescription:e,skillContent:t.content,model:r.model,timeout:r.timeout});u.push(s)}let n=u.filter(o=>o.didTrigger).length>r["runs-per-query"]/2;a.push({query:g.query,shouldTrigger:g.should_trigger,didTrigger:n,correct:n===g.should_trigger,confidence:u.reduce((o,s)=>o+s.confidence,0)/u.length,evidence:u[0].evidence})}return a}printTriggerResults(i){for(let t of i){let e=t.correct?"OK":"FAIL",r=t.shouldTrigger?"should trigger":"should NOT trigger",a=t.didTrigger?"triggered":"did NOT trigger";this.log(`  [${e}] ${r} / ${a}: "${t.query.slice(0,60)}..."`)}}async improveDescription(i,t,e,r){let a=e.map(d=>`- ${d.shouldTrigger?"SHOULD trigger but DID NOT":"SHOULD NOT trigger but DID"}: "${d.query}"`).join(`
+`),g=`You are optimizing a skill description for better triggering accuracy.
 
 ## Skill Name
-${target.name}
+${i.name}
 
 ## Current Description
-${currentDescription}
+${t}
 
 ## Skill Content (first 500 chars)
-${target.content.slice(0, 500)}
+${i.content.slice(0,500)}
 
 ## Failures
-${failureText}
+${a}
 
 Propose an improved description that:
 - Fixes the trigger failures above
@@ -258,16 +56,4 @@ Propose an improved description that:
 - Uses specific keywords and contexts for when to trigger
 - Includes "Use PROACTIVELY" phrasing for important trigger contexts
 
-Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;
-    const result = await executeEval({
-      prompt,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    return result.output.trim().replace(/^["']|["']$/g, "");
-  }
-};
-export {
-  EvalOptimizeDescriptionCommand as default
-};
-//# sourceMappingURL=optimize-description.js.map
+Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;return(await b({prompt:g,timeout:r.timeout,model:r.model})).output.trim().replace(/^["']|["']$/g,"")}};export{I as default};

package/dist/commands/evals/optimize.js

@@ -1,211 +1,5 @@
-import {
-  analyzeEvalResults
-} from "../../chunk-3TIMQ3O6.js";
-import {
-  gradeEvalOutput
-} from "../../chunk-YNIH3KMU.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost,
-  apiPut
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/optimize.ts
-import { Args, Flags } from "@oclif/core";
-var EvalOptimizeCommand = class _EvalOptimizeCommand extends BaseCommand {
-  static description = "Iteratively optimize eval target content using eval results as feedback";
-  static examples = [
-    "<%= config.bin %> evals optimize <target-id>",
-    "<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    auto: Flags.boolean({ description: "Skip interactive review, apply suggestions automatically", default: false }),
-    "max-iterations": Flags.integer({ description: "Maximum optimization iterations", default: 5 }),
-    threshold: Flags.string({ description: "Stop if pass rate reaches this threshold (0-1)", default: "0.95" }),
-    model: Flags.string({ description: "Model for eval execution" }),
-    "analyzer-model": Flags.string({ description: "Model for analysis/suggestions" }),
-    "runs-per-eval": Flags.integer({ description: "Runs per eval per iteration", default: 1 }),
-    parallel: Flags.integer({ description: "Number of parallel eval executions", default: 1 }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalOptimizeCommand);
-    this.requireAuth();
-    let target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    const allEvals = await apiGet(`/api/eval-targets/${args.targetId}/evals`);
-    if (allEvals.length === 0) {
-      this.error("No evals found. Create some first with `rulemetric evals create`.");
-    }
-    const shuffled = [...allEvals].sort(() => Math.random() - 0.5);
-    const splitIdx = Math.ceil(shuffled.length * 0.6);
-    const trainEvals = shuffled.slice(0, splitIdx);
-    const testEvals = shuffled.slice(splitIdx);
-    this.log(`Optimizing "${target.name}" v${target.version}`);
-    this.log(`Evals: ${trainEvals.length} train, ${testEvals.length} test`);
-    const threshold = parseFloat(flags.threshold);
-    this.log(`Config: max ${flags["max-iterations"]} iterations, threshold ${(threshold * 100).toFixed(0)}%`);
-    if (flags.auto) this.log("Mode: auto (no interactive review)");
-    this.log("");
-    let bestVersion = target.version;
-    let bestTestScore = 0;
-    let noImprovementCount = 0;
-    for (let iteration = 1; iteration <= flags["max-iterations"]; iteration++) {
-      this.log(`
-=== Iteration ${iteration} ===`);
-      const optRun = await apiPost("/api/optimization-runs", {
-        evalTargetId: target.id,
-        type: "content",
-        iteration,
-        inputVersion: target.version,
-        status: "running"
-      });
-      this.log("Running train evals...");
-      const trainResults = await this.runEvalsOnTarget(target, trainEvals, flags);
-      const trainScore = this.computePassRate(trainResults);
-      this.log(`Train pass rate: ${(trainScore * 100).toFixed(1)}%`);
-      if (trainScore >= threshold) {
-        this.log(`Train score ${(trainScore * 100).toFixed(1)}% >= threshold ${(threshold * 100).toFixed(0)}%. Stopping.`);
-        const testResults2 = await this.runEvalsOnTarget(target, testEvals, flags);
-        const testScore2 = this.computePassRate(testResults2);
-        this.log(`Final test score: ${(testScore2 * 100).toFixed(1)}%`);
-        await apiPost("/api/optimization-runs", {
-          evalTargetId: target.id,
-          type: "content",
-          iteration,
-          inputVersion: target.version,
-          trainScore,
-          testScore: testScore2,
-          analysis: { reason: "threshold_reached" },
-          status: "completed"
-        });
-        break;
-      }
-      this.log("Analyzing results...");
-      const evalResults = trainResults.map((r) => ({
-        evalName: r.evalName,
-        prompt: r.prompt,
-        expectations: r.expectations,
-        grading: r.grading
-      }));
-      const analysis = await analyzeEvalResults({
-        content: target.content,
-        evalResults,
-        model: flags["analyzer-model"] ?? flags.model,
-        timeout: flags.timeout
-      });
-      this.log(`
-Assessment: ${analysis.overallAssessment}`);
-      this.log(`Suggestions: ${analysis.suggestions.length}`);
-      for (const s of analysis.suggestions) {
-        this.log(`  - [${s.section}] ${s.reasoning}`);
-      }
-      if (!flags.auto) {
-        this.log("\nProposed changes above. Applying in auto mode would proceed automatically.");
-        this.log("(Interactive approval not yet implemented \u2014 use --auto for now)");
-      }
-      if (analysis.proposedContent === target.content) {
-        this.log("No changes proposed. Stopping.");
-        await apiPost("/api/optimization-runs", {
-          evalTargetId: target.id,
-          type: "content",
-          iteration,
-          inputVersion: target.version,
-          trainScore,
-          analysis: { reason: "no_changes" },
-          status: "completed"
-        });
-        break;
-      }
-      const newTarget = await apiPut(`/api/eval-targets/${target.id}`, {
-        content: analysis.proposedContent
-      });
-      this.log(`Created v${newTarget.version}`);
-      this.log("Running test evals...");
-      const testResults = await this.runEvalsOnTarget(newTarget, testEvals, flags);
-      const testScore = this.computePassRate(testResults);
-      this.log(`Test pass rate: ${(testScore * 100).toFixed(1)}%`);
-      await apiPost("/api/optimization-runs", {
-        evalTargetId: newTarget.id,
-        type: "content",
-        iteration,
-        inputVersion: target.version,
-        outputVersion: newTarget.version,
-        trainScore,
-        testScore,
-        analysis: {
-          suggestions: analysis.suggestions,
-          overallAssessment: analysis.overallAssessment
-        },
-        status: "completed"
-      });
-      if (testScore > bestTestScore) {
-        bestTestScore = testScore;
-        bestVersion = newTarget.version;
-        noImprovementCount = 0;
-      } else {
-        noImprovementCount++;
-        if (noImprovementCount >= 2) {
-          this.log("No improvement for 2 consecutive iterations. Stopping.");
-          break;
-        }
-      }
-      target = newTarget;
-    }
-    this.log(`
-=== Optimization Complete ===`);
-    this.log(`Best version: v${bestVersion} (test score: ${(bestTestScore * 100).toFixed(1)}%)`);
-  }
-  async runEvalsOnTarget(target, evalSet, flags) {
-    const results = [];
-    for (const eval_ of evalSet) {
-      const execResult = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: target.content,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      if (execResult.error) {
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
-        continue;
-      }
-      if (eval_.expectations.length > 0) {
-        const grading = await gradeEvalOutput({
-          prompt: eval_.prompt,
-          output: execResult.output,
-          expectations: eval_.expectations,
-          model: flags.model,
-          timeout: flags.timeout
-        });
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading });
-      } else {
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
-      }
-    }
-    return results;
-  }
-  computePassRate(results) {
-    const graded = results.filter((r) => Object.keys(r.grading).length > 0);
-    if (graded.length === 0) return 0;
-    const rates = graded.map((r) => r.grading.summary?.pass_rate ?? 0);
-    return rates.reduce((a, b) => a + b, 0) / rates.length;
-  }
-};
-export {
-  EvalOptimizeCommand as default
-};
-//# sourceMappingURL=optimize.js.map
+import{a as A}from"../../chunk-DSO2U4Y6.js";import{a as k}from"../../chunk-CADKBP3D.js";import{a as z}from"../../chunk-MGHFDHHM.js";import{a as I}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as y,f as m,g as R}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as b}from"../../chunk-ZLLER3HP.js";import{createInterface as C}from"node:readline/promises";import{Args as P,Flags as p}from"@oclif/core";var w=class N extends I{static{b(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:P.string({description:"Eval target ID",required:!0})};static flags={auto:p.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":p.integer({description:"Maximum optimization iterations",default:5}),threshold:p.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:p.string({description:"Model for eval execution"}),"analyzer-model":p.string({description:"Model for analysis/suggestions"}),timeout:p.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:a,flags:e}=await this.parse(N);this.requireAuth();let t=await y(`/api/eval-targets/${a.targetId}`),i=await y(`/api/eval-targets/${a.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let s=[...i].sort(()=>Math.random()-.5),g=Math.ceil(s.length*.6),u=s.slice(0,g),v=s.slice(g);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${u.length} train, ${v.length} test`);let h=parseFloat(e.threshold);this.log(`Config: max ${e["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),e.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let $=t.version,f=0,x=0;for(let r=1;r<=e["max-iterations"];r++){this.log(`
+=== Iteration ${r} ===`),this.log("Running train evals...");let T=await this.runEvalsOnTarget(t,u,e),l=this.computePassRate(T);if(this.log(`Train pass rate: ${(l*100).toFixed(1)}%`),l>=h){this.log(`Train score ${(l*100).toFixed(1)}% >= threshold ${(h*100).toFixed(0)}%. Stopping.`);let o=await this.runEvalsOnTarget(t,v,e),E=this.computePassRate(o);this.log(`Final test score: ${(E*100).toFixed(1)}%`),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:r,inputVersion:t.version,trainScore:l,testScore:E,analysis:{reason:"threshold_reached"},status:"completed"});break}this.log("Analyzing results...");let S=T.map(o=>({evalName:o.evalName,prompt:o.prompt,expectations:o.expectations,grading:o.grading})),n=await A({content:t.content,evalResults:S,model:e["analyzer-model"]??e.model,timeout:e.timeout});this.log(`
+Assessment: ${n.overallAssessment}`),this.log(`Suggestions: ${n.suggestions.length}`);for(let o of n.suggestions)this.log(`  - [${o.section}] ${o.reasoning}`);if(n.proposedContent===t.content){this.log("No changes proposed. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:r,inputVersion:t.version,trainScore:l,analysis:{reason:"no_changes"},status:"completed"});break}if(!e.auto&&!await this.confirm(`
+Apply the proposed changes as a new version? [y/N] `)){this.log("Changes not applied. Stopping."),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:r,inputVersion:t.version,trainScore:l,analysis:{reason:"user_declined",suggestions:n.suggestions},status:"completed"});break}let c=await R(`/api/eval-targets/${t.id}`,{content:n.proposedContent});this.log(`Created v${c.version}`),this.log("Running test evals...");let F=await this.runEvalsOnTarget(c,v,e),d=this.computePassRate(F);if(this.log(`Test pass rate: ${(d*100).toFixed(1)}%`),await m("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:r,inputVersion:t.version,outputVersion:c.version,trainScore:l,testScore:d,analysis:{suggestions:n.suggestions,overallAssessment:n.overallAssessment},status:"completed"}),d>f)f=d,$=c.version,x=0;else if(x++,x>=2){this.log("No improvement for 2 consecutive iterations. Stopping.");break}t=c}this.log(`
+=== Optimization Complete ===`),this.log(`Best version: v${$} (test score: ${(f*100).toFixed(1)}%)`)}async confirm(a){process.stdin.isTTY||this.error("Interactive approval requires a TTY. Re-run with --auto to apply changes without review.");let e=C({input:process.stdin,output:process.stdout});try{let t=await e.question(a);return/^y(es)?$/i.test(t.trim())}finally{e.close()}}async runEvalsOnTarget(a,e,t){let i=[];for(let s of e){let g=await z({prompt:s.prompt,targetContent:a.content,timeout:t.timeout,model:t.model});if(g.error){i.push({evalName:s.name,prompt:s.prompt,expectations:s.expectations,grading:{}});continue}if(s.expectations.length>0){let u=await k({prompt:s.prompt,output:g.output,expectations:s.expectations,model:t.model,timeout:t.timeout});i.push({evalName:s.name,prompt:s.prompt,expectations:s.expectations,grading:u})}else i.push({evalName:s.name,prompt:s.prompt,expectations:s.expectations,grading:{}})}return i}computePassRate(a){let e=a.filter(i=>Object.keys(i.grading).length>0);if(e.length===0)return 0;let t=e.map(i=>i.grading.summary?.pass_rate??0);return t.reduce((i,s)=>i+s,0)/t.length}};export{w as default};