@rulemetric/cli 0.7.3 → 0.7.4

package/dist/commands/evals/benchmark.js

@@ -1,73 +1,6 @@
-import {
-  aggregateBenchmark,
-  postBenchmark
-} from "../../chunk-PED7L5GX.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/benchmark.ts
-import { Args, Flags } from "@oclif/core";
-var EvalBenchmarkCommand = class _EvalBenchmarkCommand extends BaseCommand {
-  static description = "Aggregate eval runs into a benchmark for a target";
-  static examples = [
-    "<%= config.bin %> evals benchmark <target-id>",
-    "<%= config.bin %> evals benchmark <target-id> --iteration 2"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    iteration: Flags.integer({ description: "Filter runs by iteration", default: 1 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalBenchmarkCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    this.log(`Aggregating benchmark for "${target.name}" v${target.version}, iteration ${flags.iteration}`);
-    const result = await aggregateBenchmark(args.targetId, flags.iteration);
-    if (result.totalRuns === 0) {
-      this.error("No completed runs found. Run evals first with `rulemetric evals run`.");
-    }
-    this.log(`
-Aggregated ${result.totalRuns} runs
-`);
-    for (const [config, stats] of Object.entries(result.runSummary)) {
-      if (config === "delta") {
-        this.log(`
-Delta (with_target - without_target):`);
-        for (const [metric, value] of Object.entries(stats)) {
-          this.log(`  ${metric}: ${value}`);
-        }
-        continue;
-      }
-      this.log(`${config}:`);
-      for (const [metric, stat] of Object.entries(stats)) {
-        const s = stat;
-        this.log(`  ${metric}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`);
-      }
-    }
-    if (result.notes.length > 0) {
-      this.log("\nNotes:");
-      for (const note of result.notes) {
-        this.log(`  - ${note}`);
-      }
-    }
-    const benchmark = await postBenchmark(args.targetId, flags.iteration, result);
-    this.log(`
-Benchmark saved: ${benchmark.id}`);
-  }
-};
-export {
-  EvalBenchmarkCommand as default
-};
-//# sourceMappingURL=benchmark.js.map
+import{a as u,b as f}from"../../chunk-RVO4HV3M.js";import{a as d}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{d as l}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as m}from"../../chunk-ZLLER3HP.js";import{Args as b,Flags as $}from"@oclif/core";var o=class h extends d{static{m(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:b.string({description:"Eval target ID",required:!0})};static flags={iteration:$.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:i,flags:a}=await this.parse(h);this.requireAuth();let g=await l(`/api/eval-targets/${i.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${a.iteration}`);let t=await u(i.targetId,a.iteration);t.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
+Aggregated ${t.totalRuns} runs
+`);for(let[e,c]of Object.entries(t.runSummary)){if(e==="delta"){this.log(`
+Delta (with_target - without_target):`);for(let[n,s]of Object.entries(c))this.log(`  ${n}: ${s}`);continue}this.log(`${e}:`);for(let[n,s]of Object.entries(c)){let r=s;this.log(`  ${n}: mean=${r.mean} stddev=${r.stddev} [${r.min}, ${r.max}]`)}}if(t.notes.length>0){this.log(`
+Notes:`);for(let e of t.notes)this.log(`  - ${e}`)}let v=await f(i.targetId,a.iteration,t);this.log(`
+Benchmark saved: ${v.id}`)}};export{o as default};

package/dist/commands/evals/compare.js

@@ -1,90 +1,19 @@
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/compare.ts
-import { Args, Flags } from "@oclif/core";
-var EvalCompareCommand = class _EvalCompareCommand extends BaseCommand {
-  static description = "Blind A/B comparison between two eval target versions";
-  static examples = [
-    "<%= config.bin %> evals compare <target-a> <target-b>"
-  ];
-  static args = {
-    targetA: Args.string({ description: "First eval target ID", required: true }),
-    targetB: Args.string({ description: "Second eval target ID", required: true })
-  };
-  static flags = {
-    model: Flags.string({ description: "Model for execution and comparison" }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalCompareCommand);
-    this.requireAuth();
-    const [targetA, targetB] = await Promise.all([
-      apiGet(`/api/eval-targets/${args.targetA}`),
-      apiGet(`/api/eval-targets/${args.targetB}`)
-    ]);
-    const evals = await apiGet(`/api/eval-targets/${args.targetA}/evals`);
-    if (evals.length === 0) {
-      this.error("No evals found for target A. Create some first.");
-    }
-    this.log(`Comparing "${targetA.name}" v${targetA.version} vs "${targetB.name}" v${targetB.version}`);
-    this.log(`Running ${evals.length} eval(s)...
-`);
-    const isSwapped = Math.random() > 0.5;
-    const contentA = isSwapped ? targetB.content : targetA.content;
-    const contentB = isSwapped ? targetA.content : targetB.content;
-    const labelA = isSwapped ? targetB.name : targetA.name;
-    const labelB = isSwapped ? targetA.name : targetB.name;
-    const comparisons = [];
-    for (const eval_ of evals) {
-      this.log(`[${eval_.name}] Running A...`);
-      const resultA = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: contentA,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      this.log(`[${eval_.name}] Running B...`);
-      const resultB = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: contentB,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      comparisons.push({
-        evalName: eval_.name,
-        outputA: resultA.output,
-        outputB: resultB.output
-      });
-    }
-    this.log("\nRunning blind comparison...");
-    const comparisonPrompt = comparisons.map((c) => `
-## Eval: ${c.evalName}
+import{a as i}from"../../chunk-MGHFDHHM.js";import{a as u}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{d as a}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as p}from"../../chunk-ZLLER3HP.js";import{Args as c,Flags as v}from"@oclif/core";var m=class d extends u{static{p(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:c.string({description:"First eval target ID",required:!0}),targetB:c.string({description:"Second eval target ID",required:!0})};static flags={model:v.string({description:"Model for execution and comparison"}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:s,flags:e}=await this.parse(d);this.requireAuth();let[o,n]=await Promise.all([a(`/api/eval-targets/${s.targetA}`),a(`/api/eval-targets/${s.targetB}`)]),l=await a(`/api/eval-targets/${s.targetA}/evals`);l.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${o.name}" v${o.version} vs "${n.name}" v${n.version}`),this.log(`Running ${l.length} eval(s)...
+`);let r=Math.random()>.5,h=r?n.content:o.content,f=r?o.content:n.content,A=r?n.name:o.name,w=r?o.name:n.name,g=[];for(let t of l){this.log(`[${t.name}] Running A...`);let b=await i({prompt:t.prompt,targetContent:h,timeout:e.timeout,model:e.model});this.log(`[${t.name}] Running B...`);let E=await i({prompt:t.prompt,targetContent:f,timeout:e.timeout,model:e.model});g.push({evalName:t.name,outputA:b.output,outputB:E.output})}this.log(`
+Running blind comparison...`);let B=g.map(t=>`
+## Eval: ${t.evalName}
 
 ### Output A
-${c.outputA}
+${t.outputA}
 
 ### Output B
-${c.outputB}
-`).join("\n---\n");
-    const comparatorResult = await executeEval({
-      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
+${t.outputB}
+`).join(`
+---
+`),$=await i({prompt:`Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
 Then provide an overall winner with reasoning.
 
-${comparisonPrompt}
+${B}
 
 Respond with JSON:
 {
@@ -97,18 +26,6 @@ Respond with JSON:
     "strengths_winner": ["..."],
     "weaknesses_loser": ["..."]
   }
-}`,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    this.log("\n=== Comparison Results ===\n");
-    this.log(`A = "${labelA}" | B = "${labelB}"`);
-    if (isSwapped) this.log("(Labels were swapped for blind comparison)");
-    this.log("");
-    this.log(comparatorResult.output);
-  }
-};
-export {
-  EvalCompareCommand as default
-};
-//# sourceMappingURL=compare.js.map
+}`,timeout:e.timeout,model:e.model});this.log(`
+=== Comparison Results ===
+`),this.log(`A = "${A}" | B = "${w}"`),r&&this.log("(Labels were swapped for blind comparison)"),this.log(""),this.log($.output)}};export{m as default};

package/dist/commands/evals/create.js

@@ -1,67 +1 @@
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiPost
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/create.ts
-import { Args, Flags } from "@oclif/core";
-import { readFileSync } from "node:fs";
-var EvalCreate = class _EvalCreate extends BaseCommand {
-  static description = "Create test cases (evals) for an eval target";
-  static examples = [
-    '<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',
-    "<%= config.bin %> evals create <target-id> --from-file evals.json"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    name: Flags.string({ char: "n", description: "Eval name" }),
-    prompt: Flags.string({ char: "p", description: "Test prompt" }),
-    "expected-output": Flags.string({ description: "Expected output description" }),
-    expectations: Flags.string({ char: "e", description: "Assertion (repeatable)", multiple: true }),
-    "from-file": Flags.string({ description: "Bulk create from JSON file" })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalCreate);
-    this.requireAuth();
-    if (flags["from-file"]) {
-      const data = JSON.parse(readFileSync(flags["from-file"], "utf-8"));
-      const evalsArray = Array.isArray(data) ? data : data.evals;
-      for (const evalItem of evalsArray) {
-        const result2 = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-          name: evalItem.name,
-          prompt: evalItem.prompt,
-          expectedOutput: evalItem.expectedOutput ?? "",
-          expectations: evalItem.expectations ?? [],
-          files: evalItem.files ?? []
-        });
-        this.log(`Created eval: ${result2.name} (${result2.id})`);
-      }
-      this.log(`Created ${evalsArray.length} evals.`);
-      return;
-    }
-    if (!flags.name || !flags.prompt) {
-      this.error("Provide --name and --prompt, or use --from-file for bulk creation");
-    }
-    const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-      name: flags.name,
-      prompt: flags.prompt,
-      expectedOutput: flags["expected-output"] ?? "",
-      expectations: flags.expectations ?? []
-    });
-    this.log(`Created eval: ${result.name} (${result.id})`);
-  }
-};
-export {
-  EvalCreate as default
-};
-//# sourceMappingURL=create.js.map
+import{a as m}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{e as i}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as l}from"../../chunk-ZLLER3HP.js";import{Args as g,Flags as r}from"@oclif/core";import{readFileSync as u}from"node:fs";var s=class d extends m{static{l(this,"EvalCreate")}static description="Create test cases (evals) for an eval target";static examples=['<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',"<%= config.bin %> evals create <target-id> --from-file evals.json"];static args={targetId:g.string({description:"Eval target ID",required:!0})};static flags={name:r.string({char:"n",description:"Eval name"}),prompt:r.string({char:"p",description:"Test prompt"}),"expected-output":r.string({description:"Expected output description"}),expectations:r.string({char:"e",description:"Assertion (repeatable)",multiple:!0}),"from-file":r.string({description:"Bulk create from JSON file"})};async run(){let{args:o,flags:e}=await this.parse(d);if(this.requireAuth(),e["from-file"]){let a=JSON.parse(u(e["from-file"],"utf-8")),p=Array.isArray(a)?a:a.evals;for(let t of p){let c=await i(`/api/eval-targets/${o.targetId}/evals`,{name:t.name,prompt:t.prompt,expectedOutput:t.expectedOutput??"",expectations:t.expectations??[],files:t.files??[]});this.log(`Created eval: ${c.name} (${c.id})`)}this.log(`Created ${p.length} evals.`);return}(!e.name||!e.prompt)&&this.error("Provide --name and --prompt, or use --from-file for bulk creation");let n=await i(`/api/eval-targets/${o.targetId}/evals`,{name:e.name,prompt:e.prompt,expectedOutput:e["expected-output"]??"",expectations:e.expectations??[]});this.log(`Created eval: ${n.name} (${n.id})`)}};export{s as default};

package/dist/commands/evals/optimize-description.js

@@ -1,169 +1,28 @@
-import {
-  computeTriggerScore,
-  testTrigger
-} from "../../chunk-4D537GXE.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost,
-  apiPut
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/optimize-description.ts
-import { Args, Flags } from "@oclif/core";
-var EvalOptimizeDescriptionCommand = class _EvalOptimizeDescriptionCommand extends BaseCommand {
-  static description = "Optimize eval target description for better triggering accuracy";
-  static examples = [
-    "<%= config.bin %> evals optimize-description <target-id>",
-    "<%= config.bin %> evals optimize-description <target-id> --queries 10"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    "max-iterations": Flags.integer({ description: "Maximum optimization iterations", default: 5 }),
-    model: Flags.string({ description: "Model for trigger testing and improvement" }),
-    queries: Flags.integer({ description: "Number of trigger queries to generate", default: 20 }),
-    "runs-per-query": Flags.integer({ description: "Runs per query for reliability", default: 3 }),
-    "poll-interval": Flags.integer({ description: "Ms between approval polls", default: 5e3 }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalOptimizeDescriptionCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    this.log(`Optimizing description for "${target.name}" v${target.version}`);
-    this.log(`Current description: "${target.description}"`);
-    this.log("");
-    this.log("Generating trigger queries...");
-    const queries = await this.generateTriggerQueries(target, flags.queries, flags);
-    const triggerEval = await apiPost(`/api/eval-targets/${args.targetId}/trigger-evals`, {
-      queries
-    });
-    this.log(`
-Generated ${queries.length} trigger queries.`);
-    this.log(`Review and approve them at:`);
-    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);
-    this.log(`  (Triggers tab \u2192 click to review)`);
-    this.log("\nWaiting for approval...");
-    let approved = false;
-    while (!approved) {
-      await new Promise((resolve) => setTimeout(resolve, flags["poll-interval"]));
-      const sets = await apiGet(
-        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`
-      );
-      const current = sets.find((s) => s.id === triggerEval.id);
-      if (current) {
-        approved = true;
-        queries.length = 0;
-        queries.push(...current.queries);
-        this.log("Trigger queries approved!");
-      }
-    }
-    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);
-    const splitIdx = Math.ceil(indices.length * 0.6);
-    const trainIndices = indices.slice(0, splitIdx);
-    const testIndices = indices.slice(splitIdx);
-    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {
-      trainIndices,
-      testIndices
-    });
-    const trainQueries = trainIndices.map((i) => queries[i]);
-    const testQueries = testIndices.map((i) => queries[i]);
-    this.log(`
-Split: ${trainQueries.length} train, ${testQueries.length} test`);
-    let currentDescription = target.description;
-    let bestDescription = currentDescription;
-    let bestTestScore = 0;
-    for (let iteration = 1; iteration <= flags["max-iterations"]; iteration++) {
-      this.log(`
-=== Iteration ${iteration} ===`);
-      this.log(`Description: "${currentDescription}"`);
-      this.log("Testing train set...");
-      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);
-      const trainScore = computeTriggerScore(trainResults);
-      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);
-      this.printTriggerResults(trainResults);
-      this.log("Testing test set...");
-      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);
-      const testScore = computeTriggerScore(testResults);
-      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);
-      await apiPost("/api/optimization-runs", {
-        evalTargetId: target.id,
-        type: "description",
-        iteration,
-        inputVersion: target.version,
-        trainScore,
-        testScore,
-        analysis: {
-          description: currentDescription,
-          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),
-          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct }))
-        },
-        status: "completed"
-      });
-      if (testScore > bestTestScore) {
-        bestTestScore = testScore;
-        bestDescription = currentDescription;
-      }
-      if (trainScore >= 1) {
-        this.log("Perfect train score. Stopping.");
-        break;
-      }
-      this.log("Generating improved description...");
-      const failures = trainResults.filter((r) => !r.correct);
-      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);
-    }
-    this.log(`
-=== Optimization Complete ===`);
-    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);
-    this.log(`
-Before: "${target.description}"`);
-    this.log(`After:  "${bestDescription}"`);
-    if (bestDescription !== target.description) {
-      await apiPut(`/api/eval-targets/${target.id}`, {
-        description: bestDescription
-      });
-      this.log("\nDescription updated.");
-    } else {
-      this.log("\nNo improvement found. Description unchanged.");
-    }
-  }
-  async generateTriggerQueries(target, count, flags) {
-    let sessionSeed = "";
-    try {
-      const sessions = await apiGet("/api/sessions?limit=10");
-      if (sessions.length > 0) {
-        sessionSeed = "\n\nReal user prompts from recent sessions (use as inspiration for realistic queries):\n" + sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join("\n");
-      }
-    } catch {
-    }
-    const shouldTriggerCount = Math.ceil(count / 2);
-    const shouldNotTriggerCount = count - shouldTriggerCount;
-    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.
+import{a as O,b as S}from"../../chunk-EUHNCVJQ.js";import{a as w}from"../../chunk-MGHFDHHM.js";import{a as N}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{d as y,e as $,f as b}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as D}from"../../chunk-ZLLER3HP.js";import{Args as C,Flags as T}from"@oclif/core";var x=class k extends N{static{D(this,"EvalOptimizeDescriptionCommand")}static description="Optimize eval target description for better triggering accuracy";static examples=["<%= config.bin %> evals optimize-description <target-id>","<%= config.bin %> evals optimize-description <target-id> --queries 10"];static args={targetId:C.string({description:"Eval target ID",required:!0})};static flags={"max-iterations":T.integer({description:"Maximum optimization iterations",default:5}),model:T.string({description:"Model for trigger testing and improvement"}),queries:T.integer({description:"Number of trigger queries to generate",default:20}),"runs-per-query":T.integer({description:"Runs per query for reliability",default:3}),"poll-interval":T.integer({description:"Ms between approval polls",default:5e3}),timeout:T.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:i,flags:t}=await this.parse(k);this.requireAuth();let e=await y(`/api/eval-targets/${i.targetId}`);this.log(`Optimizing description for "${e.name}" v${e.version}`),this.log(`Current description: "${e.description}"`),this.log(""),this.log("Generating trigger queries...");let r=await this.generateTriggerQueries(e,t.queries,t),n=await $(`/api/eval-targets/${i.targetId}/trigger-evals`,{queries:r});this.log(`
+Generated ${r.length} trigger queries.`),this.log("Review and approve them at:"),this.log(`  http://localhost:5174/eval-targets/${i.targetId}`),this.log("  (Triggers tab \u2192 click to review)"),this.log(`
+Waiting for approval...`);let a=!1;for(;!a;){await new Promise(h=>setTimeout(h,t["poll-interval"]));let d=(await y(`/api/eval-targets/${i.targetId}/trigger-evals?status=approved`)).find(h=>h.id===n.id);d&&(a=!0,r.length=0,r.push(...d.queries),this.log("Trigger queries approved!"))}let g=r.map((u,d)=>d).sort(()=>Math.random()-.5),p=Math.ceil(g.length*.6),s=g.slice(0,p),o=g.slice(p);await b(`/api/trigger-evals/${n.id}`,{trainIndices:s,testIndices:o});let c=s.map(u=>r[u]),I=o.map(u=>r[u]);this.log(`
+Split: ${c.length} train, ${I.length} test`);let m=e.description,v=m,q=0;for(let u=1;u<=t["max-iterations"];u++){this.log(`
+=== Iteration ${u} ===`),this.log(`Description: "${m}"`),this.log("Testing train set...");let d=await this.testQuerySet(c,e,m,t),h=S(d);this.log(`Train accuracy: ${(h*100).toFixed(1)}%`),this.printTriggerResults(d),this.log("Testing test set...");let R=await this.testQuerySet(I,e,m,t),f=S(R);if(this.log(`Test accuracy: ${(f*100).toFixed(1)}%`),await $("/api/optimization-runs",{evalTargetId:e.id,type:"description",iteration:u,inputVersion:e.version,trainScore:h,testScore:f,analysis:{description:m,trainResults:d.map(l=>({query:l.query,shouldTrigger:l.shouldTrigger,didTrigger:l.didTrigger,correct:l.correct})),testResults:R.map(l=>({query:l.query,shouldTrigger:l.shouldTrigger,didTrigger:l.didTrigger,correct:l.correct}))},status:"completed"}),f>q&&(q=f,v=m),h>=1){this.log("Perfect train score. Stopping.");break}this.log("Generating improved description...");let Q=d.filter(l=>!l.correct);m=await this.improveDescription(e,m,Q,t)}this.log(`
+=== Optimization Complete ===`),this.log(`Best test accuracy: ${(q*100).toFixed(1)}%`),this.log(`
+Before: "${e.description}"`),this.log(`After:  "${v}"`),v!==e.description?(await b(`/api/eval-targets/${e.id}`,{description:v}),this.log(`
+Description updated.`)):this.log(`
+No improvement found. Description unchanged.`)}async generateTriggerQueries(i,t,e){let r="";try{let s=await y("/api/sessions?limit=10");s.length>0&&(r=`
+
+Real user prompts from recent sessions (use as inspiration for realistic queries):
+`+s.slice(0,5).map(o=>`- Session ${o.id}`).join(`
+`))}catch{}let n=Math.ceil(t/2),a=t-n,g=`Generate ${t} trigger eval queries for a skill/instruction.
 
 ## Skill Name
-${target.name}
+${i.name}
 
 ## Skill Description
-${target.description}
+${i.description}
 
 ## Skill Content (first 1000 chars)
-${target.content.slice(0, 1e3)}
-${sessionSeed}
+${i.content.slice(0,1e3)}
+${r}
 
-Generate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.
+Generate ${n} should-trigger queries and ${a} should-not-trigger queries.
 
 Rules:
 - Queries must be realistic \u2014 what a real user would actually type
@@ -176,81 +35,20 @@ Respond with ONLY a valid JSON array:
 [
   {"query": "realistic user prompt...", "should_trigger": true},
   {"query": "another prompt...", "should_trigger": false}
-]`;
-    const result = await executeEval({
-      prompt,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    try {
-      let jsonStr = result.output.trim();
-      const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
-      if (jsonMatch) jsonStr = jsonMatch[1].trim();
-      const arrMatch = jsonStr.match(/\[[\s\S]*\]/);
-      if (arrMatch) jsonStr = arrMatch[0];
-      return JSON.parse(jsonStr);
-    } catch {
-      return [
-        { query: `Help me use ${target.name}`, should_trigger: true },
-        { query: "Write a hello world program", should_trigger: false }
-      ];
-    }
-  }
-  async testQuerySet(queries, target, description, flags) {
-    const results = [];
-    for (const q of queries) {
-      const runs = [];
-      for (let run = 0; run < flags["runs-per-query"]; run++) {
-        const result = await testTrigger({
-          query: q.query,
-          shouldTrigger: q.should_trigger,
-          skillName: target.name,
-          skillDescription: description,
-          skillContent: target.content,
-          model: flags.model,
-          timeout: flags.timeout
-        });
-        runs.push(result);
-      }
-      const triggerCount = runs.filter((r) => r.didTrigger).length;
-      const didTrigger = triggerCount > flags["runs-per-query"] / 2;
-      results.push({
-        query: q.query,
-        shouldTrigger: q.should_trigger,
-        didTrigger,
-        correct: didTrigger === q.should_trigger,
-        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,
-        evidence: runs[0].evidence
-      });
-    }
-    return results;
-  }
-  printTriggerResults(results) {
-    for (const r of results) {
-      const status = r.correct ? "OK" : "FAIL";
-      const expected = r.shouldTrigger ? "should trigger" : "should NOT trigger";
-      const actual = r.didTrigger ? "triggered" : "did NOT trigger";
-      this.log(`  [${status}] ${expected} / ${actual}: "${r.query.slice(0, 60)}..."`);
-    }
-  }
-  async improveDescription(target, currentDescription, failures, flags) {
-    const failureText = failures.map((f) => {
-      const expected = f.shouldTrigger ? "SHOULD trigger but DID NOT" : "SHOULD NOT trigger but DID";
-      return `- ${expected}: "${f.query}"`;
-    }).join("\n");
-    const prompt = `You are optimizing a skill description for better triggering accuracy.
+]`,p=await w({prompt:g,timeout:e.timeout,model:e.model});try{let s=p.output.trim(),o=s.match(/```(?:json)?\s*([\s\S]*?)```/);o&&(s=o[1].trim());let c=s.match(/\[[\s\S]*\]/);return c&&(s=c[0]),JSON.parse(s)}catch{return[{query:`Help me use ${i.name}`,should_trigger:!0},{query:"Write a hello world program",should_trigger:!1}]}}async testQuerySet(i,t,e,r){let n=[];for(let a of i){let g=[];for(let o=0;o<r["runs-per-query"];o++){let c=await O({query:a.query,shouldTrigger:a.should_trigger,skillName:t.name,skillDescription:e,skillContent:t.content,model:r.model,timeout:r.timeout});g.push(c)}let s=g.filter(o=>o.didTrigger).length>r["runs-per-query"]/2;n.push({query:a.query,shouldTrigger:a.should_trigger,didTrigger:s,correct:s===a.should_trigger,confidence:g.reduce((o,c)=>o+c.confidence,0)/g.length,evidence:g[0].evidence})}return n}printTriggerResults(i){for(let t of i){let e=t.correct?"OK":"FAIL",r=t.shouldTrigger?"should trigger":"should NOT trigger",n=t.didTrigger?"triggered":"did NOT trigger";this.log(`  [${e}] ${r} / ${n}: "${t.query.slice(0,60)}..."`)}}async improveDescription(i,t,e,r){let n=e.map(p=>`- ${p.shouldTrigger?"SHOULD trigger but DID NOT":"SHOULD NOT trigger but DID"}: "${p.query}"`).join(`
+`),a=`You are optimizing a skill description for better triggering accuracy.
 
 ## Skill Name
-${target.name}
+${i.name}
 
 ## Current Description
-${currentDescription}
+${t}
 
 ## Skill Content (first 500 chars)
-${target.content.slice(0, 500)}
+${i.content.slice(0,500)}
 
 ## Failures
-${failureText}
+${n}
 
 Propose an improved description that:
 - Fixes the trigger failures above
@@ -258,16 +56,4 @@ Propose an improved description that:
 - Uses specific keywords and contexts for when to trigger
 - Includes "Use PROACTIVELY" phrasing for important trigger contexts
 
-Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;
-    const result = await executeEval({
-      prompt,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    return result.output.trim().replace(/^["']|["']$/g, "");
-  }
-};
-export {
-  EvalOptimizeDescriptionCommand as default
-};
-//# sourceMappingURL=optimize-description.js.map
+Respond with ONLY the new description text (no JSON, no explanation, just the description string).`;return(await w({prompt:a,timeout:r.timeout,model:r.model})).output.trim().replace(/^["']|["']$/g,"")}};export{x as default};