@rulemetric/cli 0.7.3 → 0.7.4

package/dist/commands/evals/optimize.js

@@ -1,211 +1,5 @@
-import {
-  analyzeEvalResults
-} from "../../chunk-3TIMQ3O6.js";
-import {
-  gradeEvalOutput
-} from "../../chunk-YNIH3KMU.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost,
-  apiPut
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/optimize.ts
-import { Args, Flags } from "@oclif/core";
-var EvalOptimizeCommand = class _EvalOptimizeCommand extends BaseCommand {
-  static description = "Iteratively optimize eval target content using eval results as feedback";
-  static examples = [
-    "<%= config.bin %> evals optimize <target-id>",
-    "<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    auto: Flags.boolean({ description: "Skip interactive review, apply suggestions automatically", default: false }),
-    "max-iterations": Flags.integer({ description: "Maximum optimization iterations", default: 5 }),
-    threshold: Flags.string({ description: "Stop if pass rate reaches this threshold (0-1)", default: "0.95" }),
-    model: Flags.string({ description: "Model for eval execution" }),
-    "analyzer-model": Flags.string({ description: "Model for analysis/suggestions" }),
-    "runs-per-eval": Flags.integer({ description: "Runs per eval per iteration", default: 1 }),
-    parallel: Flags.integer({ description: "Number of parallel eval executions", default: 1 }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalOptimizeCommand);
-    this.requireAuth();
-    let target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    const allEvals = await apiGet(`/api/eval-targets/${args.targetId}/evals`);
-    if (allEvals.length === 0) {
-      this.error("No evals found. Create some first with `rulemetric evals create`.");
-    }
-    const shuffled = [...allEvals].sort(() => Math.random() - 0.5);
-    const splitIdx = Math.ceil(shuffled.length * 0.6);
-    const trainEvals = shuffled.slice(0, splitIdx);
-    const testEvals = shuffled.slice(splitIdx);
-    this.log(`Optimizing "${target.name}" v${target.version}`);
-    this.log(`Evals: ${trainEvals.length} train, ${testEvals.length} test`);
-    const threshold = parseFloat(flags.threshold);
-    this.log(`Config: max ${flags["max-iterations"]} iterations, threshold ${(threshold * 100).toFixed(0)}%`);
-    if (flags.auto) this.log("Mode: auto (no interactive review)");
-    this.log("");
-    let bestVersion = target.version;
-    let bestTestScore = 0;
-    let noImprovementCount = 0;
-    for (let iteration = 1; iteration <= flags["max-iterations"]; iteration++) {
-      this.log(`
-=== Iteration ${iteration} ===`);
-      const optRun = await apiPost("/api/optimization-runs", {
-        evalTargetId: target.id,
-        type: "content",
-        iteration,
-        inputVersion: target.version,
-        status: "running"
-      });
-      this.log("Running train evals...");
-      const trainResults = await this.runEvalsOnTarget(target, trainEvals, flags);
-      const trainScore = this.computePassRate(trainResults);
-      this.log(`Train pass rate: ${(trainScore * 100).toFixed(1)}%`);
-      if (trainScore >= threshold) {
-        this.log(`Train score ${(trainScore * 100).toFixed(1)}% >= threshold ${(threshold * 100).toFixed(0)}%. Stopping.`);
-        const testResults2 = await this.runEvalsOnTarget(target, testEvals, flags);
-        const testScore2 = this.computePassRate(testResults2);
-        this.log(`Final test score: ${(testScore2 * 100).toFixed(1)}%`);
-        await apiPost("/api/optimization-runs", {
-          evalTargetId: target.id,
-          type: "content",
-          iteration,
-          inputVersion: target.version,
-          trainScore,
-          testScore: testScore2,
-          analysis: { reason: "threshold_reached" },
-          status: "completed"
-        });
-        break;
-      }
-      this.log("Analyzing results...");
-      const evalResults = trainResults.map((r) => ({
-        evalName: r.evalName,
-        prompt: r.prompt,
-        expectations: r.expectations,
-        grading: r.grading
-      }));
-      const analysis = await analyzeEvalResults({
-        content: target.content,
-        evalResults,
-        model: flags["analyzer-model"] ?? flags.model,
-        timeout: flags.timeout
-      });
-      this.log(`
-Assessment: ${analysis.overallAssessment}`);
-      this.log(`Suggestions: ${analysis.suggestions.length}`);
-      for (const s of analysis.suggestions) {
-        this.log(`  - [${s.section}] ${s.reasoning}`);
-      }
-      if (!flags.auto) {
-        this.log("\nProposed changes above. Applying in auto mode would proceed automatically.");
-        this.log("(Interactive approval not yet implemented \u2014 use --auto for now)");
-      }
-      if (analysis.proposedContent === target.content) {
-        this.log("No changes proposed. Stopping.");
-        await apiPost("/api/optimization-runs", {
-          evalTargetId: target.id,
-          type: "content",
-          iteration,
-          inputVersion: target.version,
-          trainScore,
-          analysis: { reason: "no_changes" },
-          status: "completed"
-        });
-        break;
-      }
-      const newTarget = await apiPut(`/api/eval-targets/${target.id}`, {
-        content: analysis.proposedContent
-      });
-      this.log(`Created v${newTarget.version}`);
-      this.log("Running test evals...");
-      const testResults = await this.runEvalsOnTarget(newTarget, testEvals, flags);
-      const testScore = this.computePassRate(testResults);
-      this.log(`Test pass rate: ${(testScore * 100).toFixed(1)}%`);
-      await apiPost("/api/optimization-runs", {
-        evalTargetId: newTarget.id,
-        type: "content",
-        iteration,
-        inputVersion: target.version,
-        outputVersion: newTarget.version,
-        trainScore,
-        testScore,
-        analysis: {
-          suggestions: analysis.suggestions,
-          overallAssessment: analysis.overallAssessment
-        },
-        status: "completed"
-      });
-      if (testScore > bestTestScore) {
-        bestTestScore = testScore;
-        bestVersion = newTarget.version;
-        noImprovementCount = 0;
-      } else {
-        noImprovementCount++;
-        if (noImprovementCount >= 2) {
-          this.log("No improvement for 2 consecutive iterations. Stopping.");
-          break;
-        }
-      }
-      target = newTarget;
-    }
-    this.log(`
-=== Optimization Complete ===`);
-    this.log(`Best version: v${bestVersion} (test score: ${(bestTestScore * 100).toFixed(1)}%)`);
-  }
-  async runEvalsOnTarget(target, evalSet, flags) {
-    const results = [];
-    for (const eval_ of evalSet) {
-      const execResult = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: target.content,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      if (execResult.error) {
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
-        continue;
-      }
-      if (eval_.expectations.length > 0) {
-        const grading = await gradeEvalOutput({
-          prompt: eval_.prompt,
-          output: execResult.output,
-          expectations: eval_.expectations,
-          model: flags.model,
-          timeout: flags.timeout
-        });
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading });
-      } else {
-        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });
-      }
-    }
-    return results;
-  }
-  computePassRate(results) {
-    const graded = results.filter((r) => Object.keys(r.grading).length > 0);
-    if (graded.length === 0) return 0;
-    const rates = graded.map((r) => r.grading.summary?.pass_rate ?? 0);
-    return rates.reduce((a, b) => a + b, 0) / rates.length;
-  }
-};
-export {
-  EvalOptimizeCommand as default
-};
-//# sourceMappingURL=optimize.js.map
+import{a as A}from"../../chunk-DSO2U4Y6.js";import{a as k}from"../../chunk-CADKBP3D.js";import{a as I}from"../../chunk-MGHFDHHM.js";import{a as z}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{d as y,e as c,f as T}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as R}from"../../chunk-ZLLER3HP.js";import{Args as P,Flags as o}from"@oclif/core";var w=class N extends z{static{R(this,"EvalOptimizeCommand")}static description="Iteratively optimize eval target content using eval results as feedback";static examples=["<%= config.bin %> evals optimize <target-id>","<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3"];static args={targetId:P.string({description:"Eval target ID",required:!0})};static flags={auto:o.boolean({description:"Skip interactive review, apply suggestions automatically",default:!1}),"max-iterations":o.integer({description:"Maximum optimization iterations",default:5}),threshold:o.string({description:"Stop if pass rate reaches this threshold (0-1)",default:"0.95"}),model:o.string({description:"Model for eval execution"}),"analyzer-model":o.string({description:"Model for analysis/suggestions"}),"runs-per-eval":o.integer({description:"Runs per eval per iteration",default:1}),parallel:o.integer({description:"Number of parallel eval executions",default:1}),timeout:o.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:l,flags:s}=await this.parse(N);this.requireAuth();let t=await y(`/api/eval-targets/${l.targetId}`),i=await y(`/api/eval-targets/${l.targetId}/evals`);i.length===0&&this.error("No evals found. Create some first with `rulemetric evals create`.");let e=[...i].sort(()=>Math.random()-.5),u=Math.ceil(e.length*.6),m=e.slice(0,u),v=e.slice(u);this.log(`Optimizing "${t.name}" v${t.version}`),this.log(`Evals: ${m.length} train, ${v.length} test`);let h=parseFloat(s.threshold);this.log(`Config: max ${s["max-iterations"]} iterations, threshold ${(h*100).toFixed(0)}%`),s.auto&&this.log("Mode: auto (no interactive review)"),this.log("");let $=t.version,f=0,x=0;for(let n=1;n<=s["max-iterations"];n++){this.log(`
+=== Iteration ${n} ===`);let C=await c("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:n,inputVersion:t.version,status:"running"});this.log("Running train evals...");let b=await this.runEvalsOnTarget(t,m,s),p=this.computePassRate(b);if(this.log(`Train pass rate: ${(p*100).toFixed(1)}%`),p>=h){this.log(`Train score ${(p*100).toFixed(1)}% >= threshold ${(h*100).toFixed(0)}%. Stopping.`);let a=await this.runEvalsOnTarget(t,v,s),E=this.computePassRate(a);this.log(`Final test score: ${(E*100).toFixed(1)}%`),await c("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:n,inputVersion:t.version,trainScore:p,testScore:E,analysis:{reason:"threshold_reached"},status:"completed"});break}this.log("Analyzing results...");let F=b.map(a=>({evalName:a.evalName,prompt:a.prompt,expectations:a.expectations,grading:a.grading})),r=await A({content:t.content,evalResults:F,model:s["analyzer-model"]??s.model,timeout:s.timeout});this.log(`
+Assessment: ${r.overallAssessment}`),this.log(`Suggestions: ${r.suggestions.length}`);for(let a of r.suggestions)this.log(`  - [${a.section}] ${a.reasoning}`);if(s.auto||(this.log(`
+Proposed changes above. Applying in auto mode would proceed automatically.`),this.log("(Interactive approval not yet implemented \u2014 use --auto for now)")),r.proposedContent===t.content){this.log("No changes proposed. Stopping."),await c("/api/optimization-runs",{evalTargetId:t.id,type:"content",iteration:n,inputVersion:t.version,trainScore:p,analysis:{reason:"no_changes"},status:"completed"});break}let g=await T(`/api/eval-targets/${t.id}`,{content:r.proposedContent});this.log(`Created v${g.version}`),this.log("Running test evals...");let S=await this.runEvalsOnTarget(g,v,s),d=this.computePassRate(S);if(this.log(`Test pass rate: ${(d*100).toFixed(1)}%`),await c("/api/optimization-runs",{evalTargetId:g.id,type:"content",iteration:n,inputVersion:t.version,outputVersion:g.version,trainScore:p,testScore:d,analysis:{suggestions:r.suggestions,overallAssessment:r.overallAssessment},status:"completed"}),d>f)f=d,$=g.version,x=0;else if(x++,x>=2){this.log("No improvement for 2 consecutive iterations. Stopping.");break}t=g}this.log(`
+=== Optimization Complete ===`),this.log(`Best version: v${$} (test score: ${(f*100).toFixed(1)}%)`)}async runEvalsOnTarget(l,s,t){let i=[];for(let e of s){let u=await I({prompt:e.prompt,targetContent:l.content,timeout:t.timeout,model:t.model});if(u.error){i.push({evalName:e.name,prompt:e.prompt,expectations:e.expectations,grading:{}});continue}if(e.expectations.length>0){let m=await k({prompt:e.prompt,output:u.output,expectations:e.expectations,model:t.model,timeout:t.timeout});i.push({evalName:e.name,prompt:e.prompt,expectations:e.expectations,grading:m})}else i.push({evalName:e.name,prompt:e.prompt,expectations:e.expectations,grading:{}})}return i}computePassRate(l){let s=l.filter(i=>Object.keys(i.grading).length>0);if(s.length===0)return 0;let t=s.map(i=>i.grading.summary?.pass_rate??0);return t.reduce((i,e)=>i+e,0)/t.length}};export{w as default};

package/dist/commands/evals/run.js

@@ -1,183 +1,3 @@
-import {
-  gradeEvalOutput
-} from "../../chunk-YNIH3KMU.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/run.ts
-import { Args, Flags } from "@oclif/core";
-var EvalRunCommand = class _EvalRunCommand extends BaseCommand {
-  static description = "Run evals against a target, optionally grade results, and post to API";
-  static examples = [
-    "<%= config.bin %> evals run <target-id>",
-    "<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2",
-    "<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    parallel: Flags.integer({ description: "Number of parallel eval executions", default: 1 }),
-    "runs-per-eval": Flags.integer({ description: "Number of runs per eval for variance analysis", default: 1 }),
-    iteration: Flags.integer({ description: "Iteration number for tracking", default: 1 }),
-    "skip-baseline": Flags.boolean({ description: "Skip baseline (without_target) runs", default: false }),
-    "skip-grading": Flags.boolean({ description: "Skip LLM grading step", default: false }),
-    model: Flags.string({ description: "Model for eval execution (default: user configured)" }),
-    "grader-model": Flags.string({ description: "Model for grading (default: same as execution model)" }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalRunCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    const evals = await apiGet(`/api/eval-targets/${args.targetId}/evals`);
-    if (evals.length === 0) {
-      this.error(`No evals found for target "${target.name}". Create some with \`rulemetric evals create\`.`);
-    }
-    this.log(`Running ${evals.length} eval(s) for "${target.name}" v${target.version}`);
-    this.log(`Config: ${flags["runs-per-eval"]} run(s) per eval, parallel=${flags.parallel}, iteration=${flags.iteration}`);
-    if (flags["skip-baseline"]) this.log("Skipping baseline runs");
-    if (flags["skip-grading"]) this.log("Skipping grading");
-    this.log("");
-    const configurations = [
-      { name: "with_target", content: target.content }
-    ];
-    if (!flags["skip-baseline"]) {
-      configurations.push({ name: "without_target" });
-    }
-    const workQueue = [];
-    for (const eval_ of evals) {
-      for (const config of configurations) {
-        for (let run = 1; run <= flags["runs-per-eval"]; run++) {
-          workQueue.push({
-            eval_,
-            configuration: config.name,
-            targetContent: config.content,
-            runNumber: run
-          });
-        }
-      }
-    }
-    const results = [];
-    const concurrency = flags.parallel;
-    for (let i = 0; i < workQueue.length; i += concurrency) {
-      const batch = workQueue.slice(i, i + concurrency);
-      const batchResults = await Promise.all(
-        batch.map((item) => this.executeAndGrade(item, target, flags))
-      );
-      results.push(...batchResults);
-    }
-    this.log("");
-    this.log("Posting results to API...");
-    for (const result of results) {
-      await apiPost("/api/eval-runs", result);
-    }
-    this.log("");
-    this.log("=== Results Summary ===");
-    for (const config of configurations) {
-      const configResults = results.filter((r) => r.configuration === config.name);
-      const completedRuns = configResults.filter((r) => r.status === "completed");
-      const gradedRuns = completedRuns.filter((r) => Object.keys(r.grading).length > 0);
-      this.log(`
-${config.name}:`);
-      this.log(`  Runs: ${completedRuns.length}/${configResults.length} completed`);
-      if (gradedRuns.length > 0) {
-        const passRates = gradedRuns.map((r) => r.grading.summary?.pass_rate ?? 0);
-        const avgPassRate = passRates.reduce((a, b) => a + b, 0) / passRates.length;
-        this.log(`  Avg pass_rate: ${(avgPassRate * 100).toFixed(1)}%`);
-      }
-      for (const r of configResults) {
-        const evalName = evals.find((e) => e.id === r.evalId)?.name ?? r.evalId;
-        const passRate = r.grading.summary?.pass_rate;
-        const passRateStr = passRate !== void 0 ? ` pass_rate=${(passRate * 100).toFixed(0)}%` : "";
-        const timeStr = r.timing.duration_ms ? ` ${(r.timing.duration_ms / 1e3).toFixed(1)}s` : "";
-        this.log(`  [${r.status}] ${evalName} run#${r.runNumber}${passRateStr}${timeStr}`);
-      }
-    }
-    if (!flags["skip-baseline"] && results.length > 0) {
-      const withTarget = results.filter((r) => r.configuration === "with_target" && Object.keys(r.grading).length > 0);
-      const withoutTarget = results.filter((r) => r.configuration === "without_target" && Object.keys(r.grading).length > 0);
-      if (withTarget.length > 0 && withoutTarget.length > 0) {
-        const avgWith = withTarget.map((r) => r.grading.summary?.pass_rate ?? 0).reduce((a, b) => a + b, 0) / withTarget.length;
-        const avgWithout = withoutTarget.map((r) => r.grading.summary?.pass_rate ?? 0).reduce((a, b) => a + b, 0) / withoutTarget.length;
-        const delta = avgWith - avgWithout;
-        this.log(`
-Delta: ${delta > 0 ? "+" : ""}${(delta * 100).toFixed(1)}% (with_target vs without_target)`);
-      }
-    }
-  }
-  async executeAndGrade(item, target, flags) {
-    const { eval_, configuration, targetContent, runNumber } = item;
-    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 executing...`);
-    const execResult = await executeEval({
-      prompt: eval_.prompt,
-      targetContent,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    if (execResult.error) {
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 FAILED: ${execResult.error}`);
-      return {
-        evalId: eval_.id,
-        evalTargetId: target.id,
-        evalTargetVersion: target.version,
-        configuration,
-        runNumber,
-        iteration: flags.iteration,
-        status: "failed",
-        outputs: { error: execResult.error },
-        timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },
-        grading: {}
-      };
-    }
-    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 completed in ${(execResult.durationMs / 1e3).toFixed(1)}s`);
-    let grading = {};
-    if (!flags["skip-grading"] && eval_.expectations.length > 0) {
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading...`);
-      const gradingResult = await gradeEvalOutput({
-        prompt: eval_.prompt,
-        output: execResult.output,
-        expectations: eval_.expectations,
-        model: flags["grader-model"] ?? flags.model,
-        timeout: flags.timeout
-      });
-      grading = gradingResult;
-      const passRate = gradingResult.summary.pass_rate;
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading done: pass_rate=${(passRate * 100).toFixed(0)}%`);
-    }
-    return {
-      evalId: eval_.id,
-      evalTargetId: target.id,
-      evalTargetVersion: target.version,
-      configuration,
-      runNumber,
-      iteration: flags.iteration,
-      status: "completed",
-      outputs: { transcript: execResult.output },
-      timing: {
-        duration_ms: execResult.durationMs,
-        total_tokens: execResult.totalTokens,
-        total_duration_seconds: execResult.durationMs / 1e3
-      },
-      grading
-    };
-  }
-};
-export {
-  EvalRunCommand as default
-};
-//# sourceMappingURL=run.js.map
+import{a as w}from"../../chunk-CADKBP3D.js";import{a as _}from"../../chunk-MGHFDHHM.js";import{a as R}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import{d as h,e as k}from"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a as $}from"../../chunk-ZLLER3HP.js";import{Args as T,Flags as c}from"@oclif/core";var b=class x extends R{static{$(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:T.string({description:"Eval target ID",required:!0})};static flags={parallel:c.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":c.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:c.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":c.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":c.boolean({description:"Skip LLM grading step",default:!1}),model:c.string({description:"Model for eval execution (default: user configured)"}),"grader-model":c.string({description:"Model for grading (default: same as execution model)"}),timeout:c.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:f,flags:i}=await this.parse(x);this.requireAuth();let s=await h(`/api/eval-targets/${f.targetId}`),r=await h(`/api/eval-targets/${f.targetId}/evals`);r.length===0&&this.error(`No evals found for target "${s.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${r.length} eval(s) for "${s.name}" v${s.version}`),this.log(`Config: ${i["runs-per-eval"]} run(s) per eval, parallel=${i.parallel}, iteration=${i.iteration}`),i["skip-baseline"]&&this.log("Skipping baseline runs"),i["skip-grading"]&&this.log("Skipping grading"),this.log("");let l=[{name:"with_target",content:s.content}];i["skip-baseline"]||l.push({name:"without_target"});let p=[];for(let e of r)for(let a of l)for(let n=1;n<=i["runs-per-eval"];n++)p.push({eval_:e,configuration:a.name,targetContent:a.content,runNumber:n});let o=[],g=i.parallel;for(let e=0;e<p.length;e+=g){let a=p.slice(e,e+g),n=await Promise.all(a.map(m=>this.executeAndGrade(m,s,i)));o.push(...n)}this.log(""),this.log("Posting results to API...");for(let e of o)await k("/api/eval-runs",e);this.log(""),this.log("=== Results Summary ===");for(let e of l){let a=o.filter(t=>t.configuration===e.name),n=a.filter(t=>t.status==="completed"),m=n.filter(t=>Object.keys(t.grading).length>0);if(this.log(`
+${e.name}:`),this.log(`  Runs: ${n.length}/${a.length} completed`),m.length>0){let t=m.map(d=>d.grading.summary?.pass_rate??0),u=t.reduce((d,v)=>d+v,0)/t.length;this.log(`  Avg pass_rate: ${(u*100).toFixed(1)}%`)}for(let t of a){let u=r.find(I=>I.id===t.evalId)?.name??t.evalId,d=t.grading.summary?.pass_rate,v=d!==void 0?` pass_rate=${(d*100).toFixed(0)}%`:"",y=t.timing.duration_ms?` ${(t.timing.duration_ms/1e3).toFixed(1)}s`:"";this.log(`  [${t.status}] ${u} run#${t.runNumber}${v}${y}`)}}if(!i["skip-baseline"]&&o.length>0){let e=o.filter(n=>n.configuration==="with_target"&&Object.keys(n.grading).length>0),a=o.filter(n=>n.configuration==="without_target"&&Object.keys(n.grading).length>0);if(e.length>0&&a.length>0){let n=e.map(u=>u.grading.summary?.pass_rate??0).reduce((u,d)=>u+d,0)/e.length,m=a.map(u=>u.grading.summary?.pass_rate??0).reduce((u,d)=>u+d,0)/a.length,t=n-m;this.log(`
+Delta: ${t>0?"+":""}${(t*100).toFixed(1)}% (with_target vs without_target)`)}}}async executeAndGrade(f,i,s){let{eval_:r,configuration:l,targetContent:p,runNumber:o}=f;this.log(`[${l}] ${r.name} run#${o} \u2014 executing...`);let g=await _({prompt:r.prompt,targetContent:p,timeout:s.timeout,model:s.model});if(g.error)return this.log(`[${l}] ${r.name} run#${o} \u2014 FAILED: ${g.error}`),{evalId:r.id,evalTargetId:i.id,evalTargetVersion:i.version,configuration:l,runNumber:o,iteration:s.iteration,status:"failed",outputs:{error:g.error},timing:{duration_ms:g.durationMs,total_tokens:g.totalTokens},grading:{}};this.log(`[${l}] ${r.name} run#${o} \u2014 completed in ${(g.durationMs/1e3).toFixed(1)}s`);let e={};if(!s["skip-grading"]&&r.expectations.length>0){this.log(`[${l}] ${r.name} run#${o} \u2014 grading...`);let a=await w({prompt:r.prompt,output:g.output,expectations:r.expectations,model:s["grader-model"]??s.model,timeout:s.timeout});e=a;let n=a.summary.pass_rate;this.log(`[${l}] ${r.name} run#${o} \u2014 grading done: pass_rate=${(n*100).toFixed(0)}%`)}return{evalId:r.id,evalTargetId:i.id,evalTargetVersion:i.version,configuration:l,runNumber:o,iteration:s.iteration,status:"completed",outputs:{transcript:g.output},timing:{duration_ms:g.durationMs,total_tokens:g.totalTokens,total_duration_seconds:g.durationMs/1e3},grading:e}}};export{b as default};

package/dist/commands/gateway/ensure.js

@@ -1,104 +1 @@
-import {
-  repinEditorProxies,
-  unpinEditorProxies
-} from "../../chunk-3S7223UR.js";
-import {
-  writeCursorUserProxyConfig,
-  writeVSCodeUserProxyConfig
-} from "../../chunk-LXKPNATC.js";
-import "../../chunk-OQBBVTDG.js";
-import {
-  GATEWAY_PORT,
-  isGatewayRunning,
-  isRulemetricGatewayListening,
-  spawnGateway
-} from "../../chunk-4LYCDS2N.js";
-import "../../chunk-7LVVKQMZ.js";
-import {
-  MITMPROXY_COMMAND_PATTERN,
-  isPidLiveAndMatching
-} from "../../chunk-5EDMGK3W.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/gateway/ensure.ts
-import { spawn } from "node:child_process";
-import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-var CONFIG_DIR = join(homedir(), ".config", "rulemetric");
-var ADDON_VERSION_FILE = join(CONFIG_DIR, "proxy-addon.version");
-var PROXY_PID_FILE = join(CONFIG_DIR, "proxy.pid");
-var DRIFT_RESTART_MARKER = join(CONFIG_DIR, "proxy-drift-restart.at");
-var DRIFT_RESTART_COOLDOWN_MS = 10 * 6e4;
-var GatewayEnsure = class _GatewayEnsure extends BaseCommand {
-  static description = "Ensure the gateway proxy is running (called by hooks, not for direct use)";
-  static hidden = true;
-  async run() {
-    await this.parse(_GatewayEnsure);
-    this.maybeRestartStaleProxy();
-    const identity = await isRulemetricGatewayListening(GATEWAY_PORT);
-    if (identity.status === "ours-live") {
-      const repinned = repinEditorProxies((editor) => {
-        if (editor === "vscode") writeVSCodeUserProxyConfig(GATEWAY_PORT);
-        else writeCursorUserProxyConfig(GATEWAY_PORT);
-      });
-      if (repinned.length > 0) {
-        console.error(`[RuleMetric] Gateway healthy again \u2014 restored ${repinned.join(", ")} proxy settings.`);
-      }
-      return;
-    }
-    if (identity.status === "foreign") {
-      if (isGatewayRunning()) return;
-      const unpinned = unpinEditorProxies(GATEWAY_PORT);
-      console.error(
-        `[RuleMetric] A non-RuleMetric process holds :${GATEWAY_PORT} \u2014 cannot start gateway. ` + (unpinned.length > 0 ? `Unpinned ${unpinned.join(", ")} proxy settings so editors are not routed through it. ` : "") + "Run `rulemetric diagnostics` if Claude Code HTTPS requests fail."
-      );
-      return;
-    }
-    try {
-      spawnGateway(this.config.version);
-    } catch (err) {
-      console.error(`[RuleMetric] Could not start gateway: ${err.message}`);
-    }
-  }
-  maybeRestartStaleProxy() {
-    try {
-      if (!existsSync(ADDON_VERSION_FILE)) return;
-      const runningVersion = readFileSync(ADDON_VERSION_FILE, "utf-8").trim();
-      if (!runningVersion || runningVersion === this.config.version) return;
-      const pidRaw = readFileSync(PROXY_PID_FILE, "utf-8").trim();
-      const pid = Number(pidRaw);
-      if (!Number.isFinite(pid) || !isPidLiveAndMatching(pid, MITMPROXY_COMMAND_PATTERN)) {
-        return;
-      }
-      try {
-        const age = Date.now() - statSync(DRIFT_RESTART_MARKER).mtimeMs;
-        if (age < DRIFT_RESTART_COOLDOWN_MS) return;
-      } catch {
-      }
-      writeFileSync(DRIFT_RESTART_MARKER, (/* @__PURE__ */ new Date()).toISOString());
-      const child = spawn(process.execPath, [process.argv[1], "proxy", "restart"], {
-        detached: true,
-        stdio: "ignore"
-      });
-      child.unref();
-      console.error(
-        `[RuleMetric] Capture proxy is running CLI ${runningVersion} but ${this.config.version} is installed \u2014 restarting it in the background.`
-      );
-    } catch {
-    }
-  }
-};
-export {
-  GatewayEnsure as default
-};
-//# sourceMappingURL=ensure.js.map
+import{b as y,c as h}from"../../chunk-Y65CSJFF.js";import{B as l,E as g}from"../../chunk-O7ZMHCMG.js";import"../../chunk-I2DZGJRR.js";import{c as e,g as p,i as m,k as f}from"../../chunk-MVM6OGAF.js";import"../../chunk-KKDHD4XA.js";import{b as d,d as u}from"../../chunk-SZ62CP55.js";import{a as c}from"../../chunk-LQRYQE6Q.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-CRAKDQ4H.js";import"../../chunk-WPJ2BND4.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-TIQ7DQ3M.js";import"../../chunk-M6AJDVCL.js";import{a}from"../../chunk-ZLLER3HP.js";import{spawn as S}from"node:child_process";import{existsSync as v,readFileSync as R,statSync as C,writeFileSync as M}from"node:fs";import{homedir as T}from"node:os";import{join as o}from"node:path";var s=o(T(),".config","rulemetric"),x=o(s,"proxy-addon.version"),_=o(s,"proxy.pid"),w=o(s,"proxy-drift-restart.at"),E=10*6e4,n=class P extends c{static{a(this,"GatewayEnsure")}static description="Ensure the gateway proxy is running (called by hooks, not for direct use)";static hidden=!0;async run(){await this.parse(P),this.maybeRestartStaleProxy();let t=await p(e);if(t.status==="ours-live"){let r=h(i=>{i==="vscode"?l(e):g(e)});r.length>0&&console.error(`[RuleMetric] Gateway healthy again \u2014 restored ${r.join(", ")} proxy settings.`);return}if(t.status==="foreign"){if(m())return;let r=y(e);console.error(`[RuleMetric] A non-RuleMetric process holds :${e} \u2014 cannot start gateway. `+(r.length>0?`Unpinned ${r.join(", ")} proxy settings so editors are not routed through it. `:"")+"Run `rulemetric diagnostics` if Claude Code HTTPS requests fail.");return}try{f(this.config.version)}catch(r){console.error(`[RuleMetric] Could not start gateway: ${r.message}`)}}maybeRestartStaleProxy(){try{if(!v(x))return;let t=R(x,"utf-8").trim();if(!t||t===this.config.version)return;let r=R(_,"utf-8").trim(),i=Number(r);if(!Number.isFinite(i)||!u(i,d))return;try{if(Date.now()-C(w).mtimeMs<E)return}catch{}M(w,new Date().toISOString()),S(process.execPath,[process.argv[1],"proxy","restart"],{detached:!0,stdio:"ignore"}).unref(),console.error(`[RuleMetric] Capture proxy is running CLI ${t} but ${this.config.version} is installed \u2014 restarting it in the background.`)}catch{}}};export{n as default};

package/dist/commands/hooks/emit-instructions.js

@@ -1,25 +1 @@
-import {
-  buildInstructionSnapshot
-} from "../../chunk-SO3T35U7.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/hooks/emit-instructions.ts
-import { Command, Flags } from "@oclif/core";
-var HooksEmitInstructions = class _HooksEmitInstructions extends Command {
-  static description = "Print a synthetic instruction snapshot (on-disk CLAUDE.md) as JSON for zero-proxy instruction linking";
-  static hidden = true;
-  static flags = {
-    "project-dir": Flags.string({ description: "Project directory to scan (defaults to cwd)" })
-  };
-  async run() {
-    const { flags } = await this.parse(_HooksEmitInstructions);
-    const snapshot = buildInstructionSnapshot(flags["project-dir"] ?? process.cwd());
-    if (snapshot) {
-      process.stdout.write(JSON.stringify(snapshot));
-    }
-  }
-};
-export {
-  HooksEmitInstructions as default
-};
-//# sourceMappingURL=emit-instructions.js.map
+import{a as s}from"../../chunk-CNQ5DTNC.js";import{a as i}from"../../chunk-ZLLER3HP.js";import{Command as n,Flags as c}from"@oclif/core";var t=class o extends n{static{i(this,"HooksEmitInstructions")}static description="Print a synthetic instruction snapshot (on-disk CLAUDE.md) as JSON for zero-proxy instruction linking";static hidden=!0;static flags={"project-dir":c.string({description:"Project directory to scan (defaults to cwd)"})};async run(){let{flags:e}=await this.parse(o),r=s(e["project-dir"]??process.cwd());r&&process.stdout.write(JSON.stringify(r))}};export{t as default};