@rulemetric/cli 0.7.3 → 0.7.5

package/dist/commands/evals/run.js

@@ -1,183 +1,3 @@
-import {
-  gradeEvalOutput
-} from "../../chunk-YNIH3KMU.js";
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet,
-  apiPost
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/run.ts
-import { Args, Flags } from "@oclif/core";
-var EvalRunCommand = class _EvalRunCommand extends BaseCommand {
-  static description = "Run evals against a target, optionally grade results, and post to API";
-  static examples = [
-    "<%= config.bin %> evals run <target-id>",
-    "<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2",
-    "<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    parallel: Flags.integer({ description: "Number of parallel eval executions", default: 1 }),
-    "runs-per-eval": Flags.integer({ description: "Number of runs per eval for variance analysis", default: 1 }),
-    iteration: Flags.integer({ description: "Iteration number for tracking", default: 1 }),
-    "skip-baseline": Flags.boolean({ description: "Skip baseline (without_target) runs", default: false }),
-    "skip-grading": Flags.boolean({ description: "Skip LLM grading step", default: false }),
-    model: Flags.string({ description: "Model for eval execution (default: user configured)" }),
-    "grader-model": Flags.string({ description: "Model for grading (default: same as execution model)" }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalRunCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    const evals = await apiGet(`/api/eval-targets/${args.targetId}/evals`);
-    if (evals.length === 0) {
-      this.error(`No evals found for target "${target.name}". Create some with \`rulemetric evals create\`.`);
-    }
-    this.log(`Running ${evals.length} eval(s) for "${target.name}" v${target.version}`);
-    this.log(`Config: ${flags["runs-per-eval"]} run(s) per eval, parallel=${flags.parallel}, iteration=${flags.iteration}`);
-    if (flags["skip-baseline"]) this.log("Skipping baseline runs");
-    if (flags["skip-grading"]) this.log("Skipping grading");
-    this.log("");
-    const configurations = [
-      { name: "with_target", content: target.content }
-    ];
-    if (!flags["skip-baseline"]) {
-      configurations.push({ name: "without_target" });
-    }
-    const workQueue = [];
-    for (const eval_ of evals) {
-      for (const config of configurations) {
-        for (let run = 1; run <= flags["runs-per-eval"]; run++) {
-          workQueue.push({
-            eval_,
-            configuration: config.name,
-            targetContent: config.content,
-            runNumber: run
-          });
-        }
-      }
-    }
-    const results = [];
-    const concurrency = flags.parallel;
-    for (let i = 0; i < workQueue.length; i += concurrency) {
-      const batch = workQueue.slice(i, i + concurrency);
-      const batchResults = await Promise.all(
-        batch.map((item) => this.executeAndGrade(item, target, flags))
-      );
-      results.push(...batchResults);
-    }
-    this.log("");
-    this.log("Posting results to API...");
-    for (const result of results) {
-      await apiPost("/api/eval-runs", result);
-    }
-    this.log("");
-    this.log("=== Results Summary ===");
-    for (const config of configurations) {
-      const configResults = results.filter((r) => r.configuration === config.name);
-      const completedRuns = configResults.filter((r) => r.status === "completed");
-      const gradedRuns = completedRuns.filter((r) => Object.keys(r.grading).length > 0);
-      this.log(`
-${config.name}:`);
-      this.log(`  Runs: ${completedRuns.length}/${configResults.length} completed`);
-      if (gradedRuns.length > 0) {
-        const passRates = gradedRuns.map((r) => r.grading.summary?.pass_rate ?? 0);
-        const avgPassRate = passRates.reduce((a, b) => a + b, 0) / passRates.length;
-        this.log(`  Avg pass_rate: ${(avgPassRate * 100).toFixed(1)}%`);
-      }
-      for (const r of configResults) {
-        const evalName = evals.find((e) => e.id === r.evalId)?.name ?? r.evalId;
-        const passRate = r.grading.summary?.pass_rate;
-        const passRateStr = passRate !== void 0 ? ` pass_rate=${(passRate * 100).toFixed(0)}%` : "";
-        const timeStr = r.timing.duration_ms ? ` ${(r.timing.duration_ms / 1e3).toFixed(1)}s` : "";
-        this.log(`  [${r.status}] ${evalName} run#${r.runNumber}${passRateStr}${timeStr}`);
-      }
-    }
-    if (!flags["skip-baseline"] && results.length > 0) {
-      const withTarget = results.filter((r) => r.configuration === "with_target" && Object.keys(r.grading).length > 0);
-      const withoutTarget = results.filter((r) => r.configuration === "without_target" && Object.keys(r.grading).length > 0);
-      if (withTarget.length > 0 && withoutTarget.length > 0) {
-        const avgWith = withTarget.map((r) => r.grading.summary?.pass_rate ?? 0).reduce((a, b) => a + b, 0) / withTarget.length;
-        const avgWithout = withoutTarget.map((r) => r.grading.summary?.pass_rate ?? 0).reduce((a, b) => a + b, 0) / withoutTarget.length;
-        const delta = avgWith - avgWithout;
-        this.log(`
-Delta: ${delta > 0 ? "+" : ""}${(delta * 100).toFixed(1)}% (with_target vs without_target)`);
-      }
-    }
-  }
-  async executeAndGrade(item, target, flags) {
-    const { eval_, configuration, targetContent, runNumber } = item;
-    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 executing...`);
-    const execResult = await executeEval({
-      prompt: eval_.prompt,
-      targetContent,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    if (execResult.error) {
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 FAILED: ${execResult.error}`);
-      return {
-        evalId: eval_.id,
-        evalTargetId: target.id,
-        evalTargetVersion: target.version,
-        configuration,
-        runNumber,
-        iteration: flags.iteration,
-        status: "failed",
-        outputs: { error: execResult.error },
-        timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },
-        grading: {}
-      };
-    }
-    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 completed in ${(execResult.durationMs / 1e3).toFixed(1)}s`);
-    let grading = {};
-    if (!flags["skip-grading"] && eval_.expectations.length > 0) {
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading...`);
-      const gradingResult = await gradeEvalOutput({
-        prompt: eval_.prompt,
-        output: execResult.output,
-        expectations: eval_.expectations,
-        model: flags["grader-model"] ?? flags.model,
-        timeout: flags.timeout
-      });
-      grading = gradingResult;
-      const passRate = gradingResult.summary.pass_rate;
-      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading done: pass_rate=${(passRate * 100).toFixed(0)}%`);
-    }
-    return {
-      evalId: eval_.id,
-      evalTargetId: target.id,
-      evalTargetVersion: target.version,
-      configuration,
-      runNumber,
-      iteration: flags.iteration,
-      status: "completed",
-      outputs: { transcript: execResult.output },
-      timing: {
-        duration_ms: execResult.durationMs,
-        total_tokens: execResult.totalTokens,
-        total_duration_seconds: execResult.durationMs / 1e3
-      },
-      grading
-    };
-  }
-};
-export {
-  EvalRunCommand as default
-};
-//# sourceMappingURL=run.js.map
+import{a as w}from"../../chunk-CADKBP3D.js";import{a as _}from"../../chunk-MGHFDHHM.js";import{a as R}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as h,f as k}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as $}from"../../chunk-ZLLER3HP.js";import{Args as T,Flags as c}from"@oclif/core";var b=class x extends R{static{$(this,"EvalRunCommand")}static description="Run evals against a target, optionally grade results, and post to API";static examples=["<%= config.bin %> evals run <target-id>","<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2","<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading"];static args={targetId:T.string({description:"Eval target ID",required:!0})};static flags={parallel:c.integer({description:"Number of parallel eval executions",default:1}),"runs-per-eval":c.integer({description:"Number of runs per eval for variance analysis",default:1}),iteration:c.integer({description:"Iteration number for tracking",default:1}),"skip-baseline":c.boolean({description:"Skip baseline (without_target) runs",default:!1}),"skip-grading":c.boolean({description:"Skip LLM grading step",default:!1}),model:c.string({description:"Model for eval execution (default: user configured)"}),"grader-model":c.string({description:"Model for grading (default: same as execution model)"}),timeout:c.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:f,flags:i}=await this.parse(x);this.requireAuth();let s=await h(`/api/eval-targets/${f.targetId}`),r=await h(`/api/eval-targets/${f.targetId}/evals`);r.length===0&&this.error(`No evals found for target "${s.name}". Create some with \`rulemetric evals create\`.`),this.log(`Running ${r.length} eval(s) for "${s.name}" v${s.version}`),this.log(`Config: ${i["runs-per-eval"]} run(s) per eval, parallel=${i.parallel}, iteration=${i.iteration}`),i["skip-baseline"]&&this.log("Skipping baseline runs"),i["skip-grading"]&&this.log("Skipping grading"),this.log("");let l=[{name:"with_target",content:s.content}];i["skip-baseline"]||l.push({name:"without_target"});let p=[];for(let e of r)for(let a of l)for(let n=1;n<=i["runs-per-eval"];n++)p.push({eval_:e,configuration:a.name,targetContent:a.content,runNumber:n});let o=[],g=i.parallel;for(let e=0;e<p.length;e+=g){let a=p.slice(e,e+g),n=await Promise.all(a.map(m=>this.executeAndGrade(m,s,i)));o.push(...n)}this.log(""),this.log("Posting results to API...");for(let e of o)await k("/api/eval-runs",e);this.log(""),this.log("=== Results Summary ===");for(let e of l){let a=o.filter(t=>t.configuration===e.name),n=a.filter(t=>t.status==="completed"),m=n.filter(t=>Object.keys(t.grading).length>0);if(this.log(`
+${e.name}:`),this.log(`  Runs: ${n.length}/${a.length} completed`),m.length>0){let t=m.map(d=>d.grading.summary?.pass_rate??0),u=t.reduce((d,v)=>d+v,0)/t.length;this.log(`  Avg pass_rate: ${(u*100).toFixed(1)}%`)}for(let t of a){let u=r.find(I=>I.id===t.evalId)?.name??t.evalId,d=t.grading.summary?.pass_rate,v=d!==void 0?` pass_rate=${(d*100).toFixed(0)}%`:"",y=t.timing.duration_ms?` ${(t.timing.duration_ms/1e3).toFixed(1)}s`:"";this.log(`  [${t.status}] ${u} run#${t.runNumber}${v}${y}`)}}if(!i["skip-baseline"]&&o.length>0){let e=o.filter(n=>n.configuration==="with_target"&&Object.keys(n.grading).length>0),a=o.filter(n=>n.configuration==="without_target"&&Object.keys(n.grading).length>0);if(e.length>0&&a.length>0){let n=e.map(u=>u.grading.summary?.pass_rate??0).reduce((u,d)=>u+d,0)/e.length,m=a.map(u=>u.grading.summary?.pass_rate??0).reduce((u,d)=>u+d,0)/a.length,t=n-m;this.log(`
+Delta: ${t>0?"+":""}${(t*100).toFixed(1)}% (with_target vs without_target)`)}}}async executeAndGrade(f,i,s){let{eval_:r,configuration:l,targetContent:p,runNumber:o}=f;this.log(`[${l}] ${r.name} run#${o} \u2014 executing...`);let g=await _({prompt:r.prompt,targetContent:p,timeout:s.timeout,model:s.model});if(g.error)return this.log(`[${l}] ${r.name} run#${o} \u2014 FAILED: ${g.error}`),{evalId:r.id,evalTargetId:i.id,evalTargetVersion:i.version,configuration:l,runNumber:o,iteration:s.iteration,status:"failed",outputs:{error:g.error},timing:{duration_ms:g.durationMs,total_tokens:g.totalTokens},grading:{}};this.log(`[${l}] ${r.name} run#${o} \u2014 completed in ${(g.durationMs/1e3).toFixed(1)}s`);let e={};if(!s["skip-grading"]&&r.expectations.length>0){this.log(`[${l}] ${r.name} run#${o} \u2014 grading...`);let a=await w({prompt:r.prompt,output:g.output,expectations:r.expectations,model:s["grader-model"]??s.model,timeout:s.timeout});e=a;let n=a.summary.pass_rate;this.log(`[${l}] ${r.name} run#${o} \u2014 grading done: pass_rate=${(n*100).toFixed(0)}%`)}return{evalId:r.id,evalTargetId:i.id,evalTargetVersion:i.version,configuration:l,runNumber:o,iteration:s.iteration,status:"completed",outputs:{transcript:g.output},timing:{duration_ms:g.durationMs,total_tokens:g.totalTokens,total_duration_seconds:g.durationMs/1e3},grading:e}}};export{b as default};

package/dist/commands/gateway/ensure.js

@@ -1,104 +1 @@
-import {
-  repinEditorProxies,
-  unpinEditorProxies
-} from "../../chunk-3S7223UR.js";
-import {
-  writeCursorUserProxyConfig,
-  writeVSCodeUserProxyConfig
-} from "../../chunk-LXKPNATC.js";
-import "../../chunk-OQBBVTDG.js";
-import {
-  GATEWAY_PORT,
-  isGatewayRunning,
-  isRulemetricGatewayListening,
-  spawnGateway
-} from "../../chunk-4LYCDS2N.js";
-import "../../chunk-7LVVKQMZ.js";
-import {
-  MITMPROXY_COMMAND_PATTERN,
-  isPidLiveAndMatching
-} from "../../chunk-5EDMGK3W.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/gateway/ensure.ts
-import { spawn } from "node:child_process";
-import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-var CONFIG_DIR = join(homedir(), ".config", "rulemetric");
-var ADDON_VERSION_FILE = join(CONFIG_DIR, "proxy-addon.version");
-var PROXY_PID_FILE = join(CONFIG_DIR, "proxy.pid");
-var DRIFT_RESTART_MARKER = join(CONFIG_DIR, "proxy-drift-restart.at");
-var DRIFT_RESTART_COOLDOWN_MS = 10 * 6e4;
-var GatewayEnsure = class _GatewayEnsure extends BaseCommand {
-  static description = "Ensure the gateway proxy is running (called by hooks, not for direct use)";
-  static hidden = true;
-  async run() {
-    await this.parse(_GatewayEnsure);
-    this.maybeRestartStaleProxy();
-    const identity = await isRulemetricGatewayListening(GATEWAY_PORT);
-    if (identity.status === "ours-live") {
-      const repinned = repinEditorProxies((editor) => {
-        if (editor === "vscode") writeVSCodeUserProxyConfig(GATEWAY_PORT);
-        else writeCursorUserProxyConfig(GATEWAY_PORT);
-      });
-      if (repinned.length > 0) {
-        console.error(`[RuleMetric] Gateway healthy again \u2014 restored ${repinned.join(", ")} proxy settings.`);
-      }
-      return;
-    }
-    if (identity.status === "foreign") {
-      if (isGatewayRunning()) return;
-      const unpinned = unpinEditorProxies(GATEWAY_PORT);
-      console.error(
-        `[RuleMetric] A non-RuleMetric process holds :${GATEWAY_PORT} \u2014 cannot start gateway. ` + (unpinned.length > 0 ? `Unpinned ${unpinned.join(", ")} proxy settings so editors are not routed through it. ` : "") + "Run `rulemetric diagnostics` if Claude Code HTTPS requests fail."
-      );
-      return;
-    }
-    try {
-      spawnGateway(this.config.version);
-    } catch (err) {
-      console.error(`[RuleMetric] Could not start gateway: ${err.message}`);
-    }
-  }
-  maybeRestartStaleProxy() {
-    try {
-      if (!existsSync(ADDON_VERSION_FILE)) return;
-      const runningVersion = readFileSync(ADDON_VERSION_FILE, "utf-8").trim();
-      if (!runningVersion || runningVersion === this.config.version) return;
-      const pidRaw = readFileSync(PROXY_PID_FILE, "utf-8").trim();
-      const pid = Number(pidRaw);
-      if (!Number.isFinite(pid) || !isPidLiveAndMatching(pid, MITMPROXY_COMMAND_PATTERN)) {
-        return;
-      }
-      try {
-        const age = Date.now() - statSync(DRIFT_RESTART_MARKER).mtimeMs;
-        if (age < DRIFT_RESTART_COOLDOWN_MS) return;
-      } catch {
-      }
-      writeFileSync(DRIFT_RESTART_MARKER, (/* @__PURE__ */ new Date()).toISOString());
-      const child = spawn(process.execPath, [process.argv[1], "proxy", "restart"], {
-        detached: true,
-        stdio: "ignore"
-      });
-      child.unref();
-      console.error(
-        `[RuleMetric] Capture proxy is running CLI ${runningVersion} but ${this.config.version} is installed \u2014 restarting it in the background.`
-      );
-    } catch {
-    }
-  }
-};
-export {
-  GatewayEnsure as default
-};
-//# sourceMappingURL=ensure.js.map
+import{b as y,c as h}from"../../chunk-JAPCCEKF.js";import{B as l,E as g}from"../../chunk-2JRDG4LJ.js";import"../../chunk-I2DZGJRR.js";import{c as e,g as p,i as m,k as f}from"../../chunk-MVM6OGAF.js";import"../../chunk-KKDHD4XA.js";import{b as d,d as u}from"../../chunk-SZ62CP55.js";import{a as c}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a}from"../../chunk-ZLLER3HP.js";import{spawn as S}from"node:child_process";import{existsSync as v,readFileSync as R,statSync as C,writeFileSync as M}from"node:fs";import{homedir as T}from"node:os";import{join as o}from"node:path";var s=o(T(),".config","rulemetric"),x=o(s,"proxy-addon.version"),_=o(s,"proxy.pid"),w=o(s,"proxy-drift-restart.at"),E=10*6e4,n=class P extends c{static{a(this,"GatewayEnsure")}static description="Ensure the gateway proxy is running (called by hooks, not for direct use)";static hidden=!0;async run(){await this.parse(P),this.maybeRestartStaleProxy();let t=await p(e);if(t.status==="ours-live"){let r=h(i=>{i==="vscode"?l(e):g(e)});r.length>0&&console.error(`[RuleMetric] Gateway healthy again \u2014 restored ${r.join(", ")} proxy settings.`);return}if(t.status==="foreign"){if(m())return;let r=y(e);console.error(`[RuleMetric] A non-RuleMetric process holds :${e} \u2014 cannot start gateway. `+(r.length>0?`Unpinned ${r.join(", ")} proxy settings so editors are not routed through it. `:"")+"Run `rulemetric diagnostics` if Claude Code HTTPS requests fail.");return}try{f(this.config.version)}catch(r){console.error(`[RuleMetric] Could not start gateway: ${r.message}`)}}maybeRestartStaleProxy(){try{if(!v(x))return;let t=R(x,"utf-8").trim();if(!t||t===this.config.version)return;let r=R(_,"utf-8").trim(),i=Number(r);if(!Number.isFinite(i)||!u(i,d))return;try{if(Date.now()-C(w).mtimeMs<E)return}catch{}M(w,new Date().toISOString()),S(process.execPath,[process.argv[1],"proxy","restart"],{detached:!0,stdio:"ignore"}).unref(),console.error(`[RuleMetric] Capture proxy is running CLI ${t} but ${this.config.version} is installed \u2014 restarting it in the background.`)}catch{}}};export{n as default};

package/dist/commands/hooks/emit-instructions.js

@@ -1,25 +1 @@
-import {
-  buildInstructionSnapshot
-} from "../../chunk-SO3T35U7.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/hooks/emit-instructions.ts
-import { Command, Flags } from "@oclif/core";
-var HooksEmitInstructions = class _HooksEmitInstructions extends Command {
-  static description = "Print a synthetic instruction snapshot (on-disk CLAUDE.md) as JSON for zero-proxy instruction linking";
-  static hidden = true;
-  static flags = {
-    "project-dir": Flags.string({ description: "Project directory to scan (defaults to cwd)" })
-  };
-  async run() {
-    const { flags } = await this.parse(_HooksEmitInstructions);
-    const snapshot = buildInstructionSnapshot(flags["project-dir"] ?? process.cwd());
-    if (snapshot) {
-      process.stdout.write(JSON.stringify(snapshot));
-    }
-  }
-};
-export {
-  HooksEmitInstructions as default
-};
-//# sourceMappingURL=emit-instructions.js.map
+import{a as s}from"../../chunk-CNQ5DTNC.js";import{a as i}from"../../chunk-ZLLER3HP.js";import{Command as n,Flags as c}from"@oclif/core";var t=class o extends n{static{i(this,"HooksEmitInstructions")}static description="Print a synthetic instruction snapshot (on-disk CLAUDE.md) as JSON for zero-proxy instruction linking";static hidden=!0;static flags={"project-dir":c.string({description:"Project directory to scan (defaults to cwd)"})};async run(){let{flags:e}=await this.parse(o),r=s(e["project-dir"]??process.cwd());r&&process.stdout.write(JSON.stringify(r))}};export{t as default};