@rulemetric/cli 0.7.3 → 0.7.5

package/dist/commands/evals/agent.js

@@ -1,310 +1,3 @@
-import {
-  runAgentLoop
-} from "../../chunk-RAXRZYOB.js";
-import "../../chunk-GIUBARWS.js";
-import "../../chunk-6NBS5XFS.js";
-import "../../chunk-W53GKIZQ.js";
-import "../../chunk-OCKHW72U.js";
-import "../../chunk-RQ2TMLKG.js";
-import "../../chunk-DMX5YOTM.js";
-import "../../chunk-OO7JDFS4.js";
-import "../../chunk-E3BIT53W.js";
-import "../../chunk-QSN77T7C.js";
-import {
-  closeDb
-} from "../../chunk-EI5BHWXA.js";
-import "../../chunk-XK42ZBDE.js";
-import "../../chunk-RMTT4KDY.js";
-import "../../chunk-VA5FFOS7.js";
-import "../../chunk-XZXS2W24.js";
-import "../../chunk-DIQUV5FR.js";
-import "../../chunk-JULOLTZS.js";
-import "../../chunk-Y4BJXXYV.js";
-import "../../chunk-DGHWRQXL.js";
-import "../../chunk-J7N3DLH6.js";
-import "../../chunk-52WIYRZH.js";
-import "../../chunk-KJKBDQ4D.js";
-import "../../chunk-OQSQC7VB.js";
-import "../../chunk-EKP32DLN.js";
-import "../../chunk-F5N6CLJJ.js";
-import "../../chunk-KRBQLMOP.js";
-import "../../chunk-42GFSAJP.js";
-import "../../chunk-3TIMQ3O6.js";
-import "../../chunk-YNIH3KMU.js";
-import "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZUJBM6HE.js";
-import {
-  getDefaultConfigDir,
-  loadToken
-} from "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// ../../packages/telemetry/dist/init.js
-import { trace, context } from "@opentelemetry/api";
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { Resource } from "@opentelemetry/resources";
-import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
-
-// ../../packages/telemetry/dist/otlp-headers.js
-function parseOtlpHeaders(raw) {
-  if (!raw)
-    return {};
-  const out = {};
-  for (const entry of raw.split(",")) {
-    const eq = entry.indexOf("=");
-    if (eq === -1)
-      continue;
-    const key = entry.slice(0, eq).trim();
-    const value = entry.slice(eq + 1).trim();
-    if (!key)
-      continue;
-    out[key] = value;
-  }
-  return out;
-}
-
-// ../../packages/telemetry/dist/init.js
-var sdk = null;
-function isTelemetryEnabled() {
-  return process.env.TELEMETRY_ENABLED !== "false";
-}
-function initTelemetry(config) {
-  if (!isTelemetryEnabled()) {
-    return "off";
-  }
-  if (sdk !== null) {
-    return "off";
-  }
-  const resource = new Resource({
-    [ATTR_SERVICE_NAME]: config.serviceName,
-    [ATTR_SERVICE_VERSION]: config.serviceVersion ?? "0.1.0"
-  });
-  const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-  let mode;
-  let spanProcessor;
-  if (endpoint) {
-    const headers = parseOtlpHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS);
-    const exporter = new OTLPTraceExporter({
-      url: `${endpoint}/v1/traces`,
-      headers
-    });
-    spanProcessor = new BatchSpanProcessor(exporter);
-    mode = "otlp";
-  } else if (process.env.OTEL_SPAN_EXPORT === "none") {
-    spanProcessor = void 0;
-    mode = "silent";
-  } else {
-    const exporter = new ConsoleSpanExporter();
-    spanProcessor = new SimpleSpanProcessor(exporter);
-    mode = "console";
-  }
-  sdk = new NodeSDK({
-    resource,
-    spanProcessors: spanProcessor ? [spanProcessor] : []
-  });
-  sdk.start();
-  return mode;
-}
-async function shutdownTelemetry() {
-  if (sdk) {
-    await sdk.shutdown();
-    sdk = null;
-  }
-}
-
-// ../../packages/telemetry/dist/hono-middleware.js
-import { SpanKind, SpanStatusCode } from "@opentelemetry/api";
-
-// ../../packages/telemetry/dist/span-attributes.js
-import { trace as trace2, context as otelContext } from "@opentelemetry/api";
-
-// ../../packages/telemetry/dist/sentry.js
-import * as Sentry from "@sentry/node";
-var enabled = false;
-function initSentry(config) {
-  if (enabled)
-    return false;
-  const dsn = process.env.SENTRY_DSN;
-  if (!dsn)
-    return false;
-  Sentry.init({
-    dsn,
-    serverName: config.serviceName,
-    environment: process.env.NODE_ENV ?? "development",
-    // We rely on OpenTelemetry for tracing — Sentry stays errors-only here.
-    // Override at the env-var level if you ever want Sentry performance too:
-    // `SENTRY_TRACES_SAMPLE_RATE=0.05` (Sentry SDK reads it natively).
-    tracesSampleRate: 0
-  });
-  enabled = true;
-  return true;
-}
-async function flushSentry(timeoutMs) {
-  if (!enabled)
-    return;
-  await Sentry.flush(timeoutMs);
-}
-
-// src/commands/evals/agent.ts
-import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { execFileSync } from "node:child_process";
-var AGENT_LOCK_PATH = join(getDefaultConfigDir(), "evals-agent.lock");
-function pidAlive(pid) {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (e) {
-    return e.code === "EPERM";
-  }
-}
-function isLiveAgent(pid) {
-  if (!pidAlive(pid)) return false;
-  try {
-    const cmd = execFileSync("ps", ["-p", String(pid), "-o", "command="], {
-      encoding: "utf-8",
-      stdio: ["ignore", "pipe", "ignore"]
-    });
-    return /run\.js\s+evals\s+agent|evals\s+agent/.test(cmd);
-  } catch {
-    return true;
-  }
-}
-function acquireAgentLock() {
-  try {
-    mkdirSync(dirname(AGENT_LOCK_PATH), { recursive: true });
-  } catch {
-  }
-  for (let attempt = 0; attempt < 2; attempt++) {
-    try {
-      writeFileSync(AGENT_LOCK_PATH, String(process.pid), { flag: "wx" });
-      return { ok: true };
-    } catch (e) {
-      if (e.code !== "EEXIST") throw e;
-      let holder = 0;
-      try {
-        holder = Number(readFileSync(AGENT_LOCK_PATH, "utf-8").trim());
-      } catch {
-      }
-      if (holder && holder !== process.pid && isLiveAgent(holder)) {
-        return { ok: false, holder };
-      }
-      try {
-        unlinkSync(AGENT_LOCK_PATH);
-      } catch {
-      }
-    }
-  }
-  try {
-    writeFileSync(AGENT_LOCK_PATH, String(process.pid));
-  } catch {
-  }
-  return { ok: true };
-}
-function releaseAgentLock() {
-  try {
-    if (existsSync(AGENT_LOCK_PATH) && Number(readFileSync(AGENT_LOCK_PATH, "utf-8").trim()) === process.pid) {
-      unlinkSync(AGENT_LOCK_PATH);
-    }
-  } catch {
-  }
-}
-var EvalAgentCommand = class _EvalAgentCommand extends BaseCommand {
-  static description = "The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";
-  static examples = [
-    "<%= config.bin %> evals agent"
-  ];
-  async run() {
-    await this.parse(_EvalAgentCommand);
-    this.requireAuth();
-    const lock = acquireAgentLock();
-    if (!lock.ok) {
-      this.log(
-        `Another rulemetric evals agent is already running (pid ${lock.holder}). Refusing to start a second instance.`
-      );
-      return;
-    }
-    process.env.PG_APPLICATION_NAME ||= "rulemetric-agent";
-    const sentryEnabled = initSentry({ serviceName: "rulemetric-agent" });
-    if (sentryEnabled) this.log("[sentry] error capture enabled");
-    const otelMode = initTelemetry({ serviceName: "rulemetric-agent" });
-    if (otelMode === "otlp") {
-      this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`);
-    } else if (otelMode === "console") {
-      this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)");
-    } else {
-      this.log("[otel] disabled (TELEMETRY_ENABLED=false)");
-    }
-    const token = loadToken();
-    if (!token && !process.env.RULEMETRIC_API_KEY) {
-      this.error("No auth token found. Run `rulemetric auth login` first.");
-    }
-    this.log("Starting thin agent (Phase 4b parallel-run)...");
-    this.log("Long-polling /api/work/next. Press Ctrl+C to stop.\n");
-    const controller = new AbortController();
-    const shutdown = async (signal) => {
-      this.log(`
-[${signal}] shutting down agent...`);
-      controller.abort();
-      await Promise.allSettled([
-        flushSentry(2e3),
-        shutdownTelemetry(),
-        // Drain the postgres-js pool so this agent's Supavisor client sessions
-        // are released (Terminate) instead of orphaned as ghosts on every
-        // launchd bounce. Pool-exhaustion RCA 2026-06-21.
-        closeDb()
-      ]);
-      releaseAgentLock();
-      process.exit(0);
-    };
-    process.on("SIGINT", () => {
-      void shutdown("SIGINT");
-    });
-    process.on("SIGTERM", () => {
-      void shutdown("SIGTERM");
-    });
-    let restartForUpgrade = false;
-    try {
-      await runAgentLoop({
-        signal: controller.signal,
-        log: (msg) => this.log(msg),
-        version: this.config.version,
-        readInstalledVersion: () => {
-          try {
-            const pkg = JSON.parse(
-              readFileSync(join(this.config.root, "package.json"), "utf-8")
-            );
-            return pkg.version ?? null;
-          } catch {
-            return null;
-          }
-        },
-        onVersionDrift: () => {
-          restartForUpgrade = true;
-        }
-      });
-      if (restartForUpgrade) {
-        this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code.");
-        await Promise.allSettled([flushSentry(2e3), shutdownTelemetry(), closeDb()]);
-        releaseAgentLock();
-        process.exit(1);
-      }
-    } finally {
-      releaseAgentLock();
-      await closeDb();
-    }
-  }
-};
-export {
-  EvalAgentCommand as default
-};
-//# sourceMappingURL=agent.js.map
+import{e as N}from"../../chunk-43YBXX7R.js";import"../../chunk-TIOKY2IA.js";import"../../chunk-TQMG647B.js";import"../../chunk-W2MOTOQE.js";import"../../chunk-VAXRCSSH.js";import"../../chunk-6QKI756T.js";import"../../chunk-RSKQRVPJ.js";import"../../chunk-AYUJY37O.js";import"../../chunk-QFVGUZDO.js";import"../../chunk-KAF3CYRW.js";import"../../chunk-UF6US7VU.js";import"../../chunk-DPFNJZUO.js";import"../../chunk-RTWRHIKY.js";import"../../chunk-2WLVDSAP.js";import"../../chunk-ZL4MZMUK.js";import"../../chunk-N7TGRFVZ.js";import{i as f}from"../../chunk-H4SV7FN4.js";import"../../chunk-NEACEHQD.js";import"../../chunk-32EEBARV.js";import"../../chunk-5RBLIVBF.js";import"../../chunk-2TC6WCUT.js";import"../../chunk-6V4WM73L.js";import"../../chunk-HS7X43XE.js";import"../../chunk-TDD7IOV2.js";import"../../chunk-MODXEGCU.js";import"../../chunk-7DFNBRLY.js";import"../../chunk-N7ODX6PO.js";import"../../chunk-67QCYUJU.js";import"../../chunk-DSO2U4Y6.js";import"../../chunk-CADKBP3D.js";import"../../chunk-MGHFDHHM.js";import{a as _}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import"../../chunk-H2ORWIEY.js";import{e as R,l as w}from"../../chunk-O6C2UGBY.js";import{a as e}from"../../chunk-ZLLER3HP.js";import{trace as z,context as Q}from"@opentelemetry/api";import{NodeSDK as A}from"@opentelemetry/sdk-node";import{OTLPTraceExporter as I}from"@opentelemetry/exporter-trace-otlp-http";import{BatchSpanProcessor as L,ConsoleSpanExporter as C,SimpleSpanProcessor as D}from"@opentelemetry/sdk-trace-node";import{Resource as M}from"@opentelemetry/resources";import{ATTR_SERVICE_NAME as V,ATTR_SERVICE_VERSION as q}from"@opentelemetry/semantic-conventions";function b(t){if(!t)return{};let r={};for(let n of t.split(",")){let o=n.indexOf("=");if(o===-1)continue;let s=n.slice(0,o).trim(),i=n.slice(o+1).trim();s&&(r[s]=i)}return r}e(b,"parseOtlpHeaders");var c=null;function g(){return process.env.TELEMETRY_ENABLED!=="false"}e(g,"isTelemetryEnabled");function E(t){if(!g()||c!==null)return"off";let r=new M({[V]:t.serviceName,[q]:t.serviceVersion??"0.1.0"}),n=process.env.OTEL_EXPORTER_OTLP_ENDPOINT,o,s;if(n){let i=b(process.env.OTEL_EXPORTER_OTLP_HEADERS),u=new I({url:`${n}/v1/traces`,headers:i});s=new L(u),o="otlp"}else if(process.env.OTEL_SPAN_EXPORT==="none")s=void 0,o="silent";else{let i=new C;s=new D(i),o="console"}return c=new A({resource:r,spanProcessors:s?[s]:[]}),c.start(),o}e(E,"initTelemetry");async function d(){c&&(await c.shutdown(),c=null)}e(d,"shutdownTelemetry");import{SpanKind as ae,SpanStatusCode as ce}from"@opentelemetry/api";import{trace as de,context as me}from"@opentelemetry/api";import*as l from"@sentry/node";var h=!1;function y(t){if(h)return!1;let r=process.env.SENTRY_DSN;return r?(l.init({dsn:r,serverName:t.serviceName,environment:process.env.NODE_ENV??"development",tracesSampleRate:0}),h=!0,!0):!1}e(y,"initSentry");async function m(t){h&&await l.flush(t)}e(m,"flushSentry");import{existsSync as G,mkdirSync as U,readFileSync as v,unlinkSync as k,writeFileSync as O}from"node:fs";import{dirname as X,join as P}from"node:path";import{execFileSync as $}from"node:child_process";var a=P(w(),"evals-agent.lock");function j(t){try{return process.kill(t,0),!0}catch(r){return r.code==="EPERM"}}e(j,"pidAlive");function B(t){if(!j(t))return!1;try{let r=$("ps",["-p",String(t),"-o","command="],{encoding:"utf-8",stdio:["ignore","pipe","ignore"]});return/run\.js\s+evals\s+agent|evals\s+agent/.test(r)}catch{return!0}}e(B,"isLiveAgent");function H(){try{U(X(a),{recursive:!0})}catch{}for(let t=0;t<2;t++)try{return O(a,String(process.pid),{flag:"wx"}),{ok:!0}}catch(r){if(r.code!=="EEXIST")throw r;let n=0;try{n=Number(v(a,"utf-8").trim())}catch{}if(n&&n!==process.pid&&B(n))return{ok:!1,holder:n};try{k(a)}catch{}}try{O(a,String(process.pid))}catch{}return{ok:!0}}e(H,"acquireAgentLock");function S(){try{G(a)&&Number(v(a,"utf-8").trim())===process.pid&&k(a)}catch{}}e(S,"releaseAgentLock");var T=class t extends _{static{e(this,"EvalAgentCommand")}static description="The worker daemon: long-polls /api/work/next, claims jobs, and runs them locally. Installed as a launchd service by `rulemetric service install`.";static examples=["<%= config.bin %> evals agent"];async run(){await this.parse(t),this.requireAuth();let r=H();if(!r.ok){this.log(`Another rulemetric evals agent is already running (pid ${r.holder}). Refusing to start a second instance.`);return}process.env.PG_APPLICATION_NAME||="rulemetric-agent",y({serviceName:"rulemetric-agent"})&&this.log("[sentry] error capture enabled");let o=E({serviceName:"rulemetric-agent"});o==="otlp"?this.log(`[otel] traces exporting to ${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}`):o==="console"?this.log("[otel] cron spans printing to stdout (set OTEL_EXPORTER_OTLP_ENDPOINT to ship to a backend)"):this.log("[otel] disabled (TELEMETRY_ENABLED=false)"),!R()&&!process.env.RULEMETRIC_API_KEY&&this.error("No auth token found. Run `rulemetric auth login` first."),this.log("Starting thin agent (Phase 4b parallel-run)..."),this.log(`Long-polling /api/work/next. Press Ctrl+C to stop.
+`);let i=new AbortController,u=e(async p=>{this.log(`
+[${p}] shutting down agent...`),i.abort(),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(0)},"shutdown");process.on("SIGINT",()=>{u("SIGINT")}),process.on("SIGTERM",()=>{u("SIGTERM")});let x=!1;try{await N({signal:i.signal,log:e(p=>this.log(p),"log"),version:this.config.version,readInstalledVersion:e(()=>{try{return JSON.parse(v(P(this.config.root,"package.json"),"utf-8")).version??null}catch{return null}},"readInstalledVersion"),onVersionDrift:e(()=>{x=!0},"onVersionDrift")}),x&&(this.log("[upgrade] Installed CLI version changed \u2014 restarting worker onto the new code."),await Promise.allSettled([m(2e3),d(),f()]),S(),process.exit(1))}finally{S(),await f()}}};export{T as default};

package/dist/commands/evals/benchmark.js

@@ -1,73 +1,6 @@
-import {
-  aggregateBenchmark,
-  postBenchmark
-} from "../../chunk-PED7L5GX.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/benchmark.ts
-import { Args, Flags } from "@oclif/core";
-var EvalBenchmarkCommand = class _EvalBenchmarkCommand extends BaseCommand {
-  static description = "Aggregate eval runs into a benchmark for a target";
-  static examples = [
-    "<%= config.bin %> evals benchmark <target-id>",
-    "<%= config.bin %> evals benchmark <target-id> --iteration 2"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    iteration: Flags.integer({ description: "Filter runs by iteration", default: 1 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalBenchmarkCommand);
-    this.requireAuth();
-    const target = await apiGet(`/api/eval-targets/${args.targetId}`);
-    this.log(`Aggregating benchmark for "${target.name}" v${target.version}, iteration ${flags.iteration}`);
-    const result = await aggregateBenchmark(args.targetId, flags.iteration);
-    if (result.totalRuns === 0) {
-      this.error("No completed runs found. Run evals first with `rulemetric evals run`.");
-    }
-    this.log(`
-Aggregated ${result.totalRuns} runs
-`);
-    for (const [config, stats] of Object.entries(result.runSummary)) {
-      if (config === "delta") {
-        this.log(`
-Delta (with_target - without_target):`);
-        for (const [metric, value] of Object.entries(stats)) {
-          this.log(`  ${metric}: ${value}`);
-        }
-        continue;
-      }
-      this.log(`${config}:`);
-      for (const [metric, stat] of Object.entries(stats)) {
-        const s = stat;
-        this.log(`  ${metric}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`);
-      }
-    }
-    if (result.notes.length > 0) {
-      this.log("\nNotes:");
-      for (const note of result.notes) {
-        this.log(`  - ${note}`);
-      }
-    }
-    const benchmark = await postBenchmark(args.targetId, flags.iteration, result);
-    this.log(`
-Benchmark saved: ${benchmark.id}`);
-  }
-};
-export {
-  EvalBenchmarkCommand as default
-};
-//# sourceMappingURL=benchmark.js.map
+import{a as u,b as f}from"../../chunk-G63KMH6U.js";import{a as d}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as l}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as m}from"../../chunk-ZLLER3HP.js";import{Args as b,Flags as $}from"@oclif/core";var o=class h extends d{static{m(this,"EvalBenchmarkCommand")}static description="Aggregate eval runs into a benchmark for a target";static examples=["<%= config.bin %> evals benchmark <target-id>","<%= config.bin %> evals benchmark <target-id> --iteration 2"];static args={targetId:b.string({description:"Eval target ID",required:!0})};static flags={iteration:$.integer({description:"Filter runs by iteration",default:1})};async run(){let{args:i,flags:a}=await this.parse(h);this.requireAuth();let g=await l(`/api/eval-targets/${i.targetId}`);this.log(`Aggregating benchmark for "${g.name}" v${g.version}, iteration ${a.iteration}`);let t=await u(i.targetId,a.iteration);t.totalRuns===0&&this.error("No completed runs found. Run evals first with `rulemetric evals run`."),this.log(`
+Aggregated ${t.totalRuns} runs
+`);for(let[e,c]of Object.entries(t.runSummary)){if(e==="delta"){this.log(`
+Delta (with_target - without_target):`);for(let[n,s]of Object.entries(c))this.log(`  ${n}: ${s}`);continue}this.log(`${e}:`);for(let[n,s]of Object.entries(c)){let r=s;this.log(`  ${n}: mean=${r.mean} stddev=${r.stddev} [${r.min}, ${r.max}]`)}}if(t.notes.length>0){this.log(`
+Notes:`);for(let e of t.notes)this.log(`  - ${e}`)}let v=await f(i.targetId,a.iteration,t);this.log(`
+Benchmark saved: ${v.id}`)}};export{o as default};

package/dist/commands/evals/compare.js

@@ -1,90 +1,19 @@
-import {
-  executeEval
-} from "../../chunk-QN34TGD3.js";
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiGet
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/compare.ts
-import { Args, Flags } from "@oclif/core";
-var EvalCompareCommand = class _EvalCompareCommand extends BaseCommand {
-  static description = "Blind A/B comparison between two eval target versions";
-  static examples = [
-    "<%= config.bin %> evals compare <target-a> <target-b>"
-  ];
-  static args = {
-    targetA: Args.string({ description: "First eval target ID", required: true }),
-    targetB: Args.string({ description: "Second eval target ID", required: true })
-  };
-  static flags = {
-    model: Flags.string({ description: "Model for execution and comparison" }),
-    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalCompareCommand);
-    this.requireAuth();
-    const [targetA, targetB] = await Promise.all([
-      apiGet(`/api/eval-targets/${args.targetA}`),
-      apiGet(`/api/eval-targets/${args.targetB}`)
-    ]);
-    const evals = await apiGet(`/api/eval-targets/${args.targetA}/evals`);
-    if (evals.length === 0) {
-      this.error("No evals found for target A. Create some first.");
-    }
-    this.log(`Comparing "${targetA.name}" v${targetA.version} vs "${targetB.name}" v${targetB.version}`);
-    this.log(`Running ${evals.length} eval(s)...
-`);
-    const isSwapped = Math.random() > 0.5;
-    const contentA = isSwapped ? targetB.content : targetA.content;
-    const contentB = isSwapped ? targetA.content : targetB.content;
-    const labelA = isSwapped ? targetB.name : targetA.name;
-    const labelB = isSwapped ? targetA.name : targetB.name;
-    const comparisons = [];
-    for (const eval_ of evals) {
-      this.log(`[${eval_.name}] Running A...`);
-      const resultA = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: contentA,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      this.log(`[${eval_.name}] Running B...`);
-      const resultB = await executeEval({
-        prompt: eval_.prompt,
-        targetContent: contentB,
-        timeout: flags.timeout,
-        model: flags.model
-      });
-      comparisons.push({
-        evalName: eval_.name,
-        outputA: resultA.output,
-        outputB: resultB.output
-      });
-    }
-    this.log("\nRunning blind comparison...");
-    const comparisonPrompt = comparisons.map((c) => `
-## Eval: ${c.evalName}
+import{a as i}from"../../chunk-MGHFDHHM.js";import{a as u}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{e as a}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as p}from"../../chunk-ZLLER3HP.js";import{Args as c,Flags as v}from"@oclif/core";var m=class d extends u{static{p(this,"EvalCompareCommand")}static description="Blind A/B comparison between two eval target versions";static examples=["<%= config.bin %> evals compare <target-a> <target-b>"];static args={targetA:c.string({description:"First eval target ID",required:!0}),targetB:c.string({description:"Second eval target ID",required:!0})};static flags={model:v.string({description:"Model for execution and comparison"}),timeout:v.integer({description:"Timeout per eval in ms",default:12e4})};async run(){let{args:s,flags:e}=await this.parse(d);this.requireAuth();let[o,n]=await Promise.all([a(`/api/eval-targets/${s.targetA}`),a(`/api/eval-targets/${s.targetB}`)]),l=await a(`/api/eval-targets/${s.targetA}/evals`);l.length===0&&this.error("No evals found for target A. Create some first."),this.log(`Comparing "${o.name}" v${o.version} vs "${n.name}" v${n.version}`),this.log(`Running ${l.length} eval(s)...
+`);let r=Math.random()>.5,h=r?n.content:o.content,f=r?o.content:n.content,A=r?n.name:o.name,w=r?o.name:n.name,g=[];for(let t of l){this.log(`[${t.name}] Running A...`);let b=await i({prompt:t.prompt,targetContent:h,timeout:e.timeout,model:e.model});this.log(`[${t.name}] Running B...`);let E=await i({prompt:t.prompt,targetContent:f,timeout:e.timeout,model:e.model});g.push({evalName:t.name,outputA:b.output,outputB:E.output})}this.log(`
+Running blind comparison...`);let B=g.map(t=>`
+## Eval: ${t.evalName}
 
 ### Output A
-${c.outputA}
+${t.outputA}
 
 ### Output B
-${c.outputB}
-`).join("\n---\n");
-    const comparatorResult = await executeEval({
-      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
+${t.outputB}
+`).join(`
+---
+`),$=await i({prompt:`Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
 Then provide an overall winner with reasoning.
 
-${comparisonPrompt}
+${B}
 
 Respond with JSON:
 {
@@ -97,18 +26,6 @@ Respond with JSON:
     "strengths_winner": ["..."],
     "weaknesses_loser": ["..."]
   }
-}`,
-      timeout: flags.timeout,
-      model: flags.model
-    });
-    this.log("\n=== Comparison Results ===\n");
-    this.log(`A = "${labelA}" | B = "${labelB}"`);
-    if (isSwapped) this.log("(Labels were swapped for blind comparison)");
-    this.log("");
-    this.log(comparatorResult.output);
-  }
-};
-export {
-  EvalCompareCommand as default
-};
-//# sourceMappingURL=compare.js.map
+}`,timeout:e.timeout,model:e.model});this.log(`
+=== Comparison Results ===
+`),this.log(`A = "${A}" | B = "${w}"`),r&&this.log("(Labels were swapped for blind comparison)"),this.log(""),this.log($.output)}};export{m as default};

package/dist/commands/evals/create.js

@@ -1,67 +1 @@
-import {
-  BaseCommand
-} from "../../chunk-2HK7RGZB.js";
-import "../../chunk-R56PH7YE.js";
-import "../../chunk-GQNJO2P2.js";
-import "../../chunk-37LN5O7M.js";
-import "../../chunk-EKJ2CABV.js";
-import {
-  apiPost
-} from "../../chunk-ZUJBM6HE.js";
-import "../../chunk-W2YERO7E.js";
-import "../../chunk-NSBPE2FW.js";
-
-// src/commands/evals/create.ts
-import { Args, Flags } from "@oclif/core";
-import { readFileSync } from "node:fs";
-var EvalCreate = class _EvalCreate extends BaseCommand {
-  static description = "Create test cases (evals) for an eval target";
-  static examples = [
-    '<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',
-    "<%= config.bin %> evals create <target-id> --from-file evals.json"
-  ];
-  static args = {
-    targetId: Args.string({ description: "Eval target ID", required: true })
-  };
-  static flags = {
-    name: Flags.string({ char: "n", description: "Eval name" }),
-    prompt: Flags.string({ char: "p", description: "Test prompt" }),
-    "expected-output": Flags.string({ description: "Expected output description" }),
-    expectations: Flags.string({ char: "e", description: "Assertion (repeatable)", multiple: true }),
-    "from-file": Flags.string({ description: "Bulk create from JSON file" })
-  };
-  async run() {
-    const { args, flags } = await this.parse(_EvalCreate);
-    this.requireAuth();
-    if (flags["from-file"]) {
-      const data = JSON.parse(readFileSync(flags["from-file"], "utf-8"));
-      const evalsArray = Array.isArray(data) ? data : data.evals;
-      for (const evalItem of evalsArray) {
-        const result2 = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-          name: evalItem.name,
-          prompt: evalItem.prompt,
-          expectedOutput: evalItem.expectedOutput ?? "",
-          expectations: evalItem.expectations ?? [],
-          files: evalItem.files ?? []
-        });
-        this.log(`Created eval: ${result2.name} (${result2.id})`);
-      }
-      this.log(`Created ${evalsArray.length} evals.`);
-      return;
-    }
-    if (!flags.name || !flags.prompt) {
-      this.error("Provide --name and --prompt, or use --from-file for bulk creation");
-    }
-    const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-      name: flags.name,
-      prompt: flags.prompt,
-      expectedOutput: flags["expected-output"] ?? "",
-      expectations: flags.expectations ?? []
-    });
-    this.log(`Created eval: ${result.name} (${result.id})`);
-  }
-};
-export {
-  EvalCreate as default
-};
-//# sourceMappingURL=create.js.map
+import{a as c}from"../../chunk-PPKPCXJD.js";import"../../chunk-IZTIYR5Q.js";import"../../chunk-V44SRRYR.js";import"../../chunk-U6XEU573.js";import"../../chunk-VLOX6KOM.js";import"../../chunk-3AV3GTMP.js";import{f as i}from"../../chunk-H2ORWIEY.js";import"../../chunk-O6C2UGBY.js";import{a as m}from"../../chunk-ZLLER3HP.js";import{Args as f,Flags as r}from"@oclif/core";import{readFileSync as g}from"node:fs";var s=class d extends c{static{m(this,"EvalCreate")}static description="Create test cases (evals) for an eval target";static examples=['<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',"<%= config.bin %> evals create <target-id> --from-file evals.json"];static args={targetId:f.string({description:"Eval target ID",required:!0})};static flags={name:r.string({char:"n",description:"Eval name"}),prompt:r.string({char:"p",description:"Test prompt"}),expectations:r.string({char:"e",description:"Assertion (repeatable)",multiple:!0}),"from-file":r.string({description:"Bulk create from JSON file"})};async run(){let{args:n,flags:e}=await this.parse(d);if(this.requireAuth(),e["from-file"]){let a=JSON.parse(g(e["from-file"],"utf-8")),p=Array.isArray(a)?a:a.evals;for(let t of p){let l=await i(`/api/eval-targets/${n.targetId}/evals`,{name:t.name,prompt:t.prompt,expectations:t.expectations??[],files:t.files??[]});this.log(`Created eval: ${l.name} (${l.id})`)}this.log(`Created ${p.length} evals.`);return}(!e.name||!e.prompt)&&this.error("Provide --name and --prompt, or use --from-file for bulk creation");let o=await i(`/api/eval-targets/${n.targetId}/evals`,{name:e.name,prompt:e.prompt,expectations:e.expectations??[]});this.log(`Created eval: ${o.name} (${o.id})`)}};export{s as default};