@rulemetric/cli 0.2.0 → 0.2.2

package/dist/commands/evals/compare.js

@@ -1,65 +1,76 @@
-import { Args, Flags } from '@oclif/core';
-import { BaseCommand } from '../../base-command.js';
-import { apiGet } from '../../lib/api-client.js';
-import { executeEval } from '../../lib/eval-executor.js';
-export default class EvalCompareCommand extends BaseCommand {
-    static description = 'Blind A/B comparison between two eval target versions';
-    static examples = [
-        '<%= config.bin %> evals compare <target-a> <target-b>',
-    ];
-    static args = {
-        targetA: Args.string({ description: 'First eval target ID', required: true }),
-        targetB: Args.string({ description: 'Second eval target ID', required: true }),
-    };
-    static flags = {
-        model: Flags.string({ description: 'Model for execution and comparison' }),
-        timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),
-    };
-    async run() {
-        const { args, flags } = await this.parse(EvalCompareCommand);
-        this.requireAuth();
-        const [targetA, targetB] = await Promise.all([
-            apiGet(`/api/eval-targets/${args.targetA}`),
-            apiGet(`/api/eval-targets/${args.targetB}`),
-        ]);
-        // Use evals from target A (they should be testing the same thing)
-        const evals = await apiGet(`/api/eval-targets/${args.targetA}/evals`);
-        if (evals.length === 0) {
-            this.error('No evals found for target A. Create some first.');
-        }
-        this.log(`Comparing "${targetA.name}" v${targetA.version} vs "${targetB.name}" v${targetB.version}`);
-        this.log(`Running ${evals.length} eval(s)...\n`);
-        // Randomize A/B assignment for blindness
-        const isSwapped = Math.random() > 0.5;
-        const contentA = isSwapped ? targetB.content : targetA.content;
-        const contentB = isSwapped ? targetA.content : targetB.content;
-        const labelA = isSwapped ? targetB.name : targetA.name;
-        const labelB = isSwapped ? targetA.name : targetB.name;
-        const comparisons = [];
-        for (const eval_ of evals) {
-            this.log(`[${eval_.name}] Running A...`);
-            const resultA = await executeEval({
-                prompt: eval_.prompt,
-                targetContent: contentA,
-                timeout: flags.timeout,
-                model: flags.model,
-            });
-            this.log(`[${eval_.name}] Running B...`);
-            const resultB = await executeEval({
-                prompt: eval_.prompt,
-                targetContent: contentB,
-                timeout: flags.timeout,
-                model: flags.model,
-            });
-            comparisons.push({
-                evalName: eval_.name,
-                outputA: resultA.output,
-                outputB: resultB.output,
-            });
-        }
-        // Run blind comparison via LLM
-        this.log('\nRunning blind comparison...');
-        const comparisonPrompt = comparisons.map((c) => `
+import {
+  executeEval
+} from "../../chunk-QN34TGD3.js";
+import {
+  BaseCommand
+} from "../../chunk-S56LOTGQ.js";
+import "../../chunk-OGVORIFM.js";
+import "../../chunk-2AM54TUK.js";
+import {
+  apiGet
+} from "../../chunk-EJOI2ZGU.js";
+import "../../chunk-KZRFUXGF.js";
+import "../../chunk-EKJ2CABV.js";
+import "../../chunk-NSBPE2FW.js";
+
+// src/commands/evals/compare.ts
+import { Args, Flags } from "@oclif/core";
+var EvalCompareCommand = class _EvalCompareCommand extends BaseCommand {
+  static description = "Blind A/B comparison between two eval target versions";
+  static examples = [
+    "<%= config.bin %> evals compare <target-a> <target-b>"
+  ];
+  static args = {
+    targetA: Args.string({ description: "First eval target ID", required: true }),
+    targetB: Args.string({ description: "Second eval target ID", required: true })
+  };
+  static flags = {
+    model: Flags.string({ description: "Model for execution and comparison" }),
+    timeout: Flags.integer({ description: "Timeout per eval in ms", default: 12e4 })
+  };
+  async run() {
+    const { args, flags } = await this.parse(_EvalCompareCommand);
+    this.requireAuth();
+    const [targetA, targetB] = await Promise.all([
+      apiGet(`/api/eval-targets/${args.targetA}`),
+      apiGet(`/api/eval-targets/${args.targetB}`)
+    ]);
+    const evals = await apiGet(`/api/eval-targets/${args.targetA}/evals`);
+    if (evals.length === 0) {
+      this.error("No evals found for target A. Create some first.");
+    }
+    this.log(`Comparing "${targetA.name}" v${targetA.version} vs "${targetB.name}" v${targetB.version}`);
+    this.log(`Running ${evals.length} eval(s)...
+`);
+    const isSwapped = Math.random() > 0.5;
+    const contentA = isSwapped ? targetB.content : targetA.content;
+    const contentB = isSwapped ? targetA.content : targetB.content;
+    const labelA = isSwapped ? targetB.name : targetA.name;
+    const labelB = isSwapped ? targetA.name : targetB.name;
+    const comparisons = [];
+    for (const eval_ of evals) {
+      this.log(`[${eval_.name}] Running A...`);
+      const resultA = await executeEval({
+        prompt: eval_.prompt,
+        targetContent: contentA,
+        timeout: flags.timeout,
+        model: flags.model
+      });
+      this.log(`[${eval_.name}] Running B...`);
+      const resultB = await executeEval({
+        prompt: eval_.prompt,
+        targetContent: contentB,
+        timeout: flags.timeout,
+        model: flags.model
+      });
+      comparisons.push({
+        evalName: eval_.name,
+        outputA: resultA.output,
+        outputB: resultB.output
+      });
+    }
+    this.log("\nRunning blind comparison...");
+    const comparisonPrompt = comparisons.map((c) => `
 ## Eval: ${c.evalName}
 
 ### Output A
@@ -67,9 +78,9 @@ ${c.outputA}
 
 ### Output B
 ${c.outputB}
-`).join('\n---\n');
-        const comparatorResult = await executeEval({
-            prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
+`).join("\n---\n");
+    const comparatorResult = await executeEval({
+      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
 Then provide an overall winner with reasoning.
 
 ${comparisonPrompt}
@@ -86,16 +97,17 @@ Respond with JSON:
     "weaknesses_loser": ["..."]
   }
 }`,
-            timeout: flags.timeout,
-            model: flags.model,
-        });
-        // Unblind and display results
-        this.log('\n=== Comparison Results ===\n');
-        this.log(`A = "${labelA}" | B = "${labelB}"`);
-        if (isSwapped)
-            this.log('(Labels were swapped for blind comparison)');
-        this.log('');
-        this.log(comparatorResult.output);
-    }
-}
-//# sourceMappingURL=compare.js.map
+      timeout: flags.timeout,
+      model: flags.model
+    });
+    this.log("\n=== Comparison Results ===\n");
+    this.log(`A = "${labelA}" | B = "${labelB}"`);
+    if (isSwapped) this.log("(Labels were swapped for blind comparison)");
+    this.log("");
+    this.log(comparatorResult.output);
+  }
+};
+export {
+  EvalCompareCommand as default
+};
+//# sourceMappingURL=compare.js.map

package/dist/commands/evals/compare.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../../src/commands/evals/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAgBzD,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,WAAW;IACzD,MAAM,CAAU,WAAW,GAAG,uDAAuD,CAAC;IAEtF,MAAM,CAAU,QAAQ,GAAG;QACzB,uDAAuD;KACxD,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7E,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,uBAAuB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KAC/E,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,oCAAoC,EAAE,CAAC;QAC1E,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;KACpF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3C,MAAM,CAAa,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC;YACvD,MAAM,CAAa,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,kEAAkE;QAClE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAS,qBAAqB,IAAI,CAAC,OAAO,QAAQ,CAAC,CAAC;QAC9E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,QAAQ,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,GAAG,CAAC,WAAW,KAAK,CAAC,MAAM,eAAe,CAAC,CAAC;QAEjD,yCAAyC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QACvD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAEvD,MAAM,WAAW,GAIZ,EAAE,CAAC;QAER,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,aAAa,EAAE,QAAQ;gBACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,aAAa,EAAE,QAAQ;gBACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YAEH,WAAW,CAAC,IAAI,CAAC;gBACf,QAAQ,EAAE,KAAK,CAAC,IAAI;gBACpB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,OAAO,EAAE,OAAO,CAAC,MAAM;aACxB,CAAC,CAAC;QACL,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAE1C,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;WACzC,CAAC,CAAC,QAAQ;;;EAGnB,CAAC,CAAC,OAAO;;;EAGT,CAAC,CAAC,OAAO;CACV,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEf,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC;YACzC,MAAM,EAAE;;;EAGZ,gBAAgB;;;;;;;;;;;;;EAahB;YACI,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,8BAA8B;QAC9B,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,MAAM,YAAY,MAAM,GAAG,CAAC,CAAC;QAC9C,IAAI,SAAS;YAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QACtE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC"}
+{
+  "version": 3,
+  "sources": ["../../../src/commands/evals/compare.ts"],
+  "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  content: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectations: string[];\n}\n\nexport default class EvalCompareCommand extends BaseCommand {\n  static override description = 'Blind A/B comparison between two eval target versions';\n\n  static override examples = [\n    '<%= config.bin %> evals compare <target-a> <target-b>',\n  ];\n\n  static override args = {\n    targetA: Args.string({ description: 'First eval target ID', required: true }),\n    targetB: Args.string({ description: 'Second eval target ID', required: true }),\n  };\n\n  static override flags = {\n    model: Flags.string({ description: 'Model for execution and comparison' }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCompareCommand);\n    this.requireAuth();\n\n    const [targetA, targetB] = await Promise.all([\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetA}`),\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetB}`),\n    ]);\n\n    // Use evals from target A (they should be testing the same thing)\n    const evals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetA}/evals`);\n    if (evals.length === 0) {\n      this.error('No evals found for target A. Create some first.');\n    }\n\n    this.log(`Comparing \"${targetA.name}\" v${targetA.version} vs \"${targetB.name}\" v${targetB.version}`);\n    this.log(`Running ${evals.length} eval(s)...\\n`);\n\n    // Randomize A/B assignment for blindness\n    const isSwapped = Math.random() > 0.5;\n    const contentA = isSwapped ? targetB.content : targetA.content;\n    const contentB = isSwapped ? targetA.content : targetB.content;\n    const labelA = isSwapped ? targetB.name : targetA.name;\n    const labelB = isSwapped ? targetA.name : targetB.name;\n\n    const comparisons: Array<{\n      evalName: string;\n      outputA: string;\n      outputB: string;\n    }> = [];\n\n    for (const eval_ of evals) {\n      this.log(`[${eval_.name}] Running A...`);\n      const resultA = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentA,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      this.log(`[${eval_.name}] Running B...`);\n      const resultB = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentB,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      comparisons.push({\n        evalName: eval_.name,\n        outputA: resultA.output,\n        outputB: resultB.output,\n      });\n    }\n\n    // Run blind comparison via LLM\n    this.log('\\nRunning blind comparison...');\n\n    const comparisonPrompt = comparisons.map((c) => `\n## Eval: ${c.evalName}\n\n### Output A\n${c.outputA}\n\n### Output B\n${c.outputB}\n`).join('\\n---\\n');\n\n    const comparatorResult = await executeEval({\n      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.\nThen provide an overall winner with reasoning.\n\n${comparisonPrompt}\n\nRespond with JSON:\n{\n  \"per_eval\": [\n    { \"eval_name\": \"...\", \"winner\": \"A|B|TIE\", \"reasoning\": \"...\" }\n  ],\n  \"overall\": {\n    \"winner\": \"A|B|TIE\",\n    \"reasoning\": \"...\",\n    \"strengths_winner\": [\"...\"],\n    \"weaknesses_loser\": [\"...\"]\n  }\n}`,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    // Unblind and display results\n    this.log('\\n=== Comparison Results ===\\n');\n    this.log(`A = \"${labelA}\" | B = \"${labelB}\"`);\n    if (isSwapped) this.log('(Labels were swapped for blind comparison)');\n    this.log('');\n    this.log(comparatorResult.output);\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmB5B,IAAqB,qBAArB,MAAqB,4BAA2B,YAAY;AAAA,EAC1D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,SAAS,KAAK,OAAO,EAAE,aAAa,wBAAwB,UAAU,KAAK,CAAC;AAAA,IAC5E,SAAS,KAAK,OAAO,EAAE,aAAa,yBAAyB,UAAU,KAAK,CAAC;AAAA,EAC/E;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,OAAO,MAAM,OAAO,EAAE,aAAa,qCAAqC,CAAC;AAAA,IACzE,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,mBAAkB;AAC3D,SAAK,YAAY;AAEjB,UAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC3C,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,MACtD,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,IACxD,CAAC;AAGD,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,OAAO,QAAQ;AAC5E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,iDAAiD;AAAA,IAC9D;AAEA,SAAK,IAAI,cAAc,QAAQ,IAAI,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,EAAE;AACnG,SAAK,IAAI,WAAW,MAAM,MAAM;AAAA,CAAe;AAG/C,UAAM,YAAY,KAAK,OAAO,IAAI;AAClC,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAClD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAElD,UAAM,cAID,CAAC;AAEN,eAAW,SAAS,OAAO;AACzB,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,kBAAY,KAAK;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AAGA,SAAK,IAAI,+BAA+B;AAExC,UAAM,mBAAmB,YAAY,IAAI,CAAC,MAAM;AAAA,WACzC,EAAE,QAAQ;AAAA;AAAA;AAAA,EAGnB,EAAE,OAAO;AAAA;AAAA;AAAA,EAGT,EAAE,OAAO;AAAA,CACV,EAAE,KAAK,SAAS;AAEb,UAAM,mBAAmB,MAAM,YAAY;AAAA,MACzC,QAAQ;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAcZ,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAGD,SAAK,IAAI,gCAAgC;AACzC,SAAK,IAAI,QAAQ,MAAM,YAAY,MAAM,GAAG;AAC5C,QAAI,UAAW,MAAK,IAAI,4CAA4C;AACpE,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,iBAAiB,MAAM;AAAA,EAClC;AACF;",
+  "names": []
+}

package/dist/commands/evals/create.js

@@ -1,52 +1,66 @@
-import { Args, Flags } from '@oclif/core';
-import { readFileSync } from 'node:fs';
-import { BaseCommand } from '../../base-command.js';
-import { apiPost } from '../../lib/api-client.js';
-export default class EvalCreate extends BaseCommand {
-    static description = 'Create test cases (evals) for an eval target';
-    static examples = [
-        '<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',
-        '<%= config.bin %> evals create <target-id> --from-file evals.json',
-    ];
-    static args = {
-        targetId: Args.string({ description: 'Eval target ID', required: true }),
-    };
-    static flags = {
-        name: Flags.string({ char: 'n', description: 'Eval name' }),
-        prompt: Flags.string({ char: 'p', description: 'Test prompt' }),
-        'expected-output': Flags.string({ description: 'Expected output description' }),
-        expectations: Flags.string({ char: 'e', description: 'Assertion (repeatable)', multiple: true }),
-        'from-file': Flags.string({ description: 'Bulk create from JSON file' }),
-    };
-    async run() {
-        const { args, flags } = await this.parse(EvalCreate);
-        this.requireAuth();
-        if (flags['from-file']) {
-            const data = JSON.parse(readFileSync(flags['from-file'], 'utf-8'));
-            const evalsArray = Array.isArray(data) ? data : data.evals;
-            for (const evalItem of evalsArray) {
-                const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-                    name: evalItem.name,
-                    prompt: evalItem.prompt,
-                    expectedOutput: evalItem.expectedOutput ?? '',
-                    expectations: evalItem.expectations ?? [],
-                    files: evalItem.files ?? [],
-                });
-                this.log(`Created eval: ${result.name} (${result.id})`);
-            }
-            this.log(`Created ${evalsArray.length} evals.`);
-            return;
-        }
-        if (!flags.name || !flags.prompt) {
-            this.error('Provide --name and --prompt, or use --from-file for bulk creation');
-        }
-        const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
-            name: flags.name,
-            prompt: flags.prompt,
-            expectedOutput: flags['expected-output'] ?? '',
-            expectations: flags.expectations ?? [],
+import {
+  BaseCommand
+} from "../../chunk-S56LOTGQ.js";
+import "../../chunk-OGVORIFM.js";
+import "../../chunk-2AM54TUK.js";
+import {
+  apiPost
+} from "../../chunk-EJOI2ZGU.js";
+import "../../chunk-KZRFUXGF.js";
+import "../../chunk-EKJ2CABV.js";
+import "../../chunk-NSBPE2FW.js";
+
+// src/commands/evals/create.ts
+import { Args, Flags } from "@oclif/core";
+import { readFileSync } from "node:fs";
+var EvalCreate = class _EvalCreate extends BaseCommand {
+  static description = "Create test cases (evals) for an eval target";
+  static examples = [
+    '<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',
+    "<%= config.bin %> evals create <target-id> --from-file evals.json"
+  ];
+  static args = {
+    targetId: Args.string({ description: "Eval target ID", required: true })
+  };
+  static flags = {
+    name: Flags.string({ char: "n", description: "Eval name" }),
+    prompt: Flags.string({ char: "p", description: "Test prompt" }),
+    "expected-output": Flags.string({ description: "Expected output description" }),
+    expectations: Flags.string({ char: "e", description: "Assertion (repeatable)", multiple: true }),
+    "from-file": Flags.string({ description: "Bulk create from JSON file" })
+  };
+  async run() {
+    const { args, flags } = await this.parse(_EvalCreate);
+    this.requireAuth();
+    if (flags["from-file"]) {
+      const data = JSON.parse(readFileSync(flags["from-file"], "utf-8"));
+      const evalsArray = Array.isArray(data) ? data : data.evals;
+      for (const evalItem of evalsArray) {
+        const result2 = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
+          name: evalItem.name,
+          prompt: evalItem.prompt,
+          expectedOutput: evalItem.expectedOutput ?? "",
+          expectations: evalItem.expectations ?? [],
+          files: evalItem.files ?? []
         });
-        this.log(`Created eval: ${result.name} (${result.id})`);
+        this.log(`Created eval: ${result2.name} (${result2.id})`);
+      }
+      this.log(`Created ${evalsArray.length} evals.`);
+      return;
     }
-}
-//# sourceMappingURL=create.js.map
+    if (!flags.name || !flags.prompt) {
+      this.error("Provide --name and --prompt, or use --from-file for bulk creation");
+    }
+    const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
+      name: flags.name,
+      prompt: flags.prompt,
+      expectedOutput: flags["expected-output"] ?? "",
+      expectations: flags.expectations ?? []
+    });
+    this.log(`Created eval: ${result.name} (${result.id})`);
+  }
+};
+export {
+  EvalCreate as default
+};
+//# sourceMappingURL=create.js.map

package/dist/commands/evals/create.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/commands/evals/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAUlD,MAAM,CAAC,OAAO,OAAO,UAAW,SAAQ,WAAW;IACjD,MAAM,CAAU,WAAW,GAAG,8CAA8C,CAAC;IAE7E,MAAM,CAAU,QAAQ,GAAG;QACzB,6GAA6G;QAC7G,mEAAmE;KACpE,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;QAC3D,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;QAC/D,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,6BAA6B,EAAE,CAAC;QAC/E,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChG,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,4BAA4B,EAAE,CAAC;KACzE,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,IAAI,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACnE,MAAM,UAAU,GAAgB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAExE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,MAAM,OAAO,CAA+B,qBAAqB,IAAI,CAAC,QAAQ,QAAQ,EAAE;oBACrG,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,EAAE;oBAC7C,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE;oBACzC,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;iBAC5B,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAA+B,qBAAqB,IAAI,CAAC,QAAQ,QAAQ,EAAE;YACrG,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,cAAc,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE;YAC9C,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,EAAE;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC"}
+{
+  "version": 3,
+  "sources": ["../../../src/commands/evals/create.ts"],
+  "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { readFileSync } from 'node:fs';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiPost } from '../../lib/api-client.js';\n\ninterface EvalInput {\n  name: string;\n  prompt: string;\n  expectedOutput?: string;\n  expectations?: string[];\n  files?: unknown[];\n}\n\nexport default class EvalCreate extends BaseCommand {\n  static override description = 'Create test cases (evals) for an eval target';\n\n  static override examples = [\n    '<%= config.bin %> evals create <target-id> --name \"ts-check\" --prompt \"Write a function to add two numbers\"',\n    '<%= config.bin %> evals create <target-id> --from-file evals.json',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    name: Flags.string({ char: 'n', description: 'Eval name' }),\n    prompt: Flags.string({ char: 'p', description: 'Test prompt' }),\n    'expected-output': Flags.string({ description: 'Expected output description' }),\n    expectations: Flags.string({ char: 'e', description: 'Assertion (repeatable)', multiple: true }),\n    'from-file': Flags.string({ description: 'Bulk create from JSON file' }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCreate);\n    this.requireAuth();\n\n    if (flags['from-file']) {\n      const data = JSON.parse(readFileSync(flags['from-file'], 'utf-8'));\n      const evalsArray: EvalInput[] = Array.isArray(data) ? data : data.evals;\n\n      for (const evalItem of evalsArray) {\n        const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n          name: evalItem.name,\n          prompt: evalItem.prompt,\n          expectedOutput: evalItem.expectedOutput ?? '',\n          expectations: evalItem.expectations ?? [],\n          files: evalItem.files ?? [],\n        });\n        this.log(`Created eval: ${result.name} (${result.id})`);\n      }\n\n      this.log(`Created ${evalsArray.length} evals.`);\n      return;\n    }\n\n    if (!flags.name || !flags.prompt) {\n      this.error('Provide --name and --prompt, or use --from-file for bulk creation');\n    }\n\n    const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n      name: flags.name,\n      prompt: flags.prompt,\n      expectedOutput: flags['expected-output'] ?? '',\n      expectations: flags.expectations ?? [],\n    });\n\n    this.log(`Created eval: ${result.name} (${result.id})`);\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAC5B,SAAS,oBAAoB;AAY7B,IAAqB,aAArB,MAAqB,oBAAmB,YAAY;AAAA,EAClD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,YAAY,CAAC;AAAA,IAC1D,QAAQ,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,cAAc,CAAC;AAAA,IAC9D,mBAAmB,MAAM,OAAO,EAAE,aAAa,8BAA8B,CAAC;AAAA,IAC9E,cAAc,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,0BAA0B,UAAU,KAAK,CAAC;AAAA,IAC/F,aAAa,MAAM,OAAO,EAAE,aAAa,6BAA6B,CAAC;AAAA,EACzE;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,WAAU;AACnD,SAAK,YAAY;AAEjB,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,OAAO,KAAK,MAAM,aAAa,MAAM,WAAW,GAAG,OAAO,CAAC;AACjE,YAAM,aAA0B,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK;AAElE,iBAAW,YAAY,YAAY;AACjC,cAAMA,UAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,UACrG,MAAM,SAAS;AAAA,UACf,QAAQ,SAAS;AAAA,UACjB,gBAAgB,SAAS,kBAAkB;AAAA,UAC3C,cAAc,SAAS,gBAAgB,CAAC;AAAA,UACxC,OAAO,SAAS,SAAS,CAAC;AAAA,QAC5B,CAAC;AACD,aAAK,IAAI,iBAAiBA,QAAO,IAAI,KAAKA,QAAO,EAAE,GAAG;AAAA,MACxD;AAEA,WAAK,IAAI,WAAW,WAAW,MAAM,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChC,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAEA,UAAM,SAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,MACrG,MAAM,MAAM;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM,iBAAiB,KAAK;AAAA,MAC5C,cAAc,MAAM,gBAAgB,CAAC;AAAA,IACvC,CAAC;AAED,SAAK,IAAI,iBAAiB,OAAO,IAAI,KAAK,OAAO,EAAE,GAAG;AAAA,EACxD;AACF;",
+  "names": ["result"]
+}