@rulemetric/cli 0.1.1

package/dist/commands/eval-targets/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/eval-targets/create.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGpD,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,WAAW;IACvD,OAAgB,WAAW,SAAyE;IAEpG,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,KAAK;;;;;;MAMnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAsB3B"}

package/dist/commands/eval-targets/create.js

@@ -0,0 +1,40 @@
+import { Flags } from '@oclif/core';
+import { readFileSync } from 'node:fs';
+import { BaseCommand } from '../../base-command.js';
+import { apiPost } from '../../lib/api-client.js';
+export default class EvalTargetCreate extends BaseCommand {
+    static description = 'Create an eval target (instruction, skill, prompt, or plan to test)';
+    static examples = [
+        '<%= config.bin %> eval-targets create --name my-rules --type instruction --file ./CLAUDE.md',
+        '<%= config.bin %> eval-targets create --name my-prompt --type prompt --content "Always use TypeScript"',
+    ];
+    static flags = {
+        name: Flags.string({ char: 'n', description: 'Name for the eval target', required: true }),
+        type: Flags.string({ char: 't', description: 'Target type', options: ['instruction', 'skill', 'prompt', 'plan'], required: true }),
+        file: Flags.string({ char: 'f', description: 'Path to content file' }),
+        content: Flags.string({ char: 'c', description: 'Content string (alternative to --file)' }),
+        description: Flags.string({ char: 'd', description: 'Description of the target', default: '' }),
+    };
+    async run() {
+        const { flags } = await this.parse(EvalTargetCreate);
+        this.requireAuth();
+        let content;
+        if (flags.file) {
+            content = readFileSync(flags.file, 'utf-8');
+        }
+        else if (flags.content) {
+            content = flags.content;
+        }
+        else {
+            this.error('Provide either --file or --content');
+        }
+        const result = await apiPost('/api/eval-targets', {
+            name: flags.name,
+            type: flags.type,
+            content,
+            description: flags.description,
+        });
+        this.log(`Created eval target: ${result.name} (${result.id}) v${result.version}`);
+    }
+}
+//# sourceMappingURL=create.js.map

package/dist/commands/eval-targets/create.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/commands/eval-targets/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,WAAW;IACvD,MAAM,CAAU,WAAW,GAAG,qEAAqE,CAAC;IAEpG,MAAM,CAAU,QAAQ,GAAG;QACzB,6FAA6F;QAC7F,wGAAwG;KACzG,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,0BAA0B,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC1F,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAClI,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC;QACtE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,wCAAwC,EAAE,CAAC;QAC3F,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,2BAA2B,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;KAChG,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,IAAI,OAAe,CAAC;QACpB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACzB,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAgD,mBAAmB,EAAE;YAC/F,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO;YACP,WAAW,EAAE,KAAK,CAAC,WAAW;SAC/B,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC"}

package/dist/commands/eval-targets/list.d.ts

@@ -0,0 +1,10 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalTargetList extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static flags: {
+        type: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+}
+//# sourceMappingURL=list.d.ts.map

package/dist/commands/eval-targets/list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/eval-targets/list.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAapD,MAAM,CAAC,OAAO,OAAO,cAAe,SAAQ,WAAW;IACrD,OAAgB,WAAW,SAAuB;IAElD,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,KAAK;;MAEnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAuB3B"}

package/dist/commands/eval-targets/list.js

@@ -0,0 +1,36 @@
+import { Flags } from '@oclif/core';
+import { printTable } from '@oclif/table';
+import { BaseCommand } from '../../base-command.js';
+import { apiGet } from '../../lib/api-client.js';
+import { formatDate, truncate } from '../../lib/output.js';
+export default class EvalTargetList extends BaseCommand {
+    static description = 'List eval targets';
+    static examples = [
+        '<%= config.bin %> eval-targets list',
+        '<%= config.bin %> eval-targets list --type instruction',
+    ];
+    static flags = {
+        type: Flags.string({ char: 't', description: 'Filter by type', options: ['instruction', 'skill', 'prompt', 'plan'] }),
+    };
+    async run() {
+        const { flags } = await this.parse(EvalTargetList);
+        this.requireAuth();
+        const query = flags.type ? `?type=${flags.type}` : '';
+        const results = await apiGet(`/api/eval-targets${query}`);
+        if (results.length === 0) {
+            this.log('No eval targets found. Create one with `rulemetric eval-targets create`.');
+            return;
+        }
+        printTable({
+            data: results.map((t) => ({
+                Name: truncate(t.name, 30),
+                Type: t.type,
+                Version: `v${t.version}`,
+                Description: truncate(t.description, 40),
+                Updated: formatDate(t.updatedAt),
+            })),
+            columns: ['Name', 'Type', 'Version', 'Description', 'Updated'],
+        });
+    }
+}
+//# sourceMappingURL=list.js.map

package/dist/commands/eval-targets/list.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"list.js","sourceRoot":"","sources":["../../../src/commands/eval-targets/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAW3D,MAAM,CAAC,OAAO,OAAO,cAAe,SAAQ,WAAW;IACrD,MAAM,CAAU,WAAW,GAAG,mBAAmB,CAAC;IAElD,MAAM,CAAU,QAAQ,GAAG;QACzB,qCAAqC;QACrC,wDAAwD;KACzD,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;KACtH,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAe,oBAAoB,KAAK,EAAE,CAAC,CAAC;QAExE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,0EAA0E,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QAED,UAAU,CAAC;YACT,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxB,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE;gBACxB,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;aACjC,CAAC,CAAC;YACH,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,SAAS,CAAC;SAC/D,CAAC,CAAC;IACL,CAAC"}

package/dist/commands/evals/benchmark.d.ts

@@ -0,0 +1,13 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalBenchmarkCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        iteration: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+}
+//# sourceMappingURL=benchmark.d.ts.map

package/dist/commands/evals/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/benchmark.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAIpD,MAAM,CAAC,OAAO,OAAO,oBAAqB,SAAQ,WAAW;IAC3D,OAAgB,WAAW,SAAuD;IAElF,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,IAAI;;MAElB;IAEF,OAAgB,KAAK;;MAEnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA0C3B"}

package/dist/commands/evals/benchmark.js

@@ -0,0 +1,53 @@
+import { Args, Flags } from '@oclif/core';
+import { BaseCommand } from '../../base-command.js';
+import { apiGet } from '../../lib/api-client.js';
+import { aggregateBenchmark, postBenchmark } from '../../lib/eval-benchmark.js';
+export default class EvalBenchmarkCommand extends BaseCommand {
+    static description = 'Aggregate eval runs into a benchmark for a target';
+    static examples = [
+        '<%= config.bin %> evals benchmark <target-id>',
+        '<%= config.bin %> evals benchmark <target-id> --iteration 2',
+    ];
+    static args = {
+        targetId: Args.string({ description: 'Eval target ID', required: true }),
+    };
+    static flags = {
+        iteration: Flags.integer({ description: 'Filter runs by iteration', default: 1 }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(EvalBenchmarkCommand);
+        this.requireAuth();
+        const target = await apiGet(`/api/eval-targets/${args.targetId}`);
+        this.log(`Aggregating benchmark for "${target.name}" v${target.version}, iteration ${flags.iteration}`);
+        const result = await aggregateBenchmark(args.targetId, flags.iteration);
+        if (result.totalRuns === 0) {
+            this.error('No completed runs found. Run evals first with `rulemetric evals run`.');
+        }
+        this.log(`\nAggregated ${result.totalRuns} runs\n`);
+        // Display summary
+        for (const [config, stats] of Object.entries(result.runSummary)) {
+            if (config === 'delta') {
+                this.log(`\nDelta (with_target - without_target):`);
+                for (const [metric, value] of Object.entries(stats)) {
+                    this.log(`  ${metric}: ${value}`);
+                }
+                continue;
+            }
+            this.log(`${config}:`);
+            for (const [metric, stat] of Object.entries(stats)) {
+                const s = stat;
+                this.log(`  ${metric}: mean=${s.mean} stddev=${s.stddev} [${s.min}, ${s.max}]`);
+            }
+        }
+        if (result.notes.length > 0) {
+            this.log('\nNotes:');
+            for (const note of result.notes) {
+                this.log(`  - ${note}`);
+            }
+        }
+        // Post to API
+        const benchmark = await postBenchmark(args.targetId, flags.iteration, result);
+        this.log(`\nBenchmark saved: ${benchmark.id}`);
+    }
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/commands/evals/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../../src/commands/evals/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEhF,MAAM,CAAC,OAAO,OAAO,oBAAqB,SAAQ,WAAW;IAC3D,MAAM,CAAU,WAAW,GAAG,mDAAmD,CAAC;IAElF,MAAM,CAAU,QAAQ,GAAG;QACzB,+CAA+C;QAC/C,6DAA6D;KAC9D,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,0BAA0B,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;KAClF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC/D,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAoC,qBAAqB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,GAAG,CAAC,8BAA8B,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,OAAO,eAAe,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAExG,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAExE,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,SAAS,SAAS,CAAC,CAAC;QAEpD,kBAAkB;QAClB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAChE,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;gBACpD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAA0C,CAAC,EAAE,CAAC;oBACzF,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;gBACpC,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,CAAC,GAAG,IAAkE,CAAC;gBAC7E,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,UAAU,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACrB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,cAAc;QACd,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC,GAAG,CAAC,sBAAsB,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC"}

package/dist/commands/evals/compare.d.ts

@@ -0,0 +1,15 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalCompareCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetA: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+        targetB: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        timeout: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+}
+//# sourceMappingURL=compare.d.ts.map

package/dist/commands/evals/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/compare.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAkBpD,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,WAAW;IACzD,OAAgB,WAAW,SAA2D;IAEtF,OAAgB,QAAQ,WAEtB;IAEF,OAAgB,IAAI;;;MAGlB;IAEF,OAAgB,KAAK;;;MAGnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAiG3B"}

package/dist/commands/evals/compare.js

@@ -0,0 +1,101 @@
+import { Args, Flags } from '@oclif/core';
+import { BaseCommand } from '../../base-command.js';
+import { apiGet } from '../../lib/api-client.js';
+import { executeEval } from '../../lib/eval-executor.js';
+export default class EvalCompareCommand extends BaseCommand {
+    static description = 'Blind A/B comparison between two eval target versions';
+    static examples = [
+        '<%= config.bin %> evals compare <target-a> <target-b>',
+    ];
+    static args = {
+        targetA: Args.string({ description: 'First eval target ID', required: true }),
+        targetB: Args.string({ description: 'Second eval target ID', required: true }),
+    };
+    static flags = {
+        model: Flags.string({ description: 'Model for execution and comparison' }),
+        timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(EvalCompareCommand);
+        this.requireAuth();
+        const [targetA, targetB] = await Promise.all([
+            apiGet(`/api/eval-targets/${args.targetA}`),
+            apiGet(`/api/eval-targets/${args.targetB}`),
+        ]);
+        // Use evals from target A (they should be testing the same thing)
+        const evals = await apiGet(`/api/eval-targets/${args.targetA}/evals`);
+        if (evals.length === 0) {
+            this.error('No evals found for target A. Create some first.');
+        }
+        this.log(`Comparing "${targetA.name}" v${targetA.version} vs "${targetB.name}" v${targetB.version}`);
+        this.log(`Running ${evals.length} eval(s)...\n`);
+        // Randomize A/B assignment for blindness
+        const isSwapped = Math.random() > 0.5;
+        const contentA = isSwapped ? targetB.content : targetA.content;
+        const contentB = isSwapped ? targetA.content : targetB.content;
+        const labelA = isSwapped ? targetB.name : targetA.name;
+        const labelB = isSwapped ? targetA.name : targetB.name;
+        const comparisons = [];
+        for (const eval_ of evals) {
+            this.log(`[${eval_.name}] Running A...`);
+            const resultA = await executeEval({
+                prompt: eval_.prompt,
+                targetContent: contentA,
+                timeout: flags.timeout,
+                model: flags.model,
+            });
+            this.log(`[${eval_.name}] Running B...`);
+            const resultB = await executeEval({
+                prompt: eval_.prompt,
+                targetContent: contentB,
+                timeout: flags.timeout,
+                model: flags.model,
+            });
+            comparisons.push({
+                evalName: eval_.name,
+                outputA: resultA.output,
+                outputB: resultB.output,
+            });
+        }
+        // Run blind comparison via LLM
+        this.log('\nRunning blind comparison...');
+        const comparisonPrompt = comparisons.map((c) => `
+## Eval: ${c.evalName}
+
+### Output A
+${c.outputA}
+
+### Output B
+${c.outputB}
+`).join('\n---\n');
+        const comparatorResult = await executeEval({
+            prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.
+Then provide an overall winner with reasoning.
+
+${comparisonPrompt}
+
+Respond with JSON:
+{
+  "per_eval": [
+    { "eval_name": "...", "winner": "A|B|TIE", "reasoning": "..." }
+  ],
+  "overall": {
+    "winner": "A|B|TIE",
+    "reasoning": "...",
+    "strengths_winner": ["..."],
+    "weaknesses_loser": ["..."]
+  }
+}`,
+            timeout: flags.timeout,
+            model: flags.model,
+        });
+        // Unblind and display results
+        this.log('\n=== Comparison Results ===\n');
+        this.log(`A = "${labelA}" | B = "${labelB}"`);
+        if (isSwapped)
+            this.log('(Labels were swapped for blind comparison)');
+        this.log('');
+        this.log(comparatorResult.output);
+    }
+}
+//# sourceMappingURL=compare.js.map

package/dist/commands/evals/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../../src/commands/evals/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAgBzD,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,WAAW;IACzD,MAAM,CAAU,WAAW,GAAG,uDAAuD,CAAC;IAEtF,MAAM,CAAU,QAAQ,GAAG;QACzB,uDAAuD;KACxD,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC7E,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,uBAAuB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KAC/E,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,oCAAoC,EAAE,CAAC;QAC1E,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;KACpF,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3C,MAAM,CAAa,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC;YACvD,MAAM,CAAa,qBAAqB,IAAI,CAAC,OAAO,EAAE,CAAC;SACxD,CAAC,CAAC;QAEH,kEAAkE;QAClE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAS,qBAAqB,IAAI,CAAC,OAAO,QAAQ,CAAC,CAAC;QAC9E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,QAAQ,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,GAAG,CAAC,WAAW,KAAK,CAAC,MAAM,eAAe,CAAC,CAAC;QAEjD,yCAAyC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QACvD,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAEvD,MAAM,WAAW,GAIZ,EAAE,CAAC;QAER,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,aAAa,EAAE,QAAQ;gBACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;gBAChC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,aAAa,EAAE,QAAQ;gBACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CAAC,CAAC;YAEH,WAAW,CAAC,IAAI,CAAC;gBACf,QAAQ,EAAE,KAAK,CAAC,IAAI;gBACpB,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,OAAO,EAAE,OAAO,CAAC,MAAM;aACxB,CAAC,CAAC;QACL,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;QAE1C,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;WACzC,CAAC,CAAC,QAAQ;;;EAGnB,CAAC,CAAC,OAAO;;;EAGT,CAAC,CAAC,OAAO;CACV,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEf,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC;YACzC,MAAM,EAAE;;;EAGZ,gBAAgB;;;;;;;;;;;;;EAahB;YACI,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;QAEH,8BAA8B;QAC9B,IAAI,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,QAAQ,MAAM,YAAY,MAAM,GAAG,CAAC,CAAC;QAC9C,IAAI,SAAS;YAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QACtE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC"}

package/dist/commands/evals/create.d.ts

@@ -0,0 +1,17 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalCreate extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        name: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        prompt: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'expected-output': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        expectations: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'from-file': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+}
+//# sourceMappingURL=create.d.ts.map

package/dist/commands/evals/create.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/create.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAWpD,MAAM,CAAC,OAAO,OAAO,UAAW,SAAQ,WAAW;IACjD,OAAgB,WAAW,SAAkD;IAE7E,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,IAAI;;MAElB;IAEF,OAAgB,KAAK;;;;;;MAMnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAoC3B"}

package/dist/commands/evals/create.js

@@ -0,0 +1,52 @@
+import { Args, Flags } from '@oclif/core';
+import { readFileSync } from 'node:fs';
+import { BaseCommand } from '../../base-command.js';
+import { apiPost } from '../../lib/api-client.js';
+export default class EvalCreate extends BaseCommand {
+    static description = 'Create test cases (evals) for an eval target';
+    static examples = [
+        '<%= config.bin %> evals create <target-id> --name "ts-check" --prompt "Write a function to add two numbers"',
+        '<%= config.bin %> evals create <target-id> --from-file evals.json',
+    ];
+    static args = {
+        targetId: Args.string({ description: 'Eval target ID', required: true }),
+    };
+    static flags = {
+        name: Flags.string({ char: 'n', description: 'Eval name' }),
+        prompt: Flags.string({ char: 'p', description: 'Test prompt' }),
+        'expected-output': Flags.string({ description: 'Expected output description' }),
+        expectations: Flags.string({ char: 'e', description: 'Assertion (repeatable)', multiple: true }),
+        'from-file': Flags.string({ description: 'Bulk create from JSON file' }),
+    };
+    async run() {
+        const { args, flags } = await this.parse(EvalCreate);
+        this.requireAuth();
+        if (flags['from-file']) {
+            const data = JSON.parse(readFileSync(flags['from-file'], 'utf-8'));
+            const evalsArray = Array.isArray(data) ? data : data.evals;
+            for (const evalItem of evalsArray) {
+                const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
+                    name: evalItem.name,
+                    prompt: evalItem.prompt,
+                    expectedOutput: evalItem.expectedOutput ?? '',
+                    expectations: evalItem.expectations ?? [],
+                    files: evalItem.files ?? [],
+                });
+                this.log(`Created eval: ${result.name} (${result.id})`);
+            }
+            this.log(`Created ${evalsArray.length} evals.`);
+            return;
+        }
+        if (!flags.name || !flags.prompt) {
+            this.error('Provide --name and --prompt, or use --from-file for bulk creation');
+        }
+        const result = await apiPost(`/api/eval-targets/${args.targetId}/evals`, {
+            name: flags.name,
+            prompt: flags.prompt,
+            expectedOutput: flags['expected-output'] ?? '',
+            expectations: flags.expectations ?? [],
+        });
+        this.log(`Created eval: ${result.name} (${result.id})`);
+    }
+}
+//# sourceMappingURL=create.js.map

package/dist/commands/evals/create.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/commands/evals/create.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAUlD,MAAM,CAAC,OAAO,OAAO,UAAW,SAAQ,WAAW;IACjD,MAAM,CAAU,WAAW,GAAG,8CAA8C,CAAC;IAE7E,MAAM,CAAU,QAAQ,GAAG;QACzB,6GAA6G;QAC7G,mEAAmE;KACpE,CAAC;IAEF,MAAM,CAAU,IAAI,GAAG;QACrB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;KACzE,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;QAC3D,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;QAC/D,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,6BAA6B,EAAE,CAAC;QAC/E,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,wBAAwB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAChG,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,4BAA4B,EAAE,CAAC;KACzE,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,IAAI,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACnE,MAAM,UAAU,GAAgB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAExE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;gBAClC,MAAM,MAAM,GAAG,MAAM,OAAO,CAA+B,qBAAqB,IAAI,CAAC,QAAQ,QAAQ,EAAE;oBACrG,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,EAAE;oBAC7C,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE;oBACzC,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;iBAC5B,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,SAAS,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAA+B,qBAAqB,IAAI,CAAC,QAAQ,QAAQ,EAAE;YACrG,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,cAAc,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE;YAC9C,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,EAAE;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1D,CAAC"}

package/dist/commands/evals/listen.d.ts

@@ -0,0 +1,11 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalListenCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static flags: {
+        'supabase-url': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'supabase-anon-key': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+}
+//# sourceMappingURL=listen.d.ts.map

package/dist/commands/evals/listen.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"listen.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/listen.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAIpD,MAAM,CAAC,OAAO,OAAO,iBAAkB,SAAQ,WAAW;IACxD,OAAgB,WAAW,SAAmE;IAE9F,OAAgB,QAAQ,WAEtB;IAEF,OAAgB,KAAK;;;MAWnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAqC3B"}

package/dist/commands/evals/listen.js

@@ -0,0 +1,54 @@
+import { Flags } from '@oclif/core';
+import { BaseCommand } from '../../base-command.js';
+import { startWorker, createWorkerId, getUserIdFromJwt } from '../../lib/eval-worker.js';
+import { loadToken } from '../../lib/auth.js';
+export default class EvalListenCommand extends BaseCommand {
+    static description = 'Listen for eval jobs from the web UI and execute them locally';
+    static examples = [
+        '<%= config.bin %> evals listen',
+    ];
+    static flags = {
+        'supabase-url': Flags.string({
+            description: 'Supabase project URL',
+            env: 'SUPABASE_URL',
+            required: true,
+        }),
+        'supabase-anon-key': Flags.string({
+            description: 'Supabase anon key',
+            env: 'SUPABASE_ANON_KEY',
+            required: true,
+        }),
+    };
+    async run() {
+        const { flags } = await this.parse(EvalListenCommand);
+        this.requireAuth();
+        const token = loadToken();
+        const accessToken = token?.accessToken ?? flags['supabase-anon-key'];
+        const userId = token ? getUserIdFromJwt(token.accessToken) : null;
+        if (!userId) {
+            this.error('Could not determine user ID. Run `rulemetric auth login`.');
+        }
+        this.log(`Starting eval worker (${createWorkerId()})...`);
+        this.log('Press Ctrl+C to stop.\n');
+        const cleanup = await startWorker({
+            supabaseUrl: flags['supabase-url'],
+            supabaseAnonKey: flags['supabase-anon-key'],
+            accessToken,
+            userId,
+            log: (msg) => this.log(msg),
+        });
+        // Keep alive until SIGINT
+        await new Promise((resolve) => {
+            process.on('SIGINT', () => {
+                this.log('\nShutting down worker...');
+                cleanup();
+                resolve();
+            });
+            process.on('SIGTERM', () => {
+                cleanup();
+                resolve();
+            });
+        });
+    }
+}
+//# sourceMappingURL=listen.js.map

package/dist/commands/evals/listen.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"listen.js","sourceRoot":"","sources":["../../../src/commands/evals/listen.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,MAAM,CAAC,OAAO,OAAO,iBAAkB,SAAQ,WAAW;IACxD,MAAM,CAAU,WAAW,GAAG,+DAA+D,CAAC;IAE9F,MAAM,CAAU,QAAQ,GAAG;QACzB,gCAAgC;KACjC,CAAC;IAEF,MAAM,CAAU,KAAK,GAAG;QACtB,cAAc,EAAE,KAAK,CAAC,MAAM,CAAC;YAC3B,WAAW,EAAE,sBAAsB;YACnC,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,IAAI;SACf,CAAC;QACF,mBAAmB,EAAE,KAAK,CAAC,MAAM,CAAC;YAChC,WAAW,EAAE,mBAAmB;YAChC,GAAG,EAAE,mBAAmB;YACxB,QAAQ,EAAE,IAAI;SACf,CAAC;KACH,CAAC;IAEF,KAAK,CAAC,GAAG;QACP,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,CAAC,WAAW,EAAE,CAAC;QAEnB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;QAE1B,MAAM,WAAW,GAAG,KAAK,EAAE,WAAW,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,yBAAyB,cAAc,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QAEpC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,WAAW,EAAE,KAAK,CAAC,cAAc,CAAC;YAClC,eAAe,EAAE,KAAK,CAAC,mBAAmB,CAAC;YAC3C,WAAW;YACX,MAAM;YACN,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;SAC5B,CAAC,CAAC;QAEH,0BAA0B;QAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;gBACtC,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;gBACzB,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC"}

package/dist/commands/evals/optimize-description.d.ts

@@ -0,0 +1,22 @@
+import { BaseCommand } from '../../base-command.js';
+export default class EvalOptimizeDescriptionCommand extends BaseCommand {
+    static description: string;
+    static examples: string[];
+    static args: {
+        targetId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        'max-iterations': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        queries: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'runs-per-query': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        'poll-interval': import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        timeout: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+    };
+    run(): Promise<void>;
+    private generateTriggerQueries;
+    private testQuerySet;
+    private printTriggerResults;
+    private improveDescription;
+}
+//# sourceMappingURL=optimize-description.d.ts.map

package/dist/commands/evals/optimize-description.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize-description.d.ts","sourceRoot":"","sources":["../../../src/commands/evals/optimize-description.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AA4BpD,MAAM,CAAC,OAAO,OAAO,8BAA+B,SAAQ,WAAW;IACrE,OAAgB,WAAW,SAAqE;IAEhG,OAAgB,QAAQ,WAGtB;IAEF,OAAgB,IAAI;;MAElB;IAEF,OAAgB,KAAK;;;;;;;MAOnB;IAEI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAoIZ,sBAAsB;YAsEtB,YAAY;IAyC1B,OAAO,CAAC,mBAAmB;YASb,kBAAkB;CAyCjC"}