@rulemetric/cli 0.6.8 → 0.7.0

package/dist/commands/evals/compare.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/compare.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  content: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectations: string[];\n}\n\nexport default class EvalCompareCommand extends BaseCommand {\n  static override description = 'Blind A/B comparison between two eval target versions';\n\n  static override examples = [\n    '<%= config.bin %> evals compare <target-a> <target-b>',\n  ];\n\n  static override args = {\n    targetA: Args.string({ description: 'First eval target ID', required: true }),\n    targetB: Args.string({ description: 'Second eval target ID', required: true }),\n  };\n\n  static override flags = {\n    model: Flags.string({ description: 'Model for execution and comparison' }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCompareCommand);\n    this.requireAuth();\n\n    const [targetA, targetB] = await Promise.all([\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetA}`),\n      apiGet<EvalTarget>(`/api/eval-targets/${args.targetB}`),\n    ]);\n\n    // Use evals from target A (they should be testing the same thing)\n    const evals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetA}/evals`);\n    if (evals.length === 0) {\n      this.error('No evals found for target A. Create some first.');\n    }\n\n    this.log(`Comparing \"${targetA.name}\" v${targetA.version} vs \"${targetB.name}\" v${targetB.version}`);\n    this.log(`Running ${evals.length} eval(s)...\\n`);\n\n    // Randomize A/B assignment for blindness\n    const isSwapped = Math.random() > 0.5;\n    const contentA = isSwapped ? targetB.content : targetA.content;\n    const contentB = isSwapped ? targetA.content : targetB.content;\n    const labelA = isSwapped ? targetB.name : targetA.name;\n    const labelB = isSwapped ? targetA.name : targetB.name;\n\n    const comparisons: Array<{\n      evalName: string;\n      outputA: string;\n      outputB: string;\n    }> = [];\n\n    for (const eval_ of evals) {\n      this.log(`[${eval_.name}] Running A...`);\n      const resultA = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentA,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      this.log(`[${eval_.name}] Running B...`);\n      const resultB = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: contentB,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      comparisons.push({\n        evalName: eval_.name,\n        outputA: resultA.output,\n        outputB: resultB.output,\n      });\n    }\n\n    // Run blind comparison via LLM\n    this.log('\\nRunning blind comparison...');\n\n    const comparisonPrompt = comparisons.map((c) => `\n## Eval: ${c.evalName}\n\n### Output A\n${c.outputA}\n\n### Output B\n${c.outputB}\n`).join('\\n---\\n');\n\n    const comparatorResult = await executeEval({\n      prompt: `Compare these two sets of outputs. For each eval, determine which output (A or B) is better.\nThen provide an overall winner with reasoning.\n\n${comparisonPrompt}\n\nRespond with JSON:\n{\n  \"per_eval\": [\n    { \"eval_name\": \"...\", \"winner\": \"A|B|TIE\", \"reasoning\": \"...\" }\n  ],\n  \"overall\": {\n    \"winner\": \"A|B|TIE\",\n    \"reasoning\": \"...\",\n    \"strengths_winner\": [\"...\"],\n    \"weaknesses_loser\": [\"...\"]\n  }\n}`,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    // Unblind and display results\n    this.log('\\n=== Comparison Results ===\\n');\n    this.log(`A = \"${labelA}\" | B = \"${labelB}\"`);\n    if (isSwapped) this.log('(Labels were swapped for blind comparison)');\n    this.log('');\n    this.log(comparatorResult.output);\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmB5B,IAAqB,qBAArB,MAAqB,4BAA2B,YAAY;AAAA,EAC1D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,SAAS,KAAK,OAAO,EAAE,aAAa,wBAAwB,UAAU,KAAK,CAAC;AAAA,IAC5E,SAAS,KAAK,OAAO,EAAE,aAAa,yBAAyB,UAAU,KAAK,CAAC;AAAA,EAC/E;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,OAAO,MAAM,OAAO,EAAE,aAAa,qCAAqC,CAAC;AAAA,IACzE,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,mBAAkB;AAC3D,SAAK,YAAY;AAEjB,UAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC3C,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,MACtD,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,IACxD,CAAC;AAGD,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,OAAO,QAAQ;AAC5E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,iDAAiD;AAAA,IAC9D;AAEA,SAAK,IAAI,cAAc,QAAQ,IAAI,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,EAAE;AACnG,SAAK,IAAI,WAAW,MAAM,MAAM;AAAA,CAAe;AAG/C,UAAM,YAAY,KAAK,OAAO,IAAI;AAClC,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAClD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAElD,UAAM,cAID,CAAC;AAEN,eAAW,SAAS,OAAO;AACzB,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,kBAAY,KAAK;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AAGA,SAAK,IAAI,+BAA+B;AAExC,UAAM,mBAAmB,YAAY,IAAI,CAAC,MAAM;AAAA,WACzC,EAAE,QAAQ;AAAA;AAAA;AAAA,EAGnB,EAAE,OAAO;AAAA;AAAA;AAAA,EAGT,EAAE,OAAO;AAAA,CACV,EAAE,KAAK,SAAS;AAEb,UAAM,mBAAmB,MAAM,YAAY;AAAA,MACzC,QAAQ;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAcZ,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAGD,SAAK,IAAI,gCAAgC;AACzC,SAAK,IAAI,QAAQ,MAAM,YAAY,MAAM,GAAG;AAC5C,QAAI,UAAW,MAAK,IAAI,4CAA4C;AACpE,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,iBAAiB,MAAM;AAAA,EAClC;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmB5B,IAAqB,qBAArB,MAAqB,4BAA2B,YAAY;AAAA,EAC1D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,SAAS,KAAK,OAAO,EAAE,aAAa,wBAAwB,UAAU,KAAK,CAAC;AAAA,IAC5E,SAAS,KAAK,OAAO,EAAE,aAAa,yBAAyB,UAAU,KAAK,CAAC;AAAA,EAC/E;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,OAAO,MAAM,OAAO,EAAE,aAAa,qCAAqC,CAAC;AAAA,IACzE,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,mBAAkB;AAC3D,SAAK,YAAY;AAEjB,UAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,MAC3C,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,MACtD,OAAmB,qBAAqB,KAAK,OAAO,EAAE;AAAA,IACxD,CAAC;AAGD,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,OAAO,QAAQ;AAC5E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,iDAAiD;AAAA,IAC9D;AAEA,SAAK,IAAI,cAAc,QAAQ,IAAI,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,MAAM,QAAQ,OAAO,EAAE;AACnG,SAAK,IAAI,WAAW,MAAM,MAAM;AAAA,CAAe;AAG/C,UAAM,YAAY,KAAK,OAAO,IAAI;AAClC,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,WAAW,YAAY,QAAQ,UAAU,QAAQ;AACvD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAClD,UAAM,SAAS,YAAY,QAAQ,OAAO,QAAQ;AAElD,UAAM,cAID,CAAC;AAEN,eAAW,SAAS,OAAO;AACzB,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,WAAK,IAAI,IAAI,MAAM,IAAI,gBAAgB;AACvC,YAAM,UAAU,MAAM,YAAY;AAAA,QAChC,QAAQ,MAAM;AAAA,QACd,eAAe;AAAA,QACf,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,kBAAY,KAAK;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,SAAS,QAAQ;AAAA,MACnB,CAAC;AAAA,IACH;AAGA,SAAK,IAAI,+BAA+B;AAExC,UAAM,mBAAmB,YAAY,IAAI,CAAC,MAAM;AAAA,WACzC,EAAE,QAAQ;AAAA;AAAA;AAAA,EAGnB,EAAE,OAAO;AAAA;AAAA;AAAA,EAGT,EAAE,OAAO;AAAA,CACV,EAAE,KAAK,SAAS;AAEb,UAAM,mBAAmB,MAAM,YAAY;AAAA,MACzC,QAAQ;AAAA;AAAA;AAAA,EAGZ,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAcZ,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAGD,SAAK,IAAI,gCAAgC;AACzC,SAAK,IAAI,QAAQ,MAAM,YAAY,MAAM,GAAG;AAC5C,QAAI,UAAW,MAAK,IAAI,4CAA4C;AACpE,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,iBAAiB,MAAM;AAAA,EAClC;AACF;",
   "names": []
 }

package/dist/commands/evals/create.js

@@ -1,12 +1,13 @@
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiPost
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/create.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/create.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { readFileSync } from 'node:fs';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiPost } from '../../lib/api-client.js';\n\ninterface EvalInput {\n  name: string;\n  prompt: string;\n  expectedOutput?: string;\n  expectations?: string[];\n  files?: unknown[];\n}\n\nexport default class EvalCreate extends BaseCommand {\n  static override description = 'Create test cases (evals) for an eval target';\n\n  static override examples = [\n    '<%= config.bin %> evals create <target-id> --name \"ts-check\" --prompt \"Write a function to add two numbers\"',\n    '<%= config.bin %> evals create <target-id> --from-file evals.json',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    name: Flags.string({ char: 'n', description: 'Eval name' }),\n    prompt: Flags.string({ char: 'p', description: 'Test prompt' }),\n    'expected-output': Flags.string({ description: 'Expected output description' }),\n    expectations: Flags.string({ char: 'e', description: 'Assertion (repeatable)', multiple: true }),\n    'from-file': Flags.string({ description: 'Bulk create from JSON file' }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalCreate);\n    this.requireAuth();\n\n    if (flags['from-file']) {\n      const data = JSON.parse(readFileSync(flags['from-file'], 'utf-8'));\n      const evalsArray: EvalInput[] = Array.isArray(data) ? data : data.evals;\n\n      for (const evalItem of evalsArray) {\n        const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n          name: evalItem.name,\n          prompt: evalItem.prompt,\n          expectedOutput: evalItem.expectedOutput ?? '',\n          expectations: evalItem.expectations ?? [],\n          files: evalItem.files ?? [],\n        });\n        this.log(`Created eval: ${result.name} (${result.id})`);\n      }\n\n      this.log(`Created ${evalsArray.length} evals.`);\n      return;\n    }\n\n    if (!flags.name || !flags.prompt) {\n      this.error('Provide --name and --prompt, or use --from-file for bulk creation');\n    }\n\n    const result = await apiPost<{ id: string; name: string }>(`/api/eval-targets/${args.targetId}/evals`, {\n      name: flags.name,\n      prompt: flags.prompt,\n      expectedOutput: flags['expected-output'] ?? '',\n      expectations: flags.expectations ?? [],\n    });\n\n    this.log(`Created eval: ${result.name} (${result.id})`);\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAC5B,SAAS,oBAAoB;AAY7B,IAAqB,aAArB,MAAqB,oBAAmB,YAAY;AAAA,EAClD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,YAAY,CAAC;AAAA,IAC1D,QAAQ,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,cAAc,CAAC;AAAA,IAC9D,mBAAmB,MAAM,OAAO,EAAE,aAAa,8BAA8B,CAAC;AAAA,IAC9E,cAAc,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,0BAA0B,UAAU,KAAK,CAAC;AAAA,IAC/F,aAAa,MAAM,OAAO,EAAE,aAAa,6BAA6B,CAAC;AAAA,EACzE;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,WAAU;AACnD,SAAK,YAAY;AAEjB,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,OAAO,KAAK,MAAM,aAAa,MAAM,WAAW,GAAG,OAAO,CAAC;AACjE,YAAM,aAA0B,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK;AAElE,iBAAW,YAAY,YAAY;AACjC,cAAMA,UAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,UACrG,MAAM,SAAS;AAAA,UACf,QAAQ,SAAS;AAAA,UACjB,gBAAgB,SAAS,kBAAkB;AAAA,UAC3C,cAAc,SAAS,gBAAgB,CAAC;AAAA,UACxC,OAAO,SAAS,SAAS,CAAC;AAAA,QAC5B,CAAC;AACD,aAAK,IAAI,iBAAiBA,QAAO,IAAI,KAAKA,QAAO,EAAE,GAAG;AAAA,MACxD;AAEA,WAAK,IAAI,WAAW,WAAW,MAAM,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChC,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAEA,UAAM,SAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,MACrG,MAAM,MAAM;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM,iBAAiB,KAAK;AAAA,MAC5C,cAAc,MAAM,gBAAgB,CAAC;AAAA,IACvC,CAAC;AAED,SAAK,IAAI,iBAAiB,OAAO,IAAI,KAAK,OAAO,EAAE,GAAG;AAAA,EACxD;AACF;",
+  "mappings": ";;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAC5B,SAAS,oBAAoB;AAY7B,IAAqB,aAArB,MAAqB,oBAAmB,YAAY;AAAA,EAClD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,YAAY,CAAC;AAAA,IAC1D,QAAQ,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,cAAc,CAAC;AAAA,IAC9D,mBAAmB,MAAM,OAAO,EAAE,aAAa,8BAA8B,CAAC;AAAA,IAC9E,cAAc,MAAM,OAAO,EAAE,MAAM,KAAK,aAAa,0BAA0B,UAAU,KAAK,CAAC;AAAA,IAC/F,aAAa,MAAM,OAAO,EAAE,aAAa,6BAA6B,CAAC;AAAA,EACzE;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,WAAU;AACnD,SAAK,YAAY;AAEjB,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,OAAO,KAAK,MAAM,aAAa,MAAM,WAAW,GAAG,OAAO,CAAC;AACjE,YAAM,aAA0B,MAAM,QAAQ,IAAI,IAAI,OAAO,KAAK;AAElE,iBAAW,YAAY,YAAY;AACjC,cAAMA,UAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,UACrG,MAAM,SAAS;AAAA,UACf,QAAQ,SAAS;AAAA,UACjB,gBAAgB,SAAS,kBAAkB;AAAA,UAC3C,cAAc,SAAS,gBAAgB,CAAC;AAAA,UACxC,OAAO,SAAS,SAAS,CAAC;AAAA,QAC5B,CAAC;AACD,aAAK,IAAI,iBAAiBA,QAAO,IAAI,KAAKA,QAAO,EAAE,GAAG;AAAA,MACxD;AAEA,WAAK,IAAI,WAAW,WAAW,MAAM,SAAS;AAC9C;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,QAAQ;AAChC,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAEA,UAAM,SAAS,MAAM,QAAsC,qBAAqB,KAAK,QAAQ,UAAU;AAAA,MACrG,MAAM,MAAM;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM,iBAAiB,KAAK;AAAA,MAC5C,cAAc,MAAM,gBAAgB,CAAC;AAAA,IACvC,CAAC;AAED,SAAK,IAAI,iBAAiB,OAAO,IAAI,KAAK,OAAO,EAAE,GAAG;AAAA,EACxD;AACF;",
   "names": ["result"]
 }

package/dist/commands/evals/optimize-description.js

@@ -7,15 +7,16 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost,
   apiPut
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/optimize-description.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/optimize-description.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost, apiPut } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { testTrigger, computeTriggerScore } from '../../lib/eval-trigger-tester.js';\nimport type { TriggerTestResult } from '../../lib/eval-trigger-tester.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  description: string;\n  version: number;\n}\n\ninterface TriggerQuery {\n  query: string;\n  should_trigger: boolean;\n}\n\ninterface TriggerEvalSet {\n  id: string;\n  queries: TriggerQuery[];\n  status: string;\n  trainIndices: number[];\n  testIndices: number[];\n}\n\nexport default class EvalOptimizeDescriptionCommand extends BaseCommand {\n  static override description = 'Optimize eval target description for better triggering accuracy';\n\n  static override examples = [\n    '<%= config.bin %> evals optimize-description <target-id>',\n    '<%= config.bin %> evals optimize-description <target-id> --queries 10',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),\n    model: Flags.string({ description: 'Model for trigger testing and improvement' }),\n    queries: Flags.integer({ description: 'Number of trigger queries to generate', default: 20 }),\n    'runs-per-query': Flags.integer({ description: 'Runs per query for reliability', default: 3 }),\n    'poll-interval': Flags.integer({ description: 'Ms between approval polls', default: 5000 }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalOptimizeDescriptionCommand);\n    this.requireAuth();\n\n    const target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n\n    this.log(`Optimizing description for \"${target.name}\" v${target.version}`);\n    this.log(`Current description: \"${target.description}\"`);\n    this.log('');\n\n    // Step 1: Generate trigger queries\n    this.log('Generating trigger queries...');\n    const queries = await this.generateTriggerQueries(target, flags.queries, flags);\n\n    // Step 2: Post to API for review\n    const triggerEval = await apiPost<TriggerEvalSet>(`/api/eval-targets/${args.targetId}/trigger-evals`, {\n      queries,\n    });\n\n    this.log(`\\nGenerated ${queries.length} trigger queries.`);\n    this.log(`Review and approve them at:`);\n    this.log(`  http://localhost:5174/eval-targets/${args.targetId}`);\n    this.log(`  (Triggers tab \u2192 click to review)`);\n    this.log('\\nWaiting for approval...');\n\n    // Step 3: Poll for approval\n    let approved = false;\n    while (!approved) {\n      await new Promise((resolve) => setTimeout(resolve, flags['poll-interval']));\n      const sets = await apiGet<TriggerEvalSet[]>(\n        `/api/eval-targets/${args.targetId}/trigger-evals?status=approved`,\n      );\n      const current = sets.find((s) => s.id === triggerEval.id);\n      if (current) {\n        approved = true;\n        // Use potentially edited queries\n        queries.length = 0;\n        queries.push(...current.queries);\n        this.log('Trigger queries approved!');\n      }\n    }\n\n    // Step 4: Split train/test (60/40)\n    const indices = queries.map((_, i) => i).sort(() => Math.random() - 0.5);\n    const splitIdx = Math.ceil(indices.length * 0.6);\n    const trainIndices = indices.slice(0, splitIdx);\n    const testIndices = indices.slice(splitIdx);\n\n    // Save split to API\n    await apiPut(`/api/trigger-evals/${triggerEval.id}`, {\n      trainIndices,\n      testIndices,\n    });\n\n    const trainQueries = trainIndices.map((i) => queries[i]);\n    const testQueries = testIndices.map((i) => queries[i]);\n\n    this.log(`\\nSplit: ${trainQueries.length} train, ${testQueries.length} test`);\n\n    // Step 5: Optimization loop\n    let currentDescription = target.description;\n    let bestDescription = currentDescription;\n    let bestTestScore = 0;\n\n    for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {\n      this.log(`\\n=== Iteration ${iteration} ===`);\n      this.log(`Description: \"${currentDescription}\"`);\n\n      // Test on train set (multiple runs per query)\n      this.log('Testing train set...');\n      const trainResults = await this.testQuerySet(trainQueries, target, currentDescription, flags);\n      const trainScore = computeTriggerScore(trainResults);\n      this.log(`Train accuracy: ${(trainScore * 100).toFixed(1)}%`);\n\n      this.printTriggerResults(trainResults);\n\n      // Test on test set\n      this.log('Testing test set...');\n      const testResults = await this.testQuerySet(testQueries, target, currentDescription, flags);\n      const testScore = computeTriggerScore(testResults);\n      this.log(`Test accuracy: ${(testScore * 100).toFixed(1)}%`);\n\n      // Record optimization run\n      await apiPost('/api/optimization-runs', {\n        evalTargetId: target.id,\n        type: 'description',\n        iteration,\n        inputVersion: target.version,\n        trainScore,\n        testScore,\n        analysis: {\n          description: currentDescription,\n          trainResults: trainResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n          testResults: testResults.map((r) => ({ query: r.query, shouldTrigger: r.shouldTrigger, didTrigger: r.didTrigger, correct: r.correct })),\n        },\n        status: 'completed',\n      });\n\n      // Track best by test score\n      if (testScore > bestTestScore) {\n        bestTestScore = testScore;\n        bestDescription = currentDescription;\n      }\n\n      // Stop if perfect\n      if (trainScore >= 1.0) {\n        this.log('Perfect train score. Stopping.');\n        break;\n      }\n\n      // Propose improved description\n      this.log('Generating improved description...');\n      const failures = trainResults.filter((r) => !r.correct);\n      currentDescription = await this.improveDescription(target, currentDescription, failures, flags);\n    }\n\n    // Step 6: Apply best description\n    this.log(`\\n=== Optimization Complete ===`);\n    this.log(`Best test accuracy: ${(bestTestScore * 100).toFixed(1)}%`);\n    this.log(`\\nBefore: \"${target.description}\"`);\n    this.log(`After:  \"${bestDescription}\"`);\n\n    if (bestDescription !== target.description) {\n      await apiPut(`/api/eval-targets/${target.id}`, {\n        description: bestDescription,\n      });\n      this.log('\\nDescription updated.');\n    } else {\n      this.log('\\nNo improvement found. Description unchanged.');\n    }\n  }\n\n  private async generateTriggerQueries(\n    target: EvalTarget,\n    count: number,\n    flags: { model?: string; timeout: number },\n  ): Promise<TriggerQuery[]> {\n    // Try to seed with real session data\n    let sessionSeed = '';\n    try {\n      const sessions = await apiGet<Array<{ id: string }>>('/api/sessions?limit=10');\n      if (sessions.length > 0) {\n        sessionSeed = '\\n\\nReal user prompts from recent sessions (use as inspiration for realistic queries):\\n' +\n          sessions.slice(0, 5).map((s) => `- Session ${s.id}`).join('\\n');\n      }\n    } catch {\n      // No sessions available, that's fine\n    }\n\n    const shouldTriggerCount = Math.ceil(count / 2);\n    const shouldNotTriggerCount = count - shouldTriggerCount;\n\n    const prompt = `Generate ${count} trigger eval queries for a skill/instruction.\n\n## Skill Name\n${target.name}\n\n## Skill Description\n${target.description}\n\n## Skill Content (first 1000 chars)\n${target.content.slice(0, 1000)}\n${sessionSeed}\n\nGenerate ${shouldTriggerCount} should-trigger queries and ${shouldNotTriggerCount} should-not-trigger queries.\n\nRules:\n- Queries must be realistic \u2014 what a real user would actually type\n- Include concrete details (file paths, project names, specific problems)\n- Vary length and formality\n- Should-not-trigger queries should be near-misses (share keywords but need something different)\n- Do NOT make should-not-trigger queries obviously irrelevant\n\nRespond with ONLY a valid JSON array:\n[\n  {\"query\": \"realistic user prompt...\", \"should_trigger\": true},\n  {\"query\": \"another prompt...\", \"should_trigger\": false}\n]`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    try {\n      let jsonStr = result.output.trim();\n      const jsonMatch = jsonStr.match(/```(?:json)?\\s*([\\s\\S]*?)```/);\n      if (jsonMatch) jsonStr = jsonMatch[1].trim();\n      const arrMatch = jsonStr.match(/\\[[\\s\\S]*\\]/);\n      if (arrMatch) jsonStr = arrMatch[0];\n\n      return JSON.parse(jsonStr) as TriggerQuery[];\n    } catch {\n      // Fallback: generate minimal set\n      return [\n        { query: `Help me use ${target.name}`, should_trigger: true },\n        { query: 'Write a hello world program', should_trigger: false },\n      ];\n    }\n  }\n\n  private async testQuerySet(\n    queries: TriggerQuery[],\n    target: EvalTarget,\n    description: string,\n    flags: { model?: string; timeout: number; 'runs-per-query': number },\n  ): Promise<TriggerTestResult[]> {\n    const results: TriggerTestResult[] = [];\n\n    for (const q of queries) {\n      // Run multiple times and take majority vote\n      const runs: TriggerTestResult[] = [];\n      for (let run = 0; run < flags['runs-per-query']; run++) {\n        const result = await testTrigger({\n          query: q.query,\n          shouldTrigger: q.should_trigger,\n          skillName: target.name,\n          skillDescription: description,\n          skillContent: target.content,\n          model: flags.model,\n          timeout: flags.timeout,\n        });\n        runs.push(result);\n      }\n\n      // Majority vote\n      const triggerCount = runs.filter((r) => r.didTrigger).length;\n      const didTrigger = triggerCount > flags['runs-per-query'] / 2;\n\n      results.push({\n        query: q.query,\n        shouldTrigger: q.should_trigger,\n        didTrigger,\n        correct: didTrigger === q.should_trigger,\n        confidence: runs.reduce((sum, r) => sum + r.confidence, 0) / runs.length,\n        evidence: runs[0].evidence,\n      });\n    }\n\n    return results;\n  }\n\n  private printTriggerResults(results: TriggerTestResult[]): void {\n    for (const r of results) {\n      const status = r.correct ? 'OK' : 'FAIL';\n      const expected = r.shouldTrigger ? 'should trigger' : 'should NOT trigger';\n      const actual = r.didTrigger ? 'triggered' : 'did NOT trigger';\n      this.log(`  [${status}] ${expected} / ${actual}: \"${r.query.slice(0, 60)}...\"`);\n    }\n  }\n\n  private async improveDescription(\n    target: EvalTarget,\n    currentDescription: string,\n    failures: TriggerTestResult[],\n    flags: { model?: string; timeout: number },\n  ): Promise<string> {\n    const failureText = failures.map((f) => {\n      const expected = f.shouldTrigger ? 'SHOULD trigger but DID NOT' : 'SHOULD NOT trigger but DID';\n      return `- ${expected}: \"${f.query}\"`;\n    }).join('\\n');\n\n    const prompt = `You are optimizing a skill description for better triggering accuracy.\n\n## Skill Name\n${target.name}\n\n## Current Description\n${currentDescription}\n\n## Skill Content (first 500 chars)\n${target.content.slice(0, 500)}\n\n## Failures\n${failureText}\n\nPropose an improved description that:\n- Fixes the trigger failures above\n- Keeps the description concise but comprehensive\n- Uses specific keywords and contexts for when to trigger\n- Includes \"Use PROACTIVELY\" phrasing for important trigger contexts\n\nRespond with ONLY the new description text (no JSON, no explanation, just the description string).`;\n\n    const result = await executeEval({\n      prompt,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    return result.output.trim().replace(/^[\"']|[\"']$/g, '');\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AA6B5B,IAAqB,iCAArB,MAAqB,wCAAuC,YAAY;AAAA,EACtE,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,OAAO,MAAM,OAAO,EAAE,aAAa,4CAA4C,CAAC;AAAA,IAChF,SAAS,MAAM,QAAQ,EAAE,aAAa,yCAAyC,SAAS,GAAG,CAAC;AAAA,IAC5F,kBAAkB,MAAM,QAAQ,EAAE,aAAa,kCAAkC,SAAS,EAAE,CAAC;AAAA,IAC7F,iBAAiB,MAAM,QAAQ,EAAE,aAAa,6BAA6B,SAAS,IAAK,CAAC;AAAA,IAC1F,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,+BAA8B;AACvE,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAE5E,SAAK,IAAI,+BAA+B,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzE,SAAK,IAAI,yBAAyB,OAAO,WAAW,GAAG;AACvD,SAAK,IAAI,EAAE;AAGX,SAAK,IAAI,+BAA+B;AACxC,UAAM,UAAU,MAAM,KAAK,uBAAuB,QAAQ,MAAM,SAAS,KAAK;AAG9E,UAAM,cAAc,MAAM,QAAwB,qBAAqB,KAAK,QAAQ,kBAAkB;AAAA,MACpG;AAAA,IACF,CAAC;AAED,SAAK,IAAI;AAAA,YAAe,QAAQ,MAAM,mBAAmB;AACzD,SAAK,IAAI,6BAA6B;AACtC,SAAK,IAAI,wCAAwC,KAAK,QAAQ,EAAE;AAChE,SAAK,IAAI,yCAAoC;AAC7C,SAAK,IAAI,2BAA2B;AAGpC,QAAI,WAAW;AACf,WAAO,CAAC,UAAU;AAChB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,MAAM,eAAe,CAAC,CAAC;AAC1E,YAAM,OAAO,MAAM;AAAA,QACjB,qBAAqB,KAAK,QAAQ;AAAA,MACpC;AACA,YAAM,UAAU,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE;AACxD,UAAI,SAAS;AACX,mBAAW;AAEX,gBAAQ,SAAS;AACjB,gBAAQ,KAAK,GAAG,QAAQ,OAAO;AAC/B,aAAK,IAAI,2BAA2B;AAAA,MACtC;AAAA,IACF;AAGA,UAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACvE,UAAM,WAAW,KAAK,KAAK,QAAQ,SAAS,GAAG;AAC/C,UAAM,eAAe,QAAQ,MAAM,GAAG,QAAQ;AAC9C,UAAM,cAAc,QAAQ,MAAM,QAAQ;AAG1C,UAAM,OAAO,sBAAsB,YAAY,EAAE,IAAI;AAAA,MACnD;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,eAAe,aAAa,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACvD,UAAM,cAAc,YAAY,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AAErD,SAAK,IAAI;AAAA,SAAY,aAAa,MAAM,WAAW,YAAY,MAAM,OAAO;AAG5E,QAAI,qBAAqB,OAAO;AAChC,QAAI,kBAAkB;AACtB,QAAI,gBAAgB;AAEpB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAC3C,WAAK,IAAI,iBAAiB,kBAAkB,GAAG;AAG/C,WAAK,IAAI,sBAAsB;AAC/B,YAAM,eAAe,MAAM,KAAK,aAAa,cAAc,QAAQ,oBAAoB,KAAK;AAC5F,YAAM,aAAa,oBAAoB,YAAY;AACnD,WAAK,IAAI,oBAAoB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE5D,WAAK,oBAAoB,YAAY;AAGrC,WAAK,IAAI,qBAAqB;AAC9B,YAAM,cAAc,MAAM,KAAK,aAAa,aAAa,QAAQ,oBAAoB,KAAK;AAC1F,YAAM,YAAY,oBAAoB,WAAW;AACjD,WAAK,IAAI,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG1D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa;AAAA,UACb,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,UACxI,aAAa,YAAY,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,QACxI;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,0BAAkB;AAAA,MACpB;AAGA,UAAI,cAAc,GAAK;AACrB,aAAK,IAAI,gCAAgC;AACzC;AAAA,MACF;AAGA,WAAK,IAAI,oCAAoC;AAC7C,YAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AACtD,2BAAqB,MAAM,KAAK,mBAAmB,QAAQ,oBAAoB,UAAU,KAAK;AAAA,IAChG;AAGA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,wBAAwB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,GAAG;AACnE,SAAK,IAAI;AAAA,WAAc,OAAO,WAAW,GAAG;AAC5C,SAAK,IAAI,YAAY,eAAe,GAAG;AAEvC,QAAI,oBAAoB,OAAO,aAAa;AAC1C,YAAM,OAAO,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC7C,aAAa;AAAA,MACf,CAAC;AACD,WAAK,IAAI,wBAAwB;AAAA,IACnC,OAAO;AACL,WAAK,IAAI,gDAAgD;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,QACA,OACA,OACyB;AAEzB,QAAI,cAAc;AAClB,QAAI;AACF,YAAM,WAAW,MAAM,OAA8B,wBAAwB;AAC7E,UAAI,SAAS,SAAS,GAAG;AACvB,sBAAc,6FACZ,SAAS,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,aAAa,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI;AAAA,MAClE;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,qBAAqB,KAAK,KAAK,QAAQ,CAAC;AAC9C,UAAM,wBAAwB,QAAQ;AAEtC,UAAM,SAAS,YAAY,KAAK;AAAA;AAAA;AAAA,EAGlC,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,OAAO,WAAW;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC;AAAA,EAC7B,WAAW;AAAA;AAAA,WAEF,kBAAkB,+BAA+B,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe7E,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI;AACF,UAAI,UAAU,OAAO,OAAO,KAAK;AACjC,YAAM,YAAY,QAAQ,MAAM,8BAA8B;AAC9D,UAAI,UAAW,WAAU,UAAU,CAAC,EAAE,KAAK;AAC3C,YAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAI,SAAU,WAAU,SAAS,CAAC;AAElC,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AAEN,aAAO;AAAA,QACL,EAAE,OAAO,eAAe,OAAO,IAAI,IAAI,gBAAgB,KAAK;AAAA,QAC5D,EAAE,OAAO,+BAA+B,gBAAgB,MAAM;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,aACZ,SACA,QACA,aACA,OAC8B;AAC9B,UAAM,UAA+B,CAAC;AAEtC,eAAW,KAAK,SAAS;AAEvB,YAAM,OAA4B,CAAC;AACnC,eAAS,MAAM,GAAG,MAAM,MAAM,gBAAgB,GAAG,OAAO;AACtD,cAAM,SAAS,MAAM,YAAY;AAAA,UAC/B,OAAO,EAAE;AAAA,UACT,eAAe,EAAE;AAAA,UACjB,WAAW,OAAO;AAAA,UAClB,kBAAkB;AAAA,UAClB,cAAc,OAAO;AAAA,UACrB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,aAAK,KAAK,MAAM;AAAA,MAClB;AAGA,YAAM,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE;AACtD,YAAM,aAAa,eAAe,MAAM,gBAAgB,IAAI;AAE5D,cAAQ,KAAK;AAAA,QACX,OAAO,EAAE;AAAA,QACT,eAAe,EAAE;AAAA,QACjB;AAAA,QACA,SAAS,eAAe,EAAE;AAAA,QAC1B,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,KAAK;AAAA,QAClE,UAAU,KAAK,CAAC,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,SAAoC;AAC9D,eAAW,KAAK,SAAS;AACvB,YAAM,SAAS,EAAE,UAAU,OAAO;AAClC,YAAM,WAAW,EAAE,gBAAgB,mBAAmB;AACtD,YAAM,SAAS,EAAE,aAAa,cAAc;AAC5C,WAAK,IAAI,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAChF;AAAA,EACF;AAAA,EAEA,MAAc,mBACZ,QACA,oBACA,UACA,OACiB;AACjB,UAAM,cAAc,SAAS,IAAI,CAAC,MAAM;AACtC,YAAM,WAAW,EAAE,gBAAgB,+BAA+B;AAClE,aAAO,KAAK,QAAQ,MAAM,EAAE,KAAK;AAAA,IACnC,CAAC,EAAE,KAAK,IAAI;AAEZ,UAAM,SAAS;AAAA;AAAA;AAAA,EAGjB,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,kBAAkB;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA;AAAA,EAG5B,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUT,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO,OAAO,OAAO,KAAK,EAAE,QAAQ,gBAAgB,EAAE;AAAA,EACxD;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AA6B5B,IAAqB,iCAArB,MAAqB,wCAAuC,YAAY;AAAA,EACtE,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,OAAO,MAAM,OAAO,EAAE,aAAa,4CAA4C,CAAC;AAAA,IAChF,SAAS,MAAM,QAAQ,EAAE,aAAa,yCAAyC,SAAS,GAAG,CAAC;AAAA,IAC5F,kBAAkB,MAAM,QAAQ,EAAE,aAAa,kCAAkC,SAAS,EAAE,CAAC;AAAA,IAC7F,iBAAiB,MAAM,QAAQ,EAAE,aAAa,6BAA6B,SAAS,IAAK,CAAC;AAAA,IAC1F,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,+BAA8B;AACvE,SAAK,YAAY;AAEjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAE5E,SAAK,IAAI,+BAA+B,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzE,SAAK,IAAI,yBAAyB,OAAO,WAAW,GAAG;AACvD,SAAK,IAAI,EAAE;AAGX,SAAK,IAAI,+BAA+B;AACxC,UAAM,UAAU,MAAM,KAAK,uBAAuB,QAAQ,MAAM,SAAS,KAAK;AAG9E,UAAM,cAAc,MAAM,QAAwB,qBAAqB,KAAK,QAAQ,kBAAkB;AAAA,MACpG;AAAA,IACF,CAAC;AAED,SAAK,IAAI;AAAA,YAAe,QAAQ,MAAM,mBAAmB;AACzD,SAAK,IAAI,6BAA6B;AACtC,SAAK,IAAI,wCAAwC,KAAK,QAAQ,EAAE;AAChE,SAAK,IAAI,yCAAoC;AAC7C,SAAK,IAAI,2BAA2B;AAGpC,QAAI,WAAW;AACf,WAAO,CAAC,UAAU;AAChB,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,MAAM,eAAe,CAAC,CAAC;AAC1E,YAAM,OAAO,MAAM;AAAA,QACjB,qBAAqB,KAAK,QAAQ;AAAA,MACpC;AACA,YAAM,UAAU,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,YAAY,EAAE;AACxD,UAAI,SAAS;AACX,mBAAW;AAEX,gBAAQ,SAAS;AACjB,gBAAQ,KAAK,GAAG,QAAQ,OAAO;AAC/B,aAAK,IAAI,2BAA2B;AAAA,MACtC;AAAA,IACF;AAGA,UAAM,UAAU,QAAQ,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AACvE,UAAM,WAAW,KAAK,KAAK,QAAQ,SAAS,GAAG;AAC/C,UAAM,eAAe,QAAQ,MAAM,GAAG,QAAQ;AAC9C,UAAM,cAAc,QAAQ,MAAM,QAAQ;AAG1C,UAAM,OAAO,sBAAsB,YAAY,EAAE,IAAI;AAAA,MACnD;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,eAAe,aAAa,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AACvD,UAAM,cAAc,YAAY,IAAI,CAAC,MAAM,QAAQ,CAAC,CAAC;AAErD,SAAK,IAAI;AAAA,SAAY,aAAa,MAAM,WAAW,YAAY,MAAM,OAAO;AAG5E,QAAI,qBAAqB,OAAO;AAChC,QAAI,kBAAkB;AACtB,QAAI,gBAAgB;AAEpB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAC3C,WAAK,IAAI,iBAAiB,kBAAkB,GAAG;AAG/C,WAAK,IAAI,sBAAsB;AAC/B,YAAM,eAAe,MAAM,KAAK,aAAa,cAAc,QAAQ,oBAAoB,KAAK;AAC5F,YAAM,aAAa,oBAAoB,YAAY;AACnD,WAAK,IAAI,oBAAoB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE5D,WAAK,oBAAoB,YAAY;AAGrC,WAAK,IAAI,qBAAqB;AAC9B,YAAM,cAAc,MAAM,KAAK,aAAa,aAAa,QAAQ,oBAAoB,KAAK;AAC1F,YAAM,YAAY,oBAAoB,WAAW;AACjD,WAAK,IAAI,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG1D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa;AAAA,UACb,cAAc,aAAa,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,UACxI,aAAa,YAAY,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,eAAe,EAAE,eAAe,YAAY,EAAE,YAAY,SAAS,EAAE,QAAQ,EAAE;AAAA,QACxI;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,0BAAkB;AAAA,MACpB;AAGA,UAAI,cAAc,GAAK;AACrB,aAAK,IAAI,gCAAgC;AACzC;AAAA,MACF;AAGA,WAAK,IAAI,oCAAoC;AAC7C,YAAM,WAAW,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO;AACtD,2BAAqB,MAAM,KAAK,mBAAmB,QAAQ,oBAAoB,UAAU,KAAK;AAAA,IAChG;AAGA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,wBAAwB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,GAAG;AACnE,SAAK,IAAI;AAAA,WAAc,OAAO,WAAW,GAAG;AAC5C,SAAK,IAAI,YAAY,eAAe,GAAG;AAEvC,QAAI,oBAAoB,OAAO,aAAa;AAC1C,YAAM,OAAO,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC7C,aAAa;AAAA,MACf,CAAC;AACD,WAAK,IAAI,wBAAwB;AAAA,IACnC,OAAO;AACL,WAAK,IAAI,gDAAgD;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAc,uBACZ,QACA,OACA,OACyB;AAEzB,QAAI,cAAc;AAClB,QAAI;AACF,YAAM,WAAW,MAAM,OAA8B,wBAAwB;AAC7E,UAAI,SAAS,SAAS,GAAG;AACvB,sBAAc,6FACZ,SAAS,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,aAAa,EAAE,EAAE,EAAE,EAAE,KAAK,IAAI;AAAA,MAClE;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,qBAAqB,KAAK,KAAK,QAAQ,CAAC;AAC9C,UAAM,wBAAwB,QAAQ;AAEtC,UAAM,SAAS,YAAY,KAAK;AAAA;AAAA;AAAA,EAGlC,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,OAAO,WAAW;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAI,CAAC;AAAA,EAC7B,WAAW;AAAA;AAAA,WAEF,kBAAkB,+BAA+B,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe7E,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI;AACF,UAAI,UAAU,OAAO,OAAO,KAAK;AACjC,YAAM,YAAY,QAAQ,MAAM,8BAA8B;AAC9D,UAAI,UAAW,WAAU,UAAU,CAAC,EAAE,KAAK;AAC3C,YAAM,WAAW,QAAQ,MAAM,aAAa;AAC5C,UAAI,SAAU,WAAU,SAAS,CAAC;AAElC,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AAEN,aAAO;AAAA,QACL,EAAE,OAAO,eAAe,OAAO,IAAI,IAAI,gBAAgB,KAAK;AAAA,QAC5D,EAAE,OAAO,+BAA+B,gBAAgB,MAAM;AAAA,MAChE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,aACZ,SACA,QACA,aACA,OAC8B;AAC9B,UAAM,UAA+B,CAAC;AAEtC,eAAW,KAAK,SAAS;AAEvB,YAAM,OAA4B,CAAC;AACnC,eAAS,MAAM,GAAG,MAAM,MAAM,gBAAgB,GAAG,OAAO;AACtD,cAAM,SAAS,MAAM,YAAY;AAAA,UAC/B,OAAO,EAAE;AAAA,UACT,eAAe,EAAE;AAAA,UACjB,WAAW,OAAO;AAAA,UAClB,kBAAkB;AAAA,UAClB,cAAc,OAAO;AAAA,UACrB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,aAAK,KAAK,MAAM;AAAA,MAClB;AAGA,YAAM,eAAe,KAAK,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE;AACtD,YAAM,aAAa,eAAe,MAAM,gBAAgB,IAAI;AAE5D,cAAQ,KAAK;AAAA,QACX,OAAO,EAAE;AAAA,QACT,eAAe,EAAE;AAAA,QACjB;AAAA,QACA,SAAS,eAAe,EAAE;AAAA,QAC1B,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,YAAY,CAAC,IAAI,KAAK;AAAA,QAClE,UAAU,KAAK,CAAC,EAAE;AAAA,MACpB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,SAAoC;AAC9D,eAAW,KAAK,SAAS;AACvB,YAAM,SAAS,EAAE,UAAU,OAAO;AAClC,YAAM,WAAW,EAAE,gBAAgB,mBAAmB;AACtD,YAAM,SAAS,EAAE,aAAa,cAAc;AAC5C,WAAK,IAAI,MAAM,MAAM,KAAK,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAChF;AAAA,EACF;AAAA,EAEA,MAAc,mBACZ,QACA,oBACA,UACA,OACiB;AACjB,UAAM,cAAc,SAAS,IAAI,CAAC,MAAM;AACtC,YAAM,WAAW,EAAE,gBAAgB,+BAA+B;AAClE,aAAO,KAAK,QAAQ,MAAM,EAAE,KAAK;AAAA,IACnC,CAAC,EAAE,KAAK,IAAI;AAEZ,UAAM,SAAS;AAAA;AAAA;AAAA,EAGjB,OAAO,IAAI;AAAA;AAAA;AAAA,EAGX,kBAAkB;AAAA;AAAA;AAAA,EAGlB,OAAO,QAAQ,MAAM,GAAG,GAAG,CAAC;AAAA;AAAA;AAAA,EAG5B,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUT,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO,OAAO,OAAO,KAAK,EAAE,QAAQ,gBAAgB,EAAE;AAAA,EACxD;AACF;",
   "names": []
 }

package/dist/commands/evals/optimize.js

@@ -9,15 +9,16 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost,
   apiPut
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/optimize.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/optimize.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost, apiPut } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { gradeEvalOutput } from '../../lib/eval-grader.js';\nimport { analyzeEvalResults } from '../../lib/eval-analyzer.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  description: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectedOutput: string;\n  expectations: string[];\n}\n\nexport default class EvalOptimizeCommand extends BaseCommand {\n  static override description = 'Iteratively optimize eval target content using eval results as feedback';\n\n  static override examples = [\n    '<%= config.bin %> evals optimize <target-id>',\n    '<%= config.bin %> evals optimize <target-id> --auto --max-iterations 3',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    auto: Flags.boolean({ description: 'Skip interactive review, apply suggestions automatically', default: false }),\n    'max-iterations': Flags.integer({ description: 'Maximum optimization iterations', default: 5 }),\n    threshold: Flags.string({ description: 'Stop if pass rate reaches this threshold (0-1)', default: '0.95' }),\n    model: Flags.string({ description: 'Model for eval execution' }),\n    'analyzer-model': Flags.string({ description: 'Model for analysis/suggestions' }),\n    'runs-per-eval': Flags.integer({ description: 'Runs per eval per iteration', default: 1 }),\n    parallel: Flags.integer({ description: 'Number of parallel eval executions', default: 1 }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalOptimizeCommand);\n    this.requireAuth();\n\n    let target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n    const allEvals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetId}/evals`);\n\n    if (allEvals.length === 0) {\n      this.error('No evals found. Create some first with `rulemetric evals create`.');\n    }\n\n    // Split evals into train/test (60/40)\n    const shuffled = [...allEvals].sort(() => Math.random() - 0.5);\n    const splitIdx = Math.ceil(shuffled.length * 0.6);\n    const trainEvals = shuffled.slice(0, splitIdx);\n    const testEvals = shuffled.slice(splitIdx);\n\n    this.log(`Optimizing \"${target.name}\" v${target.version}`);\n    this.log(`Evals: ${trainEvals.length} train, ${testEvals.length} test`);\n    const threshold = parseFloat(flags.threshold);\n    this.log(`Config: max ${flags['max-iterations']} iterations, threshold ${(threshold * 100).toFixed(0)}%`);\n    if (flags.auto) this.log('Mode: auto (no interactive review)');\n    this.log('');\n\n    let bestVersion = target.version;\n    let bestTestScore = 0;\n    let noImprovementCount = 0;\n\n    for (let iteration = 1; iteration <= flags['max-iterations']; iteration++) {\n      this.log(`\\n=== Iteration ${iteration} ===`);\n\n      // Record optimization run as running\n      const optRun = await apiPost<{ id: string }>('/api/optimization-runs', {\n        evalTargetId: target.id,\n        type: 'content',\n        iteration,\n        inputVersion: target.version,\n        status: 'running',\n      });\n\n      // Run train evals\n      this.log('Running train evals...');\n      const trainResults = await this.runEvalsOnTarget(target, trainEvals, flags);\n      const trainScore = this.computePassRate(trainResults);\n      this.log(`Train pass rate: ${(trainScore * 100).toFixed(1)}%`);\n\n      // Check if we've hit the threshold\n      if (trainScore >= threshold) {\n        this.log(`Train score ${(trainScore * 100).toFixed(1)}% >= threshold ${(threshold * 100).toFixed(0)}%. Stopping.`);\n        // Run test evals for final score\n        const testResults = await this.runEvalsOnTarget(target, testEvals, flags);\n        const testScore = this.computePassRate(testResults);\n        this.log(`Final test score: ${(testScore * 100).toFixed(1)}%`);\n\n        await apiPost('/api/optimization-runs', {\n          evalTargetId: target.id,\n          type: 'content',\n          iteration,\n          inputVersion: target.version,\n          trainScore,\n          testScore,\n          analysis: { reason: 'threshold_reached' },\n          status: 'completed',\n        });\n        break;\n      }\n\n      // Analyze and propose improvements\n      this.log('Analyzing results...');\n      const evalResults = trainResults.map((r) => ({\n        evalName: r.evalName,\n        prompt: r.prompt,\n        expectations: r.expectations,\n        grading: r.grading,\n      }));\n\n      const analysis = await analyzeEvalResults({\n        content: target.content,\n        evalResults,\n        model: flags['analyzer-model'] ?? flags.model,\n        timeout: flags.timeout,\n      });\n\n      this.log(`\\nAssessment: ${analysis.overallAssessment}`);\n      this.log(`Suggestions: ${analysis.suggestions.length}`);\n      for (const s of analysis.suggestions) {\n        this.log(`  - [${s.section}] ${s.reasoning}`);\n      }\n\n      // In interactive mode, ask for approval\n      if (!flags.auto) {\n        this.log('\\nProposed changes above. Applying in auto mode would proceed automatically.');\n        this.log('(Interactive approval not yet implemented \u2014 use --auto for now)');\n      }\n\n      if (analysis.proposedContent === target.content) {\n        this.log('No changes proposed. Stopping.');\n        await apiPost('/api/optimization-runs', {\n          evalTargetId: target.id,\n          type: 'content',\n          iteration,\n          inputVersion: target.version,\n          trainScore,\n          analysis: { reason: 'no_changes' },\n          status: 'completed',\n        });\n        break;\n      }\n\n      // Create new version\n      const newTarget = await apiPut<EvalTarget>(`/api/eval-targets/${target.id}`, {\n        content: analysis.proposedContent,\n      });\n      this.log(`Created v${newTarget.version}`);\n\n      // Run test evals on new version\n      this.log('Running test evals...');\n      const testResults = await this.runEvalsOnTarget(newTarget, testEvals, flags);\n      const testScore = this.computePassRate(testResults);\n      this.log(`Test pass rate: ${(testScore * 100).toFixed(1)}%`);\n\n      // Update optimization run\n      await apiPost('/api/optimization-runs', {\n        evalTargetId: newTarget.id,\n        type: 'content',\n        iteration,\n        inputVersion: target.version,\n        outputVersion: newTarget.version,\n        trainScore,\n        testScore,\n        analysis: {\n          suggestions: analysis.suggestions,\n          overallAssessment: analysis.overallAssessment,\n        },\n        status: 'completed',\n      });\n\n      // Track best\n      if (testScore > bestTestScore) {\n        bestTestScore = testScore;\n        bestVersion = newTarget.version;\n        noImprovementCount = 0;\n      } else {\n        noImprovementCount++;\n        if (noImprovementCount >= 2) {\n          this.log('No improvement for 2 consecutive iterations. Stopping.');\n          break;\n        }\n      }\n\n      target = newTarget;\n    }\n\n    this.log(`\\n=== Optimization Complete ===`);\n    this.log(`Best version: v${bestVersion} (test score: ${(bestTestScore * 100).toFixed(1)}%)`);\n  }\n\n  private async runEvalsOnTarget(\n    target: EvalTarget,\n    evalSet: Eval[],\n    flags: { model?: string; timeout: number; 'runs-per-eval': number; parallel: number },\n  ): Promise<Array<{ evalName: string; prompt: string; expectations: string[]; grading: Record<string, unknown> }>> {\n    const results: Array<{ evalName: string; prompt: string; expectations: string[]; grading: Record<string, unknown> }> = [];\n\n    for (const eval_ of evalSet) {\n      const execResult = await executeEval({\n        prompt: eval_.prompt,\n        targetContent: target.content,\n        timeout: flags.timeout,\n        model: flags.model,\n      });\n\n      if (execResult.error) {\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });\n        continue;\n      }\n\n      if (eval_.expectations.length > 0) {\n        const grading = await gradeEvalOutput({\n          prompt: eval_.prompt,\n          output: execResult.output,\n          expectations: eval_.expectations,\n          model: flags.model,\n          timeout: flags.timeout,\n        });\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: grading as unknown as Record<string, unknown> });\n      } else {\n        results.push({ evalName: eval_.name, prompt: eval_.prompt, expectations: eval_.expectations, grading: {} });\n      }\n    }\n\n    return results;\n  }\n\n  private computePassRate(results: Array<{ grading: Record<string, unknown> }>): number {\n    const graded = results.filter((r) => Object.keys(r.grading).length > 0);\n    if (graded.length === 0) return 0;\n    const rates = graded.map((r) => (r.grading as any).summary?.pass_rate ?? 0);\n    return rates.reduce((a: number, b: number) => a + b, 0) / rates.length;\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAwB5B,IAAqB,sBAArB,MAAqB,6BAA4B,YAAY;AAAA,EAC3D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,QAAQ,EAAE,aAAa,4DAA4D,SAAS,MAAM,CAAC;AAAA,IAC/G,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,WAAW,MAAM,OAAO,EAAE,aAAa,kDAAkD,SAAS,OAAO,CAAC;AAAA,IAC1G,OAAO,MAAM,OAAO,EAAE,aAAa,2BAA2B,CAAC;AAAA,IAC/D,kBAAkB,MAAM,OAAO,EAAE,aAAa,iCAAiC,CAAC;AAAA,IAChF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,+BAA+B,SAAS,EAAE,CAAC;AAAA,IACzF,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,oBAAmB;AAC5D,SAAK,YAAY;AAEjB,QAAI,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC1E,UAAM,WAAW,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAEhF,QAAI,SAAS,WAAW,GAAG;AACzB,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAGA,UAAM,WAAW,CAAC,GAAG,QAAQ,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAC7D,UAAM,WAAW,KAAK,KAAK,SAAS,SAAS,GAAG;AAChD,UAAM,aAAa,SAAS,MAAM,GAAG,QAAQ;AAC7C,UAAM,YAAY,SAAS,MAAM,QAAQ;AAEzC,SAAK,IAAI,eAAe,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzD,SAAK,IAAI,UAAU,WAAW,MAAM,WAAW,UAAU,MAAM,OAAO;AACtE,UAAM,YAAY,WAAW,MAAM,SAAS;AAC5C,SAAK,IAAI,eAAe,MAAM,gBAAgB,CAAC,2BAA2B,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AACxG,QAAI,MAAM,KAAM,MAAK,IAAI,oCAAoC;AAC7D,SAAK,IAAI,EAAE;AAEX,QAAI,cAAc,OAAO;AACzB,QAAI,gBAAgB;AACpB,QAAI,qBAAqB;AAEzB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAG3C,YAAM,SAAS,MAAM,QAAwB,0BAA0B;AAAA,QACrE,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,QAAQ;AAAA,MACV,CAAC;AAGD,WAAK,IAAI,wBAAwB;AACjC,YAAM,eAAe,MAAM,KAAK,iBAAiB,QAAQ,YAAY,KAAK;AAC1E,YAAM,aAAa,KAAK,gBAAgB,YAAY;AACpD,WAAK,IAAI,qBAAqB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG7D,UAAI,cAAc,WAAW;AAC3B,aAAK,IAAI,gBAAgB,aAAa,KAAK,QAAQ,CAAC,CAAC,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,cAAc;AAEjH,cAAMA,eAAc,MAAM,KAAK,iBAAiB,QAAQ,WAAW,KAAK;AACxE,cAAMC,aAAY,KAAK,gBAAgBD,YAAW;AAClD,aAAK,IAAI,sBAAsBC,aAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE7D,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,WAAAA;AAAA,UACA,UAAU,EAAE,QAAQ,oBAAoB;AAAA,UACxC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,WAAK,IAAI,sBAAsB;AAC/B,YAAM,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QAC3C,UAAU,EAAE;AAAA,QACZ,QAAQ,EAAE;AAAA,QACV,cAAc,EAAE;AAAA,QAChB,SAAS,EAAE;AAAA,MACb,EAAE;AAEF,YAAM,WAAW,MAAM,mBAAmB;AAAA,QACxC,SAAS,OAAO;AAAA,QAChB;AAAA,QACA,OAAO,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACxC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,WAAK,IAAI;AAAA,cAAiB,SAAS,iBAAiB,EAAE;AACtD,WAAK,IAAI,gBAAgB,SAAS,YAAY,MAAM,EAAE;AACtD,iBAAW,KAAK,SAAS,aAAa;AACpC,aAAK,IAAI,QAAQ,EAAE,OAAO,KAAK,EAAE,SAAS,EAAE;AAAA,MAC9C;AAGA,UAAI,CAAC,MAAM,MAAM;AACf,aAAK,IAAI,8EAA8E;AACvF,aAAK,IAAI,sEAAiE;AAAA,MAC5E;AAEA,UAAI,SAAS,oBAAoB,OAAO,SAAS;AAC/C,aAAK,IAAI,gCAAgC;AACzC,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,UAAU,EAAE,QAAQ,aAAa;AAAA,UACjC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,YAAM,YAAY,MAAM,OAAmB,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC3E,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,WAAK,IAAI,YAAY,UAAU,OAAO,EAAE;AAGxC,WAAK,IAAI,uBAAuB;AAChC,YAAM,cAAc,MAAM,KAAK,iBAAiB,WAAW,WAAW,KAAK;AAC3E,YAAM,YAAY,KAAK,gBAAgB,WAAW;AAClD,WAAK,IAAI,oBAAoB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG3D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,UAAU;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,eAAe,UAAU;AAAA,QACzB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,mBAAmB,SAAS;AAAA,QAC9B;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,sBAAc,UAAU;AACxB,6BAAqB;AAAA,MACvB,OAAO;AACL;AACA,YAAI,sBAAsB,GAAG;AAC3B,eAAK,IAAI,wDAAwD;AACjE;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAAA,IACX;AAEA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,kBAAkB,WAAW,kBAAkB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,IAAI;AAAA,EAC7F;AAAA,EAEA,MAAc,iBACZ,QACA,SACA,OACgH;AAChH,UAAM,UAAiH,CAAC;AAExH,eAAW,SAAS,SAAS;AAC3B,YAAM,aAAa,MAAM,YAAY;AAAA,QACnC,QAAQ,MAAM;AAAA,QACd,eAAe,OAAO;AAAA,QACtB,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,UAAI,WAAW,OAAO;AACpB,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAC1G;AAAA,MACF;AAEA,UAAI,MAAM,aAAa,SAAS,GAAG;AACjC,cAAM,UAAU,MAAM,gBAAgB;AAAA,UACpC,QAAQ,MAAM;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,cAAc,MAAM;AAAA,UACpB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,QAAuD,CAAC;AAAA,MACvJ,OAAO;AACL,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAAA,MAC5G;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,SAA8D;AACpF,UAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AACtE,QAAI,OAAO,WAAW,EAAG,QAAO;AAChC,UAAM,QAAQ,OAAO,IAAI,CAAC,MAAO,EAAE,QAAgB,SAAS,aAAa,CAAC;AAC1E,WAAO,MAAM,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,MAAM;AAAA,EAClE;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAwB5B,IAAqB,sBAArB,MAAqB,6BAA4B,YAAY;AAAA,EAC3D,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,MAAM,MAAM,QAAQ,EAAE,aAAa,4DAA4D,SAAS,MAAM,CAAC;AAAA,IAC/G,kBAAkB,MAAM,QAAQ,EAAE,aAAa,mCAAmC,SAAS,EAAE,CAAC;AAAA,IAC9F,WAAW,MAAM,OAAO,EAAE,aAAa,kDAAkD,SAAS,OAAO,CAAC;AAAA,IAC1G,OAAO,MAAM,OAAO,EAAE,aAAa,2BAA2B,CAAC;AAAA,IAC/D,kBAAkB,MAAM,OAAO,EAAE,aAAa,iCAAiC,CAAC;AAAA,IAChF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,+BAA+B,SAAS,EAAE,CAAC;AAAA,IACzF,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,oBAAmB;AAC5D,SAAK,YAAY;AAEjB,QAAI,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC1E,UAAM,WAAW,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAEhF,QAAI,SAAS,WAAW,GAAG;AACzB,WAAK,MAAM,mEAAmE;AAAA,IAChF;AAGA,UAAM,WAAW,CAAC,GAAG,QAAQ,EAAE,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAC7D,UAAM,WAAW,KAAK,KAAK,SAAS,SAAS,GAAG;AAChD,UAAM,aAAa,SAAS,MAAM,GAAG,QAAQ;AAC7C,UAAM,YAAY,SAAS,MAAM,QAAQ;AAEzC,SAAK,IAAI,eAAe,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AACzD,SAAK,IAAI,UAAU,WAAW,MAAM,WAAW,UAAU,MAAM,OAAO;AACtE,UAAM,YAAY,WAAW,MAAM,SAAS;AAC5C,SAAK,IAAI,eAAe,MAAM,gBAAgB,CAAC,2BAA2B,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AACxG,QAAI,MAAM,KAAM,MAAK,IAAI,oCAAoC;AAC7D,SAAK,IAAI,EAAE;AAEX,QAAI,cAAc,OAAO;AACzB,QAAI,gBAAgB;AACpB,QAAI,qBAAqB;AAEzB,aAAS,YAAY,GAAG,aAAa,MAAM,gBAAgB,GAAG,aAAa;AACzE,WAAK,IAAI;AAAA,gBAAmB,SAAS,MAAM;AAG3C,YAAM,SAAS,MAAM,QAAwB,0BAA0B;AAAA,QACrE,cAAc,OAAO;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,QAAQ;AAAA,MACV,CAAC;AAGD,WAAK,IAAI,wBAAwB;AACjC,YAAM,eAAe,MAAM,KAAK,iBAAiB,QAAQ,YAAY,KAAK;AAC1E,YAAM,aAAa,KAAK,gBAAgB,YAAY;AACpD,WAAK,IAAI,qBAAqB,aAAa,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG7D,UAAI,cAAc,WAAW;AAC3B,aAAK,IAAI,gBAAgB,aAAa,KAAK,QAAQ,CAAC,CAAC,mBAAmB,YAAY,KAAK,QAAQ,CAAC,CAAC,cAAc;AAEjH,cAAMA,eAAc,MAAM,KAAK,iBAAiB,QAAQ,WAAW,KAAK;AACxE,cAAMC,aAAY,KAAK,gBAAgBD,YAAW;AAClD,aAAK,IAAI,sBAAsBC,aAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAE7D,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,WAAAA;AAAA,UACA,UAAU,EAAE,QAAQ,oBAAoB;AAAA,UACxC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,WAAK,IAAI,sBAAsB;AAC/B,YAAM,cAAc,aAAa,IAAI,CAAC,OAAO;AAAA,QAC3C,UAAU,EAAE;AAAA,QACZ,QAAQ,EAAE;AAAA,QACV,cAAc,EAAE;AAAA,QAChB,SAAS,EAAE;AAAA,MACb,EAAE;AAEF,YAAM,WAAW,MAAM,mBAAmB;AAAA,QACxC,SAAS,OAAO;AAAA,QAChB;AAAA,QACA,OAAO,MAAM,gBAAgB,KAAK,MAAM;AAAA,QACxC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,WAAK,IAAI;AAAA,cAAiB,SAAS,iBAAiB,EAAE;AACtD,WAAK,IAAI,gBAAgB,SAAS,YAAY,MAAM,EAAE;AACtD,iBAAW,KAAK,SAAS,aAAa;AACpC,aAAK,IAAI,QAAQ,EAAE,OAAO,KAAK,EAAE,SAAS,EAAE;AAAA,MAC9C;AAGA,UAAI,CAAC,MAAM,MAAM;AACf,aAAK,IAAI,8EAA8E;AACvF,aAAK,IAAI,sEAAiE;AAAA,MAC5E;AAEA,UAAI,SAAS,oBAAoB,OAAO,SAAS;AAC/C,aAAK,IAAI,gCAAgC;AACzC,cAAM,QAAQ,0BAA0B;AAAA,UACtC,cAAc,OAAO;AAAA,UACrB,MAAM;AAAA,UACN;AAAA,UACA,cAAc,OAAO;AAAA,UACrB;AAAA,UACA,UAAU,EAAE,QAAQ,aAAa;AAAA,UACjC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAGA,YAAM,YAAY,MAAM,OAAmB,qBAAqB,OAAO,EAAE,IAAI;AAAA,QAC3E,SAAS,SAAS;AAAA,MACpB,CAAC;AACD,WAAK,IAAI,YAAY,UAAU,OAAO,EAAE;AAGxC,WAAK,IAAI,uBAAuB;AAChC,YAAM,cAAc,MAAM,KAAK,iBAAiB,WAAW,WAAW,KAAK;AAC3E,YAAM,YAAY,KAAK,gBAAgB,WAAW;AAClD,WAAK,IAAI,oBAAoB,YAAY,KAAK,QAAQ,CAAC,CAAC,GAAG;AAG3D,YAAM,QAAQ,0BAA0B;AAAA,QACtC,cAAc,UAAU;AAAA,QACxB,MAAM;AAAA,QACN;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,eAAe,UAAU;AAAA,QACzB;AAAA,QACA;AAAA,QACA,UAAU;AAAA,UACR,aAAa,SAAS;AAAA,UACtB,mBAAmB,SAAS;AAAA,QAC9B;AAAA,QACA,QAAQ;AAAA,MACV,CAAC;AAGD,UAAI,YAAY,eAAe;AAC7B,wBAAgB;AAChB,sBAAc,UAAU;AACxB,6BAAqB;AAAA,MACvB,OAAO;AACL;AACA,YAAI,sBAAsB,GAAG;AAC3B,eAAK,IAAI,wDAAwD;AACjE;AAAA,QACF;AAAA,MACF;AAEA,eAAS;AAAA,IACX;AAEA,SAAK,IAAI;AAAA,8BAAiC;AAC1C,SAAK,IAAI,kBAAkB,WAAW,kBAAkB,gBAAgB,KAAK,QAAQ,CAAC,CAAC,IAAI;AAAA,EAC7F;AAAA,EAEA,MAAc,iBACZ,QACA,SACA,OACgH;AAChH,UAAM,UAAiH,CAAC;AAExH,eAAW,SAAS,SAAS;AAC3B,YAAM,aAAa,MAAM,YAAY;AAAA,QACnC,QAAQ,MAAM;AAAA,QACd,eAAe,OAAO;AAAA,QACtB,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,MACf,CAAC;AAED,UAAI,WAAW,OAAO;AACpB,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAC1G;AAAA,MACF;AAEA,UAAI,MAAM,aAAa,SAAS,GAAG;AACjC,cAAM,UAAU,MAAM,gBAAgB;AAAA,UACpC,QAAQ,MAAM;AAAA,UACd,QAAQ,WAAW;AAAA,UACnB,cAAc,MAAM;AAAA,UACpB,OAAO,MAAM;AAAA,UACb,SAAS,MAAM;AAAA,QACjB,CAAC;AACD,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,QAAuD,CAAC;AAAA,MACvJ,OAAO;AACL,gBAAQ,KAAK,EAAE,UAAU,MAAM,MAAM,QAAQ,MAAM,QAAQ,cAAc,MAAM,cAAc,SAAS,CAAC,EAAE,CAAC;AAAA,MAC5G;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,gBAAgB,SAA8D;AACpF,UAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AACtE,QAAI,OAAO,WAAW,EAAG,QAAO;AAChC,UAAM,QAAQ,OAAO,IAAI,CAAC,MAAO,EAAE,QAAgB,SAAS,aAAa,CAAC;AAC1E,WAAO,MAAM,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,MAAM;AAAA,EAClE;AACF;",
   "names": ["testResults", "testScore"]
 }

package/dist/commands/evals/run.js

@@ -6,14 +6,15 @@ import {
 } from "../../chunk-QN34TGD3.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
 import {
   apiGet,
   apiPost
-} from "../../chunk-ZRLYHBPN.js";
+} from "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 

package/dist/commands/evals/run.js.map

@@ -2,6 +2,6 @@
   "version": 3,
   "sources": ["../../../src/commands/evals/run.ts"],
   "sourcesContent": ["import { Args, Flags } from '@oclif/core';\nimport { BaseCommand } from '../../base-command.js';\nimport { apiGet, apiPost } from '../../lib/api-client.js';\nimport { executeEval } from '../../lib/eval-executor.js';\nimport { gradeEvalOutput } from '../../lib/eval-grader.js';\n\ninterface EvalTarget {\n  id: string;\n  name: string;\n  type: string;\n  content: string;\n  version: number;\n}\n\ninterface Eval {\n  id: string;\n  name: string;\n  prompt: string;\n  expectedOutput: string;\n  expectations: string[];\n}\n\ninterface EvalRunResult {\n  evalId: string;\n  evalTargetId: string;\n  evalTargetVersion: number;\n  configuration: string;\n  runNumber: number;\n  iteration: number;\n  status: string;\n  outputs: Record<string, unknown>;\n  timing: Record<string, unknown>;\n  grading: Record<string, unknown>;\n}\n\nexport default class EvalRunCommand extends BaseCommand {\n  static override description = 'Run evals against a target, optionally grade results, and post to API';\n\n  static override examples = [\n    '<%= config.bin %> evals run <target-id>',\n    '<%= config.bin %> evals run <target-id> --parallel 3 --runs-per-eval 2',\n    '<%= config.bin %> evals run <target-id> --skip-baseline --skip-grading',\n  ];\n\n  static override args = {\n    targetId: Args.string({ description: 'Eval target ID', required: true }),\n  };\n\n  static override flags = {\n    parallel: Flags.integer({ description: 'Number of parallel eval executions', default: 1 }),\n    'runs-per-eval': Flags.integer({ description: 'Number of runs per eval for variance analysis', default: 1 }),\n    iteration: Flags.integer({ description: 'Iteration number for tracking', default: 1 }),\n    'skip-baseline': Flags.boolean({ description: 'Skip baseline (without_target) runs', default: false }),\n    'skip-grading': Flags.boolean({ description: 'Skip LLM grading step', default: false }),\n    model: Flags.string({ description: 'Model for eval execution (default: user configured)' }),\n    'grader-model': Flags.string({ description: 'Model for grading (default: same as execution model)' }),\n    timeout: Flags.integer({ description: 'Timeout per eval in ms', default: 120_000 }),\n  };\n\n  async run(): Promise<void> {\n    const { args, flags } = await this.parse(EvalRunCommand);\n    this.requireAuth();\n\n    // Fetch target and its evals\n    const target = await apiGet<EvalTarget>(`/api/eval-targets/${args.targetId}`);\n    const evals = await apiGet<Eval[]>(`/api/eval-targets/${args.targetId}/evals`);\n\n    if (evals.length === 0) {\n      this.error(`No evals found for target \"${target.name}\". Create some with \\`rulemetric evals create\\`.`);\n    }\n\n    this.log(`Running ${evals.length} eval(s) for \"${target.name}\" v${target.version}`);\n    this.log(`Config: ${flags['runs-per-eval']} run(s) per eval, parallel=${flags.parallel}, iteration=${flags.iteration}`);\n    if (flags['skip-baseline']) this.log('Skipping baseline runs');\n    if (flags['skip-grading']) this.log('Skipping grading');\n    this.log('');\n\n    const configurations: Array<{ name: string; content?: string }> = [\n      { name: 'with_target', content: target.content },\n    ];\n    if (!flags['skip-baseline']) {\n      configurations.push({ name: 'without_target' });\n    }\n\n    // Build the work queue\n    interface WorkItem {\n      eval_: Eval;\n      configuration: string;\n      targetContent?: string;\n      runNumber: number;\n    }\n    const workQueue: WorkItem[] = [];\n\n    for (const eval_ of evals) {\n      for (const config of configurations) {\n        for (let run = 1; run <= flags['runs-per-eval']; run++) {\n          workQueue.push({\n            eval_: eval_,\n            configuration: config.name,\n            targetContent: config.content,\n            runNumber: run,\n          });\n        }\n      }\n    }\n\n    // Execute with concurrency control\n    const results: EvalRunResult[] = [];\n    const concurrency = flags.parallel;\n\n    for (let i = 0; i < workQueue.length; i += concurrency) {\n      const batch = workQueue.slice(i, i + concurrency);\n      const batchResults = await Promise.all(\n        batch.map((item) => this.executeAndGrade(item, target, flags)),\n      );\n      results.push(...batchResults);\n    }\n\n    // Post results to API\n    this.log('');\n    this.log('Posting results to API...');\n\n    for (const result of results) {\n      await apiPost('/api/eval-runs', result);\n    }\n\n    // Summary\n    this.log('');\n    this.log('=== Results Summary ===');\n\n    for (const config of configurations) {\n      const configResults = results.filter((r) => r.configuration === config.name);\n      const completedRuns = configResults.filter((r) => r.status === 'completed');\n      const gradedRuns = completedRuns.filter((r) => Object.keys(r.grading).length > 0);\n\n      this.log(`\\n${config.name}:`);\n      this.log(`  Runs: ${completedRuns.length}/${configResults.length} completed`);\n\n      if (gradedRuns.length > 0) {\n        const passRates = gradedRuns\n          .map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0);\n        const avgPassRate = passRates.reduce((a: number, b: number) => a + b, 0) / passRates.length;\n        this.log(`  Avg pass_rate: ${(avgPassRate * 100).toFixed(1)}%`);\n      }\n\n      for (const r of configResults) {\n        const evalName = evals.find((e) => e.id === r.evalId)?.name ?? r.evalId;\n        const passRate = (r.grading as Record<string, any>).summary?.pass_rate;\n        const passRateStr = passRate !== undefined ? ` pass_rate=${(passRate * 100).toFixed(0)}%` : '';\n        const timeStr = (r.timing as Record<string, any>).duration_ms ? ` ${((r.timing as Record<string, any>).duration_ms / 1000).toFixed(1)}s` : '';\n        this.log(`  [${r.status}] ${evalName} run#${r.runNumber}${passRateStr}${timeStr}`);\n      }\n    }\n\n    if (!flags['skip-baseline'] && results.length > 0) {\n      const withTarget = results.filter((r) => r.configuration === 'with_target' && Object.keys(r.grading).length > 0);\n      const withoutTarget = results.filter((r) => r.configuration === 'without_target' && Object.keys(r.grading).length > 0);\n\n      if (withTarget.length > 0 && withoutTarget.length > 0) {\n        const avgWith = withTarget.map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0).reduce((a: number, b: number) => a + b, 0) / withTarget.length;\n        const avgWithout = withoutTarget.map((r) => (r.grading as Record<string, any>).summary?.pass_rate ?? 0).reduce((a: number, b: number) => a + b, 0) / withoutTarget.length;\n        const delta = avgWith - avgWithout;\n        this.log(`\\nDelta: ${delta > 0 ? '+' : ''}${(delta * 100).toFixed(1)}% (with_target vs without_target)`);\n      }\n    }\n  }\n\n  private async executeAndGrade(\n    item: { eval_: Eval; configuration: string; targetContent?: string; runNumber: number },\n    target: EvalTarget,\n    flags: { 'skip-grading': boolean; model?: string; 'grader-model'?: string; timeout: number; iteration: number },\n  ): Promise<EvalRunResult> {\n    const { eval_, configuration, targetContent, runNumber } = item;\n\n    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 executing...`);\n\n    // Execute the eval\n    const execResult = await executeEval({\n      prompt: eval_.prompt,\n      targetContent,\n      timeout: flags.timeout,\n      model: flags.model,\n    });\n\n    if (execResult.error) {\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 FAILED: ${execResult.error}`);\n      return {\n        evalId: eval_.id,\n        evalTargetId: target.id,\n        evalTargetVersion: target.version,\n        configuration,\n        runNumber,\n        iteration: flags.iteration,\n        status: 'failed',\n        outputs: { error: execResult.error },\n        timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },\n        grading: {},\n      };\n    }\n\n    this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 completed in ${(execResult.durationMs / 1000).toFixed(1)}s`);\n\n    // Grade if not skipped and expectations exist\n    let grading: Record<string, unknown> = {};\n\n    if (!flags['skip-grading'] && eval_.expectations.length > 0) {\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading...`);\n\n      const gradingResult = await gradeEvalOutput({\n        prompt: eval_.prompt,\n        output: execResult.output,\n        expectations: eval_.expectations,\n        model: flags['grader-model'] ?? flags.model,\n        timeout: flags.timeout,\n      });\n\n      grading = gradingResult as unknown as Record<string, unknown>;\n      const passRate = gradingResult.summary.pass_rate;\n      this.log(`[${configuration}] ${eval_.name} run#${runNumber} \u2014 grading done: pass_rate=${(passRate * 100).toFixed(0)}%`);\n    }\n\n    return {\n      evalId: eval_.id,\n      evalTargetId: target.id,\n      evalTargetVersion: target.version,\n      configuration,\n      runNumber,\n      iteration: flags.iteration,\n      status: 'completed',\n      outputs: { transcript: execResult.output },\n      timing: {\n        duration_ms: execResult.durationMs,\n        total_tokens: execResult.totalTokens,\n        total_duration_seconds: execResult.durationMs / 1000,\n      },\n      grading,\n    };\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmC5B,IAAqB,iBAArB,MAAqB,wBAAuB,YAAY;AAAA,EACtD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,iDAAiD,SAAS,EAAE,CAAC;AAAA,IAC3G,WAAW,MAAM,QAAQ,EAAE,aAAa,iCAAiC,SAAS,EAAE,CAAC;AAAA,IACrF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,uCAAuC,SAAS,MAAM,CAAC;AAAA,IACrG,gBAAgB,MAAM,QAAQ,EAAE,aAAa,yBAAyB,SAAS,MAAM,CAAC;AAAA,IACtF,OAAO,MAAM,OAAO,EAAE,aAAa,sDAAsD,CAAC;AAAA,IAC1F,gBAAgB,MAAM,OAAO,EAAE,aAAa,uDAAuD,CAAC;AAAA,IACpG,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,eAAc;AACvD,SAAK,YAAY;AAGjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC5E,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAE7E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,8BAA8B,OAAO,IAAI,kDAAkD;AAAA,IACxG;AAEA,SAAK,IAAI,WAAW,MAAM,MAAM,iBAAiB,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AAClF,SAAK,IAAI,WAAW,MAAM,eAAe,CAAC,8BAA8B,MAAM,QAAQ,eAAe,MAAM,SAAS,EAAE;AACtH,QAAI,MAAM,eAAe,EAAG,MAAK,IAAI,wBAAwB;AAC7D,QAAI,MAAM,cAAc,EAAG,MAAK,IAAI,kBAAkB;AACtD,SAAK,IAAI,EAAE;AAEX,UAAM,iBAA4D;AAAA,MAChE,EAAE,MAAM,eAAe,SAAS,OAAO,QAAQ;AAAA,IACjD;AACA,QAAI,CAAC,MAAM,eAAe,GAAG;AAC3B,qBAAe,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAAA,IAChD;AASA,UAAM,YAAwB,CAAC;AAE/B,eAAW,SAAS,OAAO;AACzB,iBAAW,UAAU,gBAAgB;AACnC,iBAAS,MAAM,GAAG,OAAO,MAAM,eAAe,GAAG,OAAO;AACtD,oBAAU,KAAK;AAAA,YACb;AAAA,YACA,eAAe,OAAO;AAAA,YACtB,eAAe,OAAO;AAAA,YACtB,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAA2B,CAAC;AAClC,UAAM,cAAc,MAAM;AAE1B,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,aAAa;AACtD,YAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,YAAM,eAAe,MAAM,QAAQ;AAAA,QACjC,MAAM,IAAI,CAAC,SAAS,KAAK,gBAAgB,MAAM,QAAQ,KAAK,CAAC;AAAA,MAC/D;AACA,cAAQ,KAAK,GAAG,YAAY;AAAA,IAC9B;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,2BAA2B;AAEpC,eAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,kBAAkB,MAAM;AAAA,IACxC;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,yBAAyB;AAElC,eAAW,UAAU,gBAAgB;AACnC,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,OAAO,IAAI;AAC3E,YAAM,gBAAgB,cAAc,OAAO,CAAC,MAAM,EAAE,WAAW,WAAW;AAC1E,YAAM,aAAa,cAAc,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAEhF,WAAK,IAAI;AAAA,EAAK,OAAO,IAAI,GAAG;AAC5B,WAAK,IAAI,WAAW,cAAc,MAAM,IAAI,cAAc,MAAM,YAAY;AAE5E,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,YAAY,WACf,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC;AACxE,cAAM,cAAc,UAAU,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,UAAU;AACrF,aAAK,IAAI,qBAAqB,cAAc,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,MAChE;AAEA,iBAAW,KAAK,eAAe;AAC7B,cAAM,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;AACjE,cAAM,WAAY,EAAE,QAAgC,SAAS;AAC7D,cAAM,cAAc,aAAa,SAAY,eAAe,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM;AAC5F,cAAM,UAAW,EAAE,OAA+B,cAAc,KAAM,EAAE,OAA+B,cAAc,KAAM,QAAQ,CAAC,CAAC,MAAM;AAC3I,aAAK,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,QAAQ,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,EAAE;AAAA,MACnF;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,eAAe,KAAK,QAAQ,SAAS,GAAG;AACjD,YAAM,aAAa,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,iBAAiB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAC/G,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,oBAAoB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAErH,UAAI,WAAW,SAAS,KAAK,cAAc,SAAS,GAAG;AACrD,cAAM,UAAU,WAAW,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,WAAW;AAC1J,cAAM,aAAa,cAAc,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,cAAc;AACnK,cAAM,QAAQ,UAAU;AACxB,aAAK,IAAI;AAAA,SAAY,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,mCAAmC;AAAA,MACzG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,gBACZ,MACA,QACA,OACwB;AACxB,UAAM,EAAE,OAAO,eAAe,eAAe,UAAU,IAAI;AAE3D,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,sBAAiB;AAG3E,UAAM,aAAa,MAAM,YAAY;AAAA,MACnC,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI,WAAW,OAAO;AACpB,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,mBAAc,WAAW,KAAK,EAAE;AAC1F,aAAO;AAAA,QACL,QAAQ,MAAM;AAAA,QACd,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B;AAAA,QACA;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,QAAQ;AAAA,QACR,SAAS,EAAE,OAAO,WAAW,MAAM;AAAA,QACnC,QAAQ,EAAE,aAAa,WAAW,YAAY,cAAc,WAAW,YAAY;AAAA,QACnF,SAAS,CAAC;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,yBAAoB,WAAW,aAAa,KAAM,QAAQ,CAAC,CAAC,GAAG;AAGzH,QAAI,UAAmC,CAAC;AAExC,QAAI,CAAC,MAAM,cAAc,KAAK,MAAM,aAAa,SAAS,GAAG;AAC3D,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oBAAe;AAEzE,YAAM,gBAAgB,MAAM,gBAAgB;AAAA,QAC1C,QAAQ,MAAM;AAAA,QACd,QAAQ,WAAW;AAAA,QACnB,cAAc,MAAM;AAAA,QACpB,OAAO,MAAM,cAAc,KAAK,MAAM;AAAA,QACtC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,gBAAU;AACV,YAAM,WAAW,cAAc,QAAQ;AACvC,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oCAA+B,WAAW,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,IACxH;AAEA,WAAO;AAAA,MACL,QAAQ,MAAM;AAAA,MACd,cAAc,OAAO;AAAA,MACrB,mBAAmB,OAAO;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,QAAQ;AAAA,MACR,SAAS,EAAE,YAAY,WAAW,OAAO;AAAA,MACzC,QAAQ;AAAA,QACN,aAAa,WAAW;AAAA,QACxB,cAAc,WAAW;AAAA,QACzB,wBAAwB,WAAW,aAAa;AAAA,MAClD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;",
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,MAAM,aAAa;AAmC5B,IAAqB,iBAArB,MAAqB,wBAAuB,YAAY;AAAA,EACtD,OAAgB,cAAc;AAAA,EAE9B,OAAgB,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,EAEA,OAAgB,OAAO;AAAA,IACrB,UAAU,KAAK,OAAO,EAAE,aAAa,kBAAkB,UAAU,KAAK,CAAC;AAAA,EACzE;AAAA,EAEA,OAAgB,QAAQ;AAAA,IACtB,UAAU,MAAM,QAAQ,EAAE,aAAa,sCAAsC,SAAS,EAAE,CAAC;AAAA,IACzF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,iDAAiD,SAAS,EAAE,CAAC;AAAA,IAC3G,WAAW,MAAM,QAAQ,EAAE,aAAa,iCAAiC,SAAS,EAAE,CAAC;AAAA,IACrF,iBAAiB,MAAM,QAAQ,EAAE,aAAa,uCAAuC,SAAS,MAAM,CAAC;AAAA,IACrG,gBAAgB,MAAM,QAAQ,EAAE,aAAa,yBAAyB,SAAS,MAAM,CAAC;AAAA,IACtF,OAAO,MAAM,OAAO,EAAE,aAAa,sDAAsD,CAAC;AAAA,IAC1F,gBAAgB,MAAM,OAAO,EAAE,aAAa,uDAAuD,CAAC;AAAA,IACpG,SAAS,MAAM,QAAQ,EAAE,aAAa,0BAA0B,SAAS,KAAQ,CAAC;AAAA,EACpF;AAAA,EAEA,MAAM,MAAqB;AACzB,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,KAAK,MAAM,eAAc;AACvD,SAAK,YAAY;AAGjB,UAAM,SAAS,MAAM,OAAmB,qBAAqB,KAAK,QAAQ,EAAE;AAC5E,UAAM,QAAQ,MAAM,OAAe,qBAAqB,KAAK,QAAQ,QAAQ;AAE7E,QAAI,MAAM,WAAW,GAAG;AACtB,WAAK,MAAM,8BAA8B,OAAO,IAAI,kDAAkD;AAAA,IACxG;AAEA,SAAK,IAAI,WAAW,MAAM,MAAM,iBAAiB,OAAO,IAAI,MAAM,OAAO,OAAO,EAAE;AAClF,SAAK,IAAI,WAAW,MAAM,eAAe,CAAC,8BAA8B,MAAM,QAAQ,eAAe,MAAM,SAAS,EAAE;AACtH,QAAI,MAAM,eAAe,EAAG,MAAK,IAAI,wBAAwB;AAC7D,QAAI,MAAM,cAAc,EAAG,MAAK,IAAI,kBAAkB;AACtD,SAAK,IAAI,EAAE;AAEX,UAAM,iBAA4D;AAAA,MAChE,EAAE,MAAM,eAAe,SAAS,OAAO,QAAQ;AAAA,IACjD;AACA,QAAI,CAAC,MAAM,eAAe,GAAG;AAC3B,qBAAe,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAAA,IAChD;AASA,UAAM,YAAwB,CAAC;AAE/B,eAAW,SAAS,OAAO;AACzB,iBAAW,UAAU,gBAAgB;AACnC,iBAAS,MAAM,GAAG,OAAO,MAAM,eAAe,GAAG,OAAO;AACtD,oBAAU,KAAK;AAAA,YACb;AAAA,YACA,eAAe,OAAO;AAAA,YACtB,eAAe,OAAO;AAAA,YACtB,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAGA,UAAM,UAA2B,CAAC;AAClC,UAAM,cAAc,MAAM;AAE1B,aAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,aAAa;AACtD,YAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,YAAM,eAAe,MAAM,QAAQ;AAAA,QACjC,MAAM,IAAI,CAAC,SAAS,KAAK,gBAAgB,MAAM,QAAQ,KAAK,CAAC;AAAA,MAC/D;AACA,cAAQ,KAAK,GAAG,YAAY;AAAA,IAC9B;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,2BAA2B;AAEpC,eAAW,UAAU,SAAS;AAC5B,YAAM,QAAQ,kBAAkB,MAAM;AAAA,IACxC;AAGA,SAAK,IAAI,EAAE;AACX,SAAK,IAAI,yBAAyB;AAElC,eAAW,UAAU,gBAAgB;AACnC,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,OAAO,IAAI;AAC3E,YAAM,gBAAgB,cAAc,OAAO,CAAC,MAAM,EAAE,WAAW,WAAW;AAC1E,YAAM,aAAa,cAAc,OAAO,CAAC,MAAM,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAEhF,WAAK,IAAI;AAAA,EAAK,OAAO,IAAI,GAAG;AAC5B,WAAK,IAAI,WAAW,cAAc,MAAM,IAAI,cAAc,MAAM,YAAY;AAE5E,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,YAAY,WACf,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC;AACxE,cAAM,cAAc,UAAU,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,UAAU;AACrF,aAAK,IAAI,qBAAqB,cAAc,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,MAChE;AAEA,iBAAW,KAAK,eAAe;AAC7B,cAAM,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;AACjE,cAAM,WAAY,EAAE,QAAgC,SAAS;AAC7D,cAAM,cAAc,aAAa,SAAY,eAAe,WAAW,KAAK,QAAQ,CAAC,CAAC,MAAM;AAC5F,cAAM,UAAW,EAAE,OAA+B,cAAc,KAAM,EAAE,OAA+B,cAAc,KAAM,QAAQ,CAAC,CAAC,MAAM;AAC3I,aAAK,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,QAAQ,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,EAAE;AAAA,MACnF;AAAA,IACF;AAEA,QAAI,CAAC,MAAM,eAAe,KAAK,QAAQ,SAAS,GAAG;AACjD,YAAM,aAAa,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,iBAAiB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAC/G,YAAM,gBAAgB,QAAQ,OAAO,CAAC,MAAM,EAAE,kBAAkB,oBAAoB,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;AAErH,UAAI,WAAW,SAAS,KAAK,cAAc,SAAS,GAAG;AACrD,cAAM,UAAU,WAAW,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,WAAW;AAC1J,cAAM,aAAa,cAAc,IAAI,CAAC,MAAO,EAAE,QAAgC,SAAS,aAAa,CAAC,EAAE,OAAO,CAAC,GAAW,MAAc,IAAI,GAAG,CAAC,IAAI,cAAc;AACnK,cAAM,QAAQ,UAAU;AACxB,aAAK,IAAI;AAAA,SAAY,QAAQ,IAAI,MAAM,EAAE,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC,mCAAmC;AAAA,MACzG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAc,gBACZ,MACA,QACA,OACwB;AACxB,UAAM,EAAE,OAAO,eAAe,eAAe,UAAU,IAAI;AAE3D,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,sBAAiB;AAG3E,UAAM,aAAa,MAAM,YAAY;AAAA,MACnC,QAAQ,MAAM;AAAA,MACd;AAAA,MACA,SAAS,MAAM;AAAA,MACf,OAAO,MAAM;AAAA,IACf,CAAC;AAED,QAAI,WAAW,OAAO;AACpB,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,mBAAc,WAAW,KAAK,EAAE;AAC1F,aAAO;AAAA,QACL,QAAQ,MAAM;AAAA,QACd,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B;AAAA,QACA;AAAA,QACA,WAAW,MAAM;AAAA,QACjB,QAAQ;AAAA,QACR,SAAS,EAAE,OAAO,WAAW,MAAM;AAAA,QACnC,QAAQ,EAAE,aAAa,WAAW,YAAY,cAAc,WAAW,YAAY;AAAA,QACnF,SAAS,CAAC;AAAA,MACZ;AAAA,IACF;AAEA,SAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,yBAAoB,WAAW,aAAa,KAAM,QAAQ,CAAC,CAAC,GAAG;AAGzH,QAAI,UAAmC,CAAC;AAExC,QAAI,CAAC,MAAM,cAAc,KAAK,MAAM,aAAa,SAAS,GAAG;AAC3D,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oBAAe;AAEzE,YAAM,gBAAgB,MAAM,gBAAgB;AAAA,QAC1C,QAAQ,MAAM;AAAA,QACd,QAAQ,WAAW;AAAA,QACnB,cAAc,MAAM;AAAA,QACpB,OAAO,MAAM,cAAc,KAAK,MAAM;AAAA,QACtC,SAAS,MAAM;AAAA,MACjB,CAAC;AAED,gBAAU;AACV,YAAM,WAAW,cAAc,QAAQ;AACvC,WAAK,IAAI,IAAI,aAAa,KAAK,MAAM,IAAI,QAAQ,SAAS,oCAA+B,WAAW,KAAK,QAAQ,CAAC,CAAC,GAAG;AAAA,IACxH;AAEA,WAAO;AAAA,MACL,QAAQ,MAAM;AAAA,MACd,cAAc,OAAO;AAAA,MACrB,mBAAmB,OAAO;AAAA,MAC1B;AAAA,MACA;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,QAAQ;AAAA,MACR,SAAS,EAAE,YAAY,WAAW,OAAO;AAAA,MACzC,QAAQ;AAAA,QACN,aAAa,WAAW;AAAA,QACxB,cAAc,WAAW;AAAA,QACzB,wBAAwB,WAAW,aAAa;AAAA,MAClD;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/commands/gateway/ensure.js

@@ -1,24 +1,50 @@
 import {
+  GATEWAY_PORT,
   isGatewayRunning,
+  isRulemetricGatewayListening,
   spawnGateway
-} from "../../chunk-DAJNO4OI.js";
+} from "../../chunk-4LYCDS2N.js";
+import "../../chunk-7LVVKQMZ.js";
+import {
+  MITMPROXY_COMMAND_PATTERN,
+  isPidLiveAndMatching
+} from "../../chunk-5EDMGK3W.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
-import "../../chunk-SLL3OBKN.js";
+import "../../chunk-GQNJO2P2.js";
+import "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZRLYHBPN.js";
+import "../../chunk-ZUJBM6HE.js";
 import "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 
 // src/commands/gateway/ensure.ts
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+var CONFIG_DIR = join(homedir(), ".config", "rulemetric");
+var ADDON_VERSION_FILE = join(CONFIG_DIR, "proxy-addon.version");
+var PROXY_PID_FILE = join(CONFIG_DIR, "proxy.pid");
+var DRIFT_RESTART_MARKER = join(CONFIG_DIR, "proxy-drift-restart.at");
+var DRIFT_RESTART_COOLDOWN_MS = 10 * 6e4;
 var GatewayEnsure = class _GatewayEnsure extends BaseCommand {
   static description = "Ensure the gateway proxy is running (called by hooks, not for direct use)";
   static hidden = true;
   async run() {
     await this.parse(_GatewayEnsure);
-    if (isGatewayRunning()) {
+    this.maybeRestartStaleProxy();
+    const identity = await isRulemetricGatewayListening(GATEWAY_PORT);
+    if (identity.status === "ours-live") {
+      return;
+    }
+    if (identity.status === "foreign") {
+      if (isGatewayRunning()) return;
+      console.error(
+        `[RuleMetric] A non-RuleMetric process holds :${GATEWAY_PORT} \u2014 cannot start gateway. Run \`rulemetric diagnostics\` if Claude Code HTTPS requests fail.`
+      );
       return;
     }
     try {
@@ -27,6 +53,33 @@ var GatewayEnsure = class _GatewayEnsure extends BaseCommand {
       console.error(`[RuleMetric] Could not start gateway: ${err.message}`);
     }
   }
+  maybeRestartStaleProxy() {
+    try {
+      if (!existsSync(ADDON_VERSION_FILE)) return;
+      const runningVersion = readFileSync(ADDON_VERSION_FILE, "utf-8").trim();
+      if (!runningVersion || runningVersion === this.config.version) return;
+      const pidRaw = readFileSync(PROXY_PID_FILE, "utf-8").trim();
+      const pid = Number(pidRaw);
+      if (!Number.isFinite(pid) || !isPidLiveAndMatching(pid, MITMPROXY_COMMAND_PATTERN)) {
+        return;
+      }
+      try {
+        const age = Date.now() - statSync(DRIFT_RESTART_MARKER).mtimeMs;
+        if (age < DRIFT_RESTART_COOLDOWN_MS) return;
+      } catch {
+      }
+      writeFileSync(DRIFT_RESTART_MARKER, (/* @__PURE__ */ new Date()).toISOString());
+      const child = spawn(process.execPath, [process.argv[1], "proxy", "restart"], {
+        detached: true,
+        stdio: "ignore"
+      });
+      child.unref();
+      console.error(
+        `[RuleMetric] Capture proxy is running CLI ${runningVersion} but ${this.config.version} is installed \u2014 restarting it in the background.`
+      );
+    } catch {
+    }
+  }
 };
 export {
   GatewayEnsure as default

package/dist/commands/gateway/ensure.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../src/commands/gateway/ensure.ts"],
-  "sourcesContent": ["import { BaseCommand } from '../../base-command.js';\nimport { isGatewayRunning, spawnGateway } from '../../lib/gateway-lifecycle.js';\n\nexport default class GatewayEnsure extends BaseCommand {\n  static override description = 'Ensure the gateway proxy is running (called by hooks, not for direct use)';\n  static override hidden = true;\n\n  async run(): Promise<void> {\n    await this.parse(GatewayEnsure);\n\n    if (isGatewayRunning()) {\n      return; // Already running, silent success\n    }\n\n    try {\n      spawnGateway(this.config.version);\n    } catch (err) {\n      // Don't crash the hook \u2014 just warn\n      console.error(`[RuleMetric] Could not start gateway: ${(err as Error).message}`);\n    }\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;AAGA,IAAqB,gBAArB,MAAqB,uBAAsB,YAAY;AAAA,EACrD,OAAgB,cAAc;AAAA,EAC9B,OAAgB,SAAS;AAAA,EAEzB,MAAM,MAAqB;AACzB,UAAM,KAAK,MAAM,cAAa;AAE9B,QAAI,iBAAiB,GAAG;AACtB;AAAA,IACF;AAEA,QAAI;AACF,mBAAa,KAAK,OAAO,OAAO;AAAA,IAClC,SAAS,KAAK;AAEZ,cAAQ,MAAM,yCAA0C,IAAc,OAAO,EAAE;AAAA,IACjF;AAAA,EACF;AACF;",
+  "sourcesContent": ["import { spawn } from 'node:child_process';\nimport { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs';\nimport { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport { BaseCommand } from '../../base-command.js';\nimport {\n  isGatewayRunning,\n  isRulemetricGatewayListening,\n  spawnGateway,\n  GATEWAY_PORT,\n} from '../../lib/gateway-lifecycle.js';\nimport { isPidLiveAndMatching, MITMPROXY_COMMAND_PATTERN } from '../../lib/pid-identity.js';\n\nconst CONFIG_DIR = join(homedir(), '.config', 'rulemetric');\nconst ADDON_VERSION_FILE = join(CONFIG_DIR, 'proxy-addon.version');\nconst PROXY_PID_FILE = join(CONFIG_DIR, 'proxy.pid');\nconst DRIFT_RESTART_MARKER = join(CONFIG_DIR, 'proxy-drift-restart.at');\nconst DRIFT_RESTART_COOLDOWN_MS = 10 * 60_000;\n\nexport default class GatewayEnsure extends BaseCommand {\n  static override description = 'Ensure the gateway proxy is running (called by hooks, not for direct use)';\n  static override hidden = true;\n\n  async run(): Promise<void> {\n    await this.parse(GatewayEnsure);\n\n    // Addon staleness: the gateway\u2194addon version-drift restart is CIRCULAR \u2014\n    // both sides compare versions frozen at their own spawn, so after\n    // `npm i -g` upgrades the CLI, a healthy old proxy keeps running old addon\n    // code forever (published capture fixes never reach the machine). THIS\n    // process is exec'd fresh by every session-start hook, so its\n    // config.version is always the on-disk truth \u2014 making it the one component\n    // that can detect the drift. Debounced, detached, never blocks the hook.\n    this.maybeRestartStaleProxy();\n\n    // Port truth first: the identity handshake distinguishes our live gateway\n    // from a foreign process squatting on :8787. Spawning against a held port\n    // just produces an EADDRINUSE child; spawning is only useful on 'dead'.\n    const identity = await isRulemetricGatewayListening(GATEWAY_PORT);\n\n    if (identity.status === 'ours-live') {\n      return; // Already running, silent success\n    }\n\n    if (identity.status === 'foreign') {\n      // A gateway from an older CLI build answers TCP but not the handshake;\n      // the PID-identity check on the pidfile confirms it's ours \u2014 leave it be.\n      if (isGatewayRunning()) return;\n      // Genuinely foreign: warn (don't crash the hook) \u2014 HTTPS_PROXY may be\n      // pinned at a port we don't own, which breaks Claude Code's HTTPS.\n      console.error(\n        `[RuleMetric] A non-RuleMetric process holds :${GATEWAY_PORT} \u2014 cannot start gateway. ` +\n          'Run `rulemetric diagnostics` if Claude Code HTTPS requests fail.',\n      );\n      return;\n    }\n\n    try {\n      spawnGateway(this.config.version);\n    } catch (err) {\n      // Don't crash the hook \u2014 just warn\n      console.error(`[RuleMetric] Could not start gateway: ${(err as Error).message}`);\n    }\n  }\n\n  private maybeRestartStaleProxy(): void {\n    try {\n      if (!existsSync(ADDON_VERSION_FILE)) return;\n      const runningVersion = readFileSync(ADDON_VERSION_FILE, 'utf-8').trim();\n      if (!runningVersion || runningVersion === this.config.version) return;\n\n      // Only restart a proxy that is actually OURS and alive \u2014 a stale version\n      // file next to a dead (or docker-marked) proxy is proxy start's problem.\n      const pidRaw = readFileSync(PROXY_PID_FILE, 'utf-8').trim();\n      const pid = Number(pidRaw);\n      if (!Number.isFinite(pid) || !isPidLiveAndMatching(pid, MITMPROXY_COMMAND_PATTERN)) {\n        return;\n      }\n\n      // Cooldown: session starts can cluster, and `proxy restart` needs time to\n      // rewrite the version file before the drift signal clears.\n      try {\n        const age = Date.now() - statSync(DRIFT_RESTART_MARKER).mtimeMs;\n        if (age < DRIFT_RESTART_COOLDOWN_MS) return;\n      } catch {\n        /* no marker yet \u2014 proceed */\n      }\n      writeFileSync(DRIFT_RESTART_MARKER, new Date().toISOString());\n\n      // process.argv[1] is this CLI's entry script (bin/run.js \u2014 dev or global\n      // install alike), so the restarted proxy runs the NEW on-disk code and\n      // re-stamps the version file, converging the drift.\n      const child = spawn(process.execPath, [process.argv[1], 'proxy', 'restart'], {\n        detached: true,\n        stdio: 'ignore',\n      });\n      child.unref();\n      console.error(\n        `[RuleMetric] Capture proxy is running CLI ${runningVersion} but ${this.config.version} is installed \u2014 restarting it in the background.`,\n      );\n    } catch {\n      /* never break the session-start hook */\n    }\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,aAAa;AACtB,SAAS,YAAY,cAAc,UAAU,qBAAqB;AAClE,SAAS,eAAe;AACxB,SAAS,YAAY;AAUrB,IAAM,aAAa,KAAK,QAAQ,GAAG,WAAW,YAAY;AAC1D,IAAM,qBAAqB,KAAK,YAAY,qBAAqB;AACjE,IAAM,iBAAiB,KAAK,YAAY,WAAW;AACnD,IAAM,uBAAuB,KAAK,YAAY,wBAAwB;AACtE,IAAM,4BAA4B,KAAK;AAEvC,IAAqB,gBAArB,MAAqB,uBAAsB,YAAY;AAAA,EACrD,OAAgB,cAAc;AAAA,EAC9B,OAAgB,SAAS;AAAA,EAEzB,MAAM,MAAqB;AACzB,UAAM,KAAK,MAAM,cAAa;AAS9B,SAAK,uBAAuB;AAK5B,UAAM,WAAW,MAAM,6BAA6B,YAAY;AAEhE,QAAI,SAAS,WAAW,aAAa;AACnC;AAAA,IACF;AAEA,QAAI,SAAS,WAAW,WAAW;AAGjC,UAAI,iBAAiB,EAAG;AAGxB,cAAQ;AAAA,QACN,gDAAgD,YAAY;AAAA,MAE9D;AACA;AAAA,IACF;AAEA,QAAI;AACF,mBAAa,KAAK,OAAO,OAAO;AAAA,IAClC,SAAS,KAAK;AAEZ,cAAQ,MAAM,yCAA0C,IAAc,OAAO,EAAE;AAAA,IACjF;AAAA,EACF;AAAA,EAEQ,yBAA+B;AACrC,QAAI;AACF,UAAI,CAAC,WAAW,kBAAkB,EAAG;AACrC,YAAM,iBAAiB,aAAa,oBAAoB,OAAO,EAAE,KAAK;AACtE,UAAI,CAAC,kBAAkB,mBAAmB,KAAK,OAAO,QAAS;AAI/D,YAAM,SAAS,aAAa,gBAAgB,OAAO,EAAE,KAAK;AAC1D,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI,CAAC,OAAO,SAAS,GAAG,KAAK,CAAC,qBAAqB,KAAK,yBAAyB,GAAG;AAClF;AAAA,MACF;AAIA,UAAI;AACF,cAAM,MAAM,KAAK,IAAI,IAAI,SAAS,oBAAoB,EAAE;AACxD,YAAI,MAAM,0BAA2B;AAAA,MACvC,QAAQ;AAAA,MAER;AACA,oBAAc,uBAAsB,oBAAI,KAAK,GAAE,YAAY,CAAC;AAK5D,YAAM,QAAQ,MAAM,QAAQ,UAAU,CAAC,QAAQ,KAAK,CAAC,GAAG,SAAS,SAAS,GAAG;AAAA,QAC3E,UAAU;AAAA,QACV,OAAO;AAAA,MACT,CAAC;AACD,YAAM,MAAM;AACZ,cAAQ;AAAA,QACN,6CAA6C,cAAc,QAAQ,KAAK,OAAO,OAAO;AAAA,MACxF;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/commands/hooks/install.js

@@ -5,14 +5,11 @@ import {
   findBinary
 } from "../../chunk-42GFSAJP.js";
 import {
-  STATUSLINE_SETTINGS_VALUE,
-  isStatuslineShimInstalled,
-  readCurrentStatusLine,
-  writeStatuslineShim
-} from "../../chunk-N2X35ITW.js";
-import {
+  RULEMETRIC_NO_PROXY,
+  claudeSettingsInstallPath,
   installMacOSProxyEnv,
   mergeClaudeCodeHooks,
+  migrateSharedClaudeSettings,
   readClaudeSettings,
   removeClaudeCodeHooks,
   writeAntigravityHooks,
@@ -25,27 +22,45 @@ import {
   writeCursorUserProxyConfig,
   writeVSCodeProxyConfig,
   writeVSCodeUserProxyConfig
-} from "../../chunk-XN23W5HL.js";
+} from "../../chunk-4KRMZLQV.js";
+import {
+  STATUSLINE_SETTINGS_VALUE,
+  STATUSLINE_SHIM_PATH,
+  isStatuslineShimInstalled,
+  readCurrentStatusLine,
+  writeStatuslineShim
+} from "../../chunk-OQBBVTDG.js";
+import "../../chunk-KRBQLMOP.js";
 import {
   GATEWAY_PORT,
   isGatewayRunning,
+  isRulemetricGatewayListening,
   spawnGateway,
-  waitForGatewayListening
-} from "../../chunk-DAJNO4OI.js";
-import "../../chunk-KRBQLMOP.js";
+  stopGateway,
+  waitForRulemetricGateway
+} from "../../chunk-4LYCDS2N.js";
 import {
-  emitSetupEvent
-} from "../../chunk-TX2HKJLA.js";
+  describePortOwner
+} from "../../chunk-7LVVKQMZ.js";
+import "../../chunk-5EDMGK3W.js";
+import {
+  ensureDeviceApiKey
+} from "../../chunk-EJ3YAFDP.js";
 import {
   BaseCommand
-} from "../../chunk-SZ7VDCD6.js";
+} from "../../chunk-2HK7RGZB.js";
 import "../../chunk-R56PH7YE.js";
+import {
+  emitSetupEvent
+} from "../../chunk-GQNJO2P2.js";
 import {
   bootstrapActiveOrg
-} from "../../chunk-SLL3OBKN.js";
+} from "../../chunk-37LN5O7M.js";
 import "../../chunk-EKJ2CABV.js";
-import "../../chunk-ZRLYHBPN.js";
-import "../../chunk-W2YERO7E.js";
+import "../../chunk-ZUJBM6HE.js";
+import {
+  loadEnvFileAuth
+} from "../../chunk-W2YERO7E.js";
 import "../../chunk-NSBPE2FW.js";
 
 // src/commands/hooks/install.ts
@@ -105,7 +120,8 @@ var HooksInstall = class _HooksInstall extends BaseCommand {
     const isGlobal = flags.global;
     const projectPath = isGlobal ? homedir() : flags["project-dir"] ?? process.cwd();
     const configDir = join(projectPath, ".claude");
-    const configPath = join(configDir, "settings.json");
+    const configPath = claudeSettingsInstallPath(configDir, isGlobal);
+    const settingsFileName = isGlobal ? "settings.json" : "settings.local.json";
     const noProxy = flags["no-proxy"];
     let proxyActive = false;
     let proxyReason = noProxy ? "no_proxy_flag" : void 0;
@@ -118,11 +134,27 @@ var HooksInstall = class _HooksInstall extends BaseCommand {
     if (recoveredBackup) {
       this.log(`[!!] ${configPath} was not valid JSON \u2014 backed it up to ${recoveredBackup} and continued with a fresh file.`);
     }
+    if (!isGlobal) {
+      const migration = migrateSharedClaudeSettings(join(configDir, "settings.json"), GATEWAY_PORT);
+      if (migration.migrated) {
+        this.log("[OK] Moved RuleMetric machine-local state out of the shared .claude/settings.json (it now lives in .claude/settings.local.json)");
+        if (migration.backupPath) {
+          this.log(`     Previous shared settings backed up \u2192 ${migration.backupPath}`);
+        }
+        if (migration.removedStatusline && existsSync(STATUSLINE_SHIM_PATH) && !isStatuslineShimInstalled(settings)) {
+          settings.statusLine = STATUSLINE_SETTINGS_VALUE;
+        }
+      }
+    }
+    const envAuth = loadEnvFileAuth();
+    if ((envAuth.accessToken || process.env.RULEMETRIC_ACCESS_TOKEN) && !envAuth.apiKey && !process.env.RULEMETRIC_API_KEY) {
+      await ensureDeviceApiKey(this.log.bind(this), this.warn.bind(this), this.config.version);
+    }
     const refreshedHooks = removeClaudeCodeHooks(settings);
     const hookBinDir = resolveHookBinDir();
     mergeClaudeCodeHooks(settings, hookBinDir);
     this.log(
-      refreshedHooks ? "[OK] Refreshed RuleMetric Claude Code session hooks \u2192 .claude/settings.json" : "[OK] Installed RuleMetric Claude Code session hooks \u2192 .claude/settings.json"
+      refreshedHooks ? `[OK] Refreshed RuleMetric Claude Code session hooks \u2192 .claude/${settingsFileName}` : `[OK] Installed RuleMetric Claude Code session hooks \u2192 .claude/${settingsFileName}`
     );
     if (!noProxy) {
       let mitmAvailable = Boolean(findBinary("mitmdump"));
@@ -172,10 +204,23 @@ var HooksInstall = class _HooksInstall extends BaseCommand {
         this.log("[OK] CA certificate already trusted");
         caTrusted = true;
       }
-      if (isGatewayLaunchdManaged()) {
+      const launchdManaged = isGatewayLaunchdManaged();
+      const portIdentity = await isRulemetricGatewayListening(GATEWAY_PORT);
+      if (launchdManaged) {
         this.log("[OK] Gateway managed by launchd (com.rulemetric.gateway)");
-      } else if (isGatewayRunning()) {
+      } else if (portIdentity.status === "ours-live") {
         this.log("[OK] Gateway already running");
+      } else if (portIdentity.status === "foreign" && isGatewayRunning()) {
+        this.log("[..] Restarting gateway (running build predates the identity handshake)...");
+        stopGateway();
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        try {
+          const pid = spawnGateway(this.config.version);
+          this.log(`[OK] Gateway restarted (PID ${pid})`);
+        } catch (err) {
+          this.log(`[!!] Could not restart gateway: ${err.message}`);
+        }
+      } else if (portIdentity.status === "foreign") {
       } else {
         try {
           const pid = spawnGateway(this.config.version);
@@ -184,12 +229,20 @@ var HooksInstall = class _HooksInstall extends BaseCommand {
           this.log(`[!!] Could not start gateway: ${err.message}`);
         }
       }
-      const gatewayLive = await waitForGatewayListening(GATEWAY_PORT, { timeoutMs: 4e3 });
+      const gatewayIdentity = await waitForRulemetricGateway(GATEWAY_PORT, { timeoutMs: 4e3 });
       if (!caGenerated) {
         proxyReason = mitmAvailable ? "ca_not_generated" : pkgManagers.length === 0 ? "no_package_manager" : "mitmproxy_install_failed";
         this.log("[!!] No CA certificate \u2014 NOT setting HTTPS_PROXY (capture stays hooks-only).");
         this.log("     Install mitmproxy, then re-run: rulemetric hooks install");
-      } else if (!gatewayLive) {
+      } else if (gatewayIdentity.status === "foreign" && !launchdManaged) {
+        proxyReason = "gateway_port_foreign";
+        const owner = describePortOwner(GATEWAY_PORT);
+        this.log(`[!!] Port :${GATEWAY_PORT} is held by a process that is NOT the RuleMetric gateway \u2014 NOT setting HTTPS_PROXY.`);
+        this.log(
+          owner ? `     Holding the port: ${owner}` : `     Could not identify the holder (try: lsof -nP -iTCP:${GATEWAY_PORT} -sTCP:LISTEN)`
+        );
+        this.log("     Capture still works via hooks. Free the port, then re-run: rulemetric hooks install");
+      } else if (gatewayIdentity.status === "dead") {
         proxyReason = "gateway_not_live";
         this.log(`[!!] Gateway is not answering on :${GATEWAY_PORT} \u2014 NOT setting HTTPS_PROXY.`);
         this.log("     Capture still works via hooks. To enable deep (proxy) capture later:");
@@ -197,7 +250,7 @@ var HooksInstall = class _HooksInstall extends BaseCommand {
       } else {
         const env = settings.env ?? {};
         env.HTTPS_PROXY = `http://localhost:${GATEWAY_PORT}`;
-        env.NO_PROXY = "localhost,127.0.0.1,*.supabase.co,*.supabase.in";
+        env.NO_PROXY = RULEMETRIC_NO_PROXY;
         if (existsSync(CERT_PATH)) {
           env.NODE_EXTRA_CA_CERTS = CERT_PATH;
         }