@agtlantis/eval 0.1.0

package/dist/cli.js

@@ -0,0 +1,4709 @@
+#!/usr/bin/env node
+
+// ../../../../node_modules/.pnpm/cac@6.7.14/node_modules/cac/dist/index.mjs
+import { EventEmitter } from "events";
+function toArr(any) {
+  return any == null ? [] : Array.isArray(any) ? any : [any];
+}
+function toVal(out, key, val, opts) {
+  var x, old = out[key], nxt = !!~opts.string.indexOf(key) ? val == null || val === true ? "" : String(val) : typeof val === "boolean" ? val : !!~opts.boolean.indexOf(key) ? val === "false" ? false : val === "true" || (out._.push((x = +val, x * 0 === 0) ? x : val), !!val) : (x = +val, x * 0 === 0) ? x : val;
+  out[key] = old == null ? nxt : Array.isArray(old) ? old.concat(nxt) : [old, nxt];
+}
+function mri2(args, opts) {
+  args = args || [];
+  opts = opts || {};
+  var k, arr, arg, name, val, out = { _: [] };
+  var i = 0, j = 0, idx = 0, len = args.length;
+  const alibi = opts.alias !== void 0;
+  const strict = opts.unknown !== void 0;
+  const defaults = opts.default !== void 0;
+  opts.alias = opts.alias || {};
+  opts.string = toArr(opts.string);
+  opts.boolean = toArr(opts.boolean);
+  if (alibi) {
+    for (k in opts.alias) {
+      arr = opts.alias[k] = toArr(opts.alias[k]);
+      for (i = 0; i < arr.length; i++) {
+        (opts.alias[arr[i]] = arr.concat(k)).splice(i, 1);
+      }
+    }
+  }
+  for (i = opts.boolean.length; i-- > 0; ) {
+    arr = opts.alias[opts.boolean[i]] || [];
+    for (j = arr.length; j-- > 0; ) opts.boolean.push(arr[j]);
+  }
+  for (i = opts.string.length; i-- > 0; ) {
+    arr = opts.alias[opts.string[i]] || [];
+    for (j = arr.length; j-- > 0; ) opts.string.push(arr[j]);
+  }
+  if (defaults) {
+    for (k in opts.default) {
+      name = typeof opts.default[k];
+      arr = opts.alias[k] = opts.alias[k] || [];
+      if (opts[name] !== void 0) {
+        opts[name].push(k);
+        for (i = 0; i < arr.length; i++) {
+          opts[name].push(arr[i]);
+        }
+      }
+    }
+  }
+  const keys = strict ? Object.keys(opts.alias) : [];
+  for (i = 0; i < len; i++) {
+    arg = args[i];
+    if (arg === "--") {
+      out._ = out._.concat(args.slice(++i));
+      break;
+    }
+    for (j = 0; j < arg.length; j++) {
+      if (arg.charCodeAt(j) !== 45) break;
+    }
+    if (j === 0) {
+      out._.push(arg);
+    } else if (arg.substring(j, j + 3) === "no-") {
+      name = arg.substring(j + 3);
+      if (strict && !~keys.indexOf(name)) {
+        return opts.unknown(arg);
+      }
+      out[name] = false;
+    } else {
+      for (idx = j + 1; idx < arg.length; idx++) {
+        if (arg.charCodeAt(idx) === 61) break;
+      }
+      name = arg.substring(j, idx);
+      val = arg.substring(++idx) || (i + 1 === len || ("" + args[i + 1]).charCodeAt(0) === 45 || args[++i]);
+      arr = j === 2 ? [name] : name;
+      for (idx = 0; idx < arr.length; idx++) {
+        name = arr[idx];
+        if (strict && !~keys.indexOf(name)) return opts.unknown("-".repeat(j) + name);
+        toVal(out, name, idx + 1 < arr.length || val, opts);
+      }
+    }
+  }
+  if (defaults) {
+    for (k in opts.default) {
+      if (out[k] === void 0) {
+        out[k] = opts.default[k];
+      }
+    }
+  }
+  if (alibi) {
+    for (k in out) {
+      arr = opts.alias[k] || [];
+      while (arr.length > 0) {
+        out[arr.shift()] = out[k];
+      }
+    }
+  }
+  return out;
+}
+var removeBrackets = (v) => v.replace(/[<[].+/, "").trim();
+var findAllBrackets = (v) => {
+  const ANGLED_BRACKET_RE_GLOBAL = /<([^>]+)>/g;
+  const SQUARE_BRACKET_RE_GLOBAL = /\[([^\]]+)\]/g;
+  const res = [];
+  const parse = (match) => {
+    let variadic = false;
+    let value = match[1];
+    if (value.startsWith("...")) {
+      value = value.slice(3);
+      variadic = true;
+    }
+    return {
+      required: match[0].startsWith("<"),
+      value,
+      variadic
+    };
+  };
+  let angledMatch;
+  while (angledMatch = ANGLED_BRACKET_RE_GLOBAL.exec(v)) {
+    res.push(parse(angledMatch));
+  }
+  let squareMatch;
+  while (squareMatch = SQUARE_BRACKET_RE_GLOBAL.exec(v)) {
+    res.push(parse(squareMatch));
+  }
+  return res;
+};
+var getMriOptions = (options) => {
+  const result = { alias: {}, boolean: [] };
+  for (const [index, option] of options.entries()) {
+    if (option.names.length > 1) {
+      result.alias[option.names[0]] = option.names.slice(1);
+    }
+    if (option.isBoolean) {
+      if (option.negated) {
+        const hasStringTypeOption = options.some((o, i) => {
+          return i !== index && o.names.some((name) => option.names.includes(name)) && typeof o.required === "boolean";
+        });
+        if (!hasStringTypeOption) {
+          result.boolean.push(option.names[0]);
+        }
+      } else {
+        result.boolean.push(option.names[0]);
+      }
+    }
+  }
+  return result;
+};
+var findLongest = (arr) => {
+  return arr.sort((a, b) => {
+    return a.length > b.length ? -1 : 1;
+  })[0];
+};
+var padRight = (str, length) => {
+  return str.length >= length ? str : `${str}${" ".repeat(length - str.length)}`;
+};
+var camelcase = (input) => {
+  return input.replace(/([a-z])-([a-z])/g, (_, p1, p2) => {
+    return p1 + p2.toUpperCase();
+  });
+};
+var setDotProp = (obj, keys, val) => {
+  let i = 0;
+  let length = keys.length;
+  let t = obj;
+  let x;
+  for (; i < length; ++i) {
+    x = t[keys[i]];
+    t = t[keys[i]] = i === length - 1 ? val : x != null ? x : !!~keys[i + 1].indexOf(".") || !(+keys[i + 1] > -1) ? {} : [];
+  }
+};
+var setByType = (obj, transforms) => {
+  for (const key of Object.keys(transforms)) {
+    const transform = transforms[key];
+    if (transform.shouldTransform) {
+      obj[key] = Array.prototype.concat.call([], obj[key]);
+      if (typeof transform.transformFunction === "function") {
+        obj[key] = obj[key].map(transform.transformFunction);
+      }
+    }
+  }
+};
+var getFileName = (input) => {
+  const m = /([^\\\/]+)$/.exec(input);
+  return m ? m[1] : "";
+};
+var camelcaseOptionName = (name) => {
+  return name.split(".").map((v, i) => {
+    return i === 0 ? camelcase(v) : v;
+  }).join(".");
+};
+var CACError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = this.constructor.name;
+    if (typeof Error.captureStackTrace === "function") {
+      Error.captureStackTrace(this, this.constructor);
+    } else {
+      this.stack = new Error(message).stack;
+    }
+  }
+};
+var Option = class {
+  constructor(rawName, description, config) {
+    this.rawName = rawName;
+    this.description = description;
+    this.config = Object.assign({}, config);
+    rawName = rawName.replace(/\.\*/g, "");
+    this.negated = false;
+    this.names = removeBrackets(rawName).split(",").map((v) => {
+      let name = v.trim().replace(/^-{1,2}/, "");
+      if (name.startsWith("no-")) {
+        this.negated = true;
+        name = name.replace(/^no-/, "");
+      }
+      return camelcaseOptionName(name);
+    }).sort((a, b) => a.length > b.length ? 1 : -1);
+    this.name = this.names[this.names.length - 1];
+    if (this.negated && this.config.default == null) {
+      this.config.default = true;
+    }
+    if (rawName.includes("<")) {
+      this.required = true;
+    } else if (rawName.includes("[")) {
+      this.required = false;
+    } else {
+      this.isBoolean = true;
+    }
+  }
+};
+var processArgs = process.argv;
+var platformInfo = `${process.platform}-${process.arch} node-${process.version}`;
+var Command = class {
+  constructor(rawName, description, config = {}, cli2) {
+    this.rawName = rawName;
+    this.description = description;
+    this.config = config;
+    this.cli = cli2;
+    this.options = [];
+    this.aliasNames = [];
+    this.name = removeBrackets(rawName);
+    this.args = findAllBrackets(rawName);
+    this.examples = [];
+  }
+  usage(text) {
+    this.usageText = text;
+    return this;
+  }
+  allowUnknownOptions() {
+    this.config.allowUnknownOptions = true;
+    return this;
+  }
+  ignoreOptionDefaultValue() {
+    this.config.ignoreOptionDefaultValue = true;
+    return this;
+  }
+  version(version, customFlags = "-v, --version") {
+    this.versionNumber = version;
+    this.option(customFlags, "Display version number");
+    return this;
+  }
+  example(example) {
+    this.examples.push(example);
+    return this;
+  }
+  option(rawName, description, config) {
+    const option = new Option(rawName, description, config);
+    this.options.push(option);
+    return this;
+  }
+  alias(name) {
+    this.aliasNames.push(name);
+    return this;
+  }
+  action(callback) {
+    this.commandAction = callback;
+    return this;
+  }
+  isMatched(name) {
+    return this.name === name || this.aliasNames.includes(name);
+  }
+  get isDefaultCommand() {
+    return this.name === "" || this.aliasNames.includes("!");
+  }
+  get isGlobalCommand() {
+    return this instanceof GlobalCommand;
+  }
+  hasOption(name) {
+    name = name.split(".")[0];
+    return this.options.find((option) => {
+      return option.names.includes(name);
+    });
+  }
+  outputHelp() {
+    const { name, commands } = this.cli;
+    const {
+      versionNumber,
+      options: globalOptions,
+      helpCallback
+    } = this.cli.globalCommand;
+    let sections = [
+      {
+        body: `${name}${versionNumber ? `/${versionNumber}` : ""}`
+      }
+    ];
+    sections.push({
+      title: "Usage",
+      body: `  $ ${name} ${this.usageText || this.rawName}`
+    });
+    const showCommands = (this.isGlobalCommand || this.isDefaultCommand) && commands.length > 0;
+    if (showCommands) {
+      const longestCommandName = findLongest(commands.map((command) => command.rawName));
+      sections.push({
+        title: "Commands",
+        body: commands.map((command) => {
+          return `  ${padRight(command.rawName, longestCommandName.length)}  ${command.description}`;
+        }).join("\n")
+      });
+      sections.push({
+        title: `For more info, run any command with the \`--help\` flag`,
+        body: commands.map((command) => `  $ ${name}${command.name === "" ? "" : ` ${command.name}`} --help`).join("\n")
+      });
+    }
+    let options = this.isGlobalCommand ? globalOptions : [...this.options, ...globalOptions || []];
+    if (!this.isGlobalCommand && !this.isDefaultCommand) {
+      options = options.filter((option) => option.name !== "version");
+    }
+    if (options.length > 0) {
+      const longestOptionName = findLongest(options.map((option) => option.rawName));
+      sections.push({
+        title: "Options",
+        body: options.map((option) => {
+          return `  ${padRight(option.rawName, longestOptionName.length)}  ${option.description} ${option.config.default === void 0 ? "" : `(default: ${option.config.default})`}`;
+        }).join("\n")
+      });
+    }
+    if (this.examples.length > 0) {
+      sections.push({
+        title: "Examples",
+        body: this.examples.map((example) => {
+          if (typeof example === "function") {
+            return example(name);
+          }
+          return example;
+        }).join("\n")
+      });
+    }
+    if (helpCallback) {
+      sections = helpCallback(sections) || sections;
+    }
+    console.log(sections.map((section) => {
+      return section.title ? `${section.title}:
+${section.body}` : section.body;
+    }).join("\n\n"));
+  }
+  outputVersion() {
+    const { name } = this.cli;
+    const { versionNumber } = this.cli.globalCommand;
+    if (versionNumber) {
+      console.log(`${name}/${versionNumber} ${platformInfo}`);
+    }
+  }
+  checkRequiredArgs() {
+    const minimalArgsCount = this.args.filter((arg) => arg.required).length;
+    if (this.cli.args.length < minimalArgsCount) {
+      throw new CACError(`missing required args for command \`${this.rawName}\``);
+    }
+  }
+  checkUnknownOptions() {
+    const { options, globalCommand } = this.cli;
+    if (!this.config.allowUnknownOptions) {
+      for (const name of Object.keys(options)) {
+        if (name !== "--" && !this.hasOption(name) && !globalCommand.hasOption(name)) {
+          throw new CACError(`Unknown option \`${name.length > 1 ? `--${name}` : `-${name}`}\``);
+        }
+      }
+    }
+  }
+  checkOptionValue() {
+    const { options: parsedOptions, globalCommand } = this.cli;
+    const options = [...globalCommand.options, ...this.options];
+    for (const option of options) {
+      const value = parsedOptions[option.name.split(".")[0]];
+      if (option.required) {
+        const hasNegated = options.some((o) => o.negated && o.names.includes(option.name));
+        if (value === true || value === false && !hasNegated) {
+          throw new CACError(`option \`${option.rawName}\` value is missing`);
+        }
+      }
+    }
+  }
+};
+var GlobalCommand = class extends Command {
+  constructor(cli2) {
+    super("@@global@@", "", {}, cli2);
+  }
+};
+var __assign = Object.assign;
+var CAC = class extends EventEmitter {
+  constructor(name = "") {
+    super();
+    this.name = name;
+    this.commands = [];
+    this.rawArgs = [];
+    this.args = [];
+    this.options = {};
+    this.globalCommand = new GlobalCommand(this);
+    this.globalCommand.usage("<command> [options]");
+  }
+  usage(text) {
+    this.globalCommand.usage(text);
+    return this;
+  }
+  command(rawName, description, config) {
+    const command = new Command(rawName, description || "", config, this);
+    command.globalCommand = this.globalCommand;
+    this.commands.push(command);
+    return command;
+  }
+  option(rawName, description, config) {
+    this.globalCommand.option(rawName, description, config);
+    return this;
+  }
+  help(callback) {
+    this.globalCommand.option("-h, --help", "Display this message");
+    this.globalCommand.helpCallback = callback;
+    this.showHelpOnExit = true;
+    return this;
+  }
+  version(version, customFlags = "-v, --version") {
+    this.globalCommand.version(version, customFlags);
+    this.showVersionOnExit = true;
+    return this;
+  }
+  example(example) {
+    this.globalCommand.example(example);
+    return this;
+  }
+  outputHelp() {
+    if (this.matchedCommand) {
+      this.matchedCommand.outputHelp();
+    } else {
+      this.globalCommand.outputHelp();
+    }
+  }
+  outputVersion() {
+    this.globalCommand.outputVersion();
+  }
+  setParsedInfo({ args, options }, matchedCommand, matchedCommandName) {
+    this.args = args;
+    this.options = options;
+    if (matchedCommand) {
+      this.matchedCommand = matchedCommand;
+    }
+    if (matchedCommandName) {
+      this.matchedCommandName = matchedCommandName;
+    }
+    return this;
+  }
+  unsetMatchedCommand() {
+    this.matchedCommand = void 0;
+    this.matchedCommandName = void 0;
+  }
+  parse(argv = processArgs, {
+    run = true
+  } = {}) {
+    this.rawArgs = argv;
+    if (!this.name) {
+      this.name = argv[1] ? getFileName(argv[1]) : "cli";
+    }
+    let shouldParse = true;
+    for (const command of this.commands) {
+      const parsed = this.mri(argv.slice(2), command);
+      const commandName = parsed.args[0];
+      if (command.isMatched(commandName)) {
+        shouldParse = false;
+        const parsedInfo = __assign(__assign({}, parsed), {
+          args: parsed.args.slice(1)
+        });
+        this.setParsedInfo(parsedInfo, command, commandName);
+        this.emit(`command:${commandName}`, command);
+      }
+    }
+    if (shouldParse) {
+      for (const command of this.commands) {
+        if (command.name === "") {
+          shouldParse = false;
+          const parsed = this.mri(argv.slice(2), command);
+          this.setParsedInfo(parsed, command);
+          this.emit(`command:!`, command);
+        }
+      }
+    }
+    if (shouldParse) {
+      const parsed = this.mri(argv.slice(2));
+      this.setParsedInfo(parsed);
+    }
+    if (this.options.help && this.showHelpOnExit) {
+      this.outputHelp();
+      run = false;
+      this.unsetMatchedCommand();
+    }
+    if (this.options.version && this.showVersionOnExit && this.matchedCommandName == null) {
+      this.outputVersion();
+      run = false;
+      this.unsetMatchedCommand();
+    }
+    const parsedArgv = { args: this.args, options: this.options };
+    if (run) {
+      this.runMatchedCommand();
+    }
+    if (!this.matchedCommand && this.args[0]) {
+      this.emit("command:*");
+    }
+    return parsedArgv;
+  }
+  mri(argv, command) {
+    const cliOptions = [
+      ...this.globalCommand.options,
+      ...command ? command.options : []
+    ];
+    const mriOptions = getMriOptions(cliOptions);
+    let argsAfterDoubleDashes = [];
+    const doubleDashesIndex = argv.indexOf("--");
+    if (doubleDashesIndex > -1) {
+      argsAfterDoubleDashes = argv.slice(doubleDashesIndex + 1);
+      argv = argv.slice(0, doubleDashesIndex);
+    }
+    let parsed = mri2(argv, mriOptions);
+    parsed = Object.keys(parsed).reduce((res, name) => {
+      return __assign(__assign({}, res), {
+        [camelcaseOptionName(name)]: parsed[name]
+      });
+    }, { _: [] });
+    const args = parsed._;
+    const options = {
+      "--": argsAfterDoubleDashes
+    };
+    const ignoreDefault = command && command.config.ignoreOptionDefaultValue ? command.config.ignoreOptionDefaultValue : this.globalCommand.config.ignoreOptionDefaultValue;
+    let transforms = /* @__PURE__ */ Object.create(null);
+    for (const cliOption of cliOptions) {
+      if (!ignoreDefault && cliOption.config.default !== void 0) {
+        for (const name of cliOption.names) {
+          options[name] = cliOption.config.default;
+        }
+      }
+      if (Array.isArray(cliOption.config.type)) {
+        if (transforms[cliOption.name] === void 0) {
+          transforms[cliOption.name] = /* @__PURE__ */ Object.create(null);
+          transforms[cliOption.name]["shouldTransform"] = true;
+          transforms[cliOption.name]["transformFunction"] = cliOption.config.type[0];
+        }
+      }
+    }
+    for (const key of Object.keys(parsed)) {
+      if (key !== "_") {
+        const keys = key.split(".");
+        setDotProp(options, keys, parsed[key]);
+        setByType(options, transforms);
+      }
+    }
+    return {
+      args,
+      options
+    };
+  }
+  runMatchedCommand() {
+    const { args, options, matchedCommand: command } = this;
+    if (!command || !command.commandAction)
+      return;
+    command.checkUnknownOptions();
+    command.checkOptionValue();
+    command.checkRequiredArgs();
+    const actionArgs = [];
+    command.args.forEach((arg, index) => {
+      if (arg.variadic) {
+        actionArgs.push(args.slice(index));
+      } else {
+        actionArgs.push(args[index]);
+      }
+    });
+    actionArgs.push(options);
+    return command.commandAction.apply(this, actionArgs);
+  }
+};
+var cac = (name = "") => new CAC(name);
+var dist_default = cac;
+
+// src/cli/config/types.ts
+function isMultiTurnConfig(testCase2) {
+  return "multiTurn" in testCase2 && testCase2.multiTurn !== void 0;
+}
+
+// src/cli/config/schema.ts
+import { z } from "zod";
+
+// src/core/errors.ts
+var EvalError = class _EvalError extends Error {
+  code;
+  cause;
+  context;
+  constructor(message, options) {
+    super(message);
+    this.name = "EvalError";
+    this.code = options.code;
+    this.cause = options.cause;
+    this.context = options.context;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, _EvalError);
+    }
+  }
+  /**
+   * Creates an EvalError from an unknown error with a specific code.
+   */
+  static from(error, code, context) {
+    if (error instanceof _EvalError) {
+      return error;
+    }
+    const cause = error instanceof Error ? error : new Error(String(error));
+    return new _EvalError(cause.message, { code, cause, context });
+  }
+  toJSON() {
+    return {
+      name: this.name,
+      message: this.message,
+      code: this.code,
+      context: this.context,
+      cause: this.cause?.message
+    };
+  }
+};
+
+// src/cli/config/schema.ts
+var llmConfigSchema = z.object({
+  provider: z.enum(["openai", "gemini"], {
+    errorMap: () => ({
+      message: "provider must be 'openai' or 'gemini'"
+    })
+  }),
+  apiKey: z.string().optional(),
+  defaultModel: z.string().optional(),
+  reasoningEffort: z.enum(["minimal", "low", "medium", "high"]).optional(),
+  defaultResponseFormat: z.object({
+    type: z.enum(["json_object", "text"])
+  }).optional()
+});
+var criterionSchema = z.object({
+  id: z.string().min(1, "Criterion id is required"),
+  name: z.string().min(1, "Criterion name is required"),
+  description: z.string().min(1, "Criterion description is required"),
+  weight: z.number().positive().optional(),
+  validator: z.function().optional()
+});
+var judgeConfigSchema = z.object({
+  llm: llmConfigSchema.optional(),
+  criteria: z.array(criterionSchema).min(1, "At least one criterion is required"),
+  passThreshold: z.number().min(0).max(100).optional(),
+  prompt: z.any().optional()
+});
+var improverConfigSchema = z.object({
+  llm: llmConfigSchema.optional(),
+  prompt: z.any().optional()
+}).optional();
+var outputConfigSchema = z.object({
+  dir: z.string().optional(),
+  filename: z.string().optional(),
+  verbose: z.boolean().optional()
+}).optional();
+var runConfigSchema = z.object({
+  concurrency: z.number().int().positive().optional(),
+  iterations: z.number().int().positive().optional(),
+  stopOnFirstFailure: z.boolean().optional()
+}).optional();
+var maxTurnsConditionSchema = z.object({
+  type: z.literal("maxTurns"),
+  count: z.number().int().positive()
+});
+var fieldSetConditionSchema = z.object({
+  type: z.literal("fieldSet"),
+  fieldPath: z.string().min(1)
+});
+var fieldValueConditionSchema = z.object({
+  type: z.literal("fieldValue"),
+  fieldPath: z.string().min(1),
+  expectedValue: z.unknown()
+});
+var customConditionSchema = z.object({
+  type: z.literal("custom"),
+  check: z.function(),
+  description: z.string().optional()
+});
+var terminationConditionSchema = z.union([
+  maxTurnsConditionSchema,
+  fieldSetConditionSchema,
+  fieldValueConditionSchema,
+  customConditionSchema
+]);
+var followUpInputSchema = z.object({
+  input: z.unknown(),
+  description: z.string().optional(),
+  turns: z.number().optional()
+});
+var multiTurnConfigSchema = z.object({
+  followUpInputs: z.array(followUpInputSchema).optional(),
+  terminateWhen: z.array(terminationConditionSchema).min(1, "At least one termination condition is required"),
+  maxTurns: z.number().int().positive().optional(),
+  onConditionMet: z.enum(["pass", "fail"]).optional(),
+  onMaxTurnsReached: z.enum(["pass", "fail"]).optional()
+});
+var testCaseSchema = z.object({
+  id: z.string().optional(),
+  input: z.unknown(),
+  tags: z.array(z.string()).optional(),
+  description: z.string().optional(),
+  expectedOutput: z.unknown().optional(),
+  files: z.array(z.any()).optional(),
+  multiTurn: multiTurnConfigSchema.optional()
+});
+var agentSchema = z.object({
+  config: z.object({
+    name: z.string(),
+    description: z.string().optional()
+  }),
+  prompt: z.object({
+    id: z.string(),
+    version: z.string(),
+    system: z.string(),
+    renderUserPrompt: z.function()
+  }),
+  execute: z.function()
+});
+var evalConfigSchema = z.object({
+  name: z.string().optional(),
+  agentDescription: z.string().optional(),
+  agent: agentSchema,
+  llm: llmConfigSchema,
+  judge: judgeConfigSchema,
+  improver: improverConfigSchema,
+  testCases: z.array(testCaseSchema).optional(),
+  output: outputConfigSchema,
+  run: runConfigSchema,
+  include: z.array(z.string().min(1, "Include pattern cannot be empty")).min(1, "Include array must have at least one pattern").optional(),
+  agents: z.record(z.string(), agentSchema).optional()
+}).refine(
+  (data) => {
+    const hasTestCases = (data.testCases?.length ?? 0) > 0;
+    const hasInclude = (data.include?.length ?? 0) > 0;
+    return hasTestCases || hasInclude;
+  },
+  {
+    message: "Either testCases or include must be provided. Use testCases for inline TypeScript tests, or include for YAML file discovery.",
+    path: ["testCases"]
+  }
+);
+function validateConfig(config) {
+  const result = evalConfigSchema.safeParse(config);
+  if (!result.success) {
+    const errors = result.error.issues.map((issue) => {
+      const path3 = issue.path.join(".");
+      return path3 ? `  - ${path3}: ${issue.message}` : `  - ${issue.message}`;
+    }).join("\n");
+    throw new EvalError(`Invalid configuration:
+${errors}`, {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */
+    });
+  }
+  return result.data;
+}
+
+// src/cli/config/loader.ts
+import { existsSync } from "fs";
+import { resolve, extname } from "path";
+import { pathToFileURL } from "url";
+import { bundleRequire } from "bundle-require";
+import fg from "fast-glob";
+var ConfigError = class extends Error {
+  constructor(message, code, context) {
+    super(message);
+    this.code = code;
+    this.context = context;
+    this.name = "ConfigError";
+  }
+};
+var DEFAULT_CONFIG_FILE = "agent-eval.config.ts";
+var SUPPORTED_EXTENSIONS = [".ts", ".mts", ".cts", ".js", ".mjs", ".cjs"];
+function resolveConfigPath(configPath = DEFAULT_CONFIG_FILE, cwd = process.cwd()) {
+  return resolve(cwd, configPath);
+}
+async function loadConfig(configPath) {
+  const absolutePath = resolve(process.cwd(), configPath);
+  if (!existsSync(absolutePath)) {
+    throw new ConfigError(
+      `Config file not found: ${configPath}
+
+Create an ${DEFAULT_CONFIG_FILE} file or specify a path:
+  npx agent-eval run ./path/to/config.ts`,
+      "CONFIG_NOT_FOUND",
+      { path: absolutePath }
+    );
+  }
+  const ext = extname(absolutePath).toLowerCase();
+  if (!SUPPORTED_EXTENSIONS.includes(ext)) {
+    throw new ConfigError(
+      `Unsupported config file extension: ${ext}
+Supported extensions: ${SUPPORTED_EXTENSIONS.join(", ")}`,
+      "CONFIG_LOAD_ERROR",
+      { path: absolutePath, extension: ext }
+    );
+  }
+  let mod;
+  try {
+    if (ext === ".ts" || ext === ".mts" || ext === ".cts") {
+      const result = await bundleRequire({
+        filepath: absolutePath,
+        format: "esm",
+        esbuildOptions: { sourcemap: "inline" }
+      });
+      mod = result.mod;
+    } else {
+      const fileUrl = pathToFileURL(absolutePath).href;
+      mod = await import(fileUrl);
+    }
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new ConfigError(
+      `Failed to load config file: ${configPath}
+
+Error: ${message}
+
+Make sure the file is valid TypeScript/JavaScript and has no syntax errors.`,
+      "CONFIG_LOAD_ERROR",
+      { path: absolutePath, originalError: message }
+    );
+  }
+  const config = "default" in mod ? mod.default : mod;
+  if (!config || typeof config !== "object") {
+    throw new ConfigError(
+      `Config file must export a default configuration object.
+
+Example:
+  import { defineConfig } from '@agtlantis/eval'
+  export default defineConfig({ ... })`,
+      "CONFIG_NO_DEFAULT_EXPORT",
+      { path: absolutePath }
+    );
+  }
+  try {
+    validateConfig(config);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new ConfigError(
+      message,
+      "CONFIG_VALIDATION_ERROR",
+      { path: absolutePath }
+    );
+  }
+  return config;
+}
+async function loadConfigWithDefaults(configPath, cwd) {
+  const resolvedPath = resolveConfigPath(configPath, cwd);
+  return loadConfig(resolvedPath);
+}
+async function discoverEvalFiles(config, options = {}) {
+  const patterns = options.include ?? config.include;
+  if (!patterns || patterns.length === 0) {
+    throw new ConfigError(
+      `No include patterns specified.
+
+Add an include field to your config:
+  include: ['evals/**/*.eval.yaml']
+
+Or use the --include CLI option:
+  npx agent-eval --include "evals/**/*.eval.yaml"`,
+      "CONFIG_NO_INCLUDE_PATTERN"
+    );
+  }
+  const cwd = options.cwd ?? process.cwd();
+  const ignore = options.ignore ?? ["**/node_modules/**"];
+  const files = await fg(patterns, {
+    absolute: true,
+    cwd,
+    ignore,
+    onlyFiles: true,
+    dot: false,
+    followSymbolicLinks: false,
+    unique: true,
+    suppressErrors: false
+  });
+  return files.sort();
+}
+
+// src/cli/yaml/schema.ts
+import { z as z2 } from "zod";
+var yamlExpectationSchema = z2.object({
+  minTurns: z2.number().int().positive().optional(),
+  maxTurns: z2.number().int().positive().optional(),
+  minScore: z2.number().min(0).max(100).optional()
+});
+var yamlTerminationConditionSchema = z2.object({
+  field: z2.string().min(1).optional(),
+  equals: z2.unknown().optional(),
+  naturalLanguage: z2.string().min(1).optional()
+}).refine((data) => data.field !== void 0 || data.naturalLanguage !== void 0, {
+  message: "Either field or naturalLanguage must be specified"
+});
+var yamlPersonaSchema = z2.object({
+  name: z2.string().min(1, "Persona name is required"),
+  description: z2.string().optional(),
+  systemPrompt: z2.string().min(1, "Persona systemPrompt is required")
+});
+var yamlTestCaseDefaultsSchema = z2.object({
+  maxTurns: z2.number().int().positive().optional(),
+  endWhen: yamlTerminationConditionSchema.optional(),
+  onConditionMet: z2.enum(["pass", "fail"]).optional(),
+  onMaxTurnsReached: z2.enum(["pass", "fail"]).optional(),
+  tags: z2.array(z2.string()).optional()
+});
+var yamlTestCaseSchema = z2.object({
+  id: z2.string().min(1, "Test case id is required"),
+  name: z2.string().optional(),
+  description: z2.string().optional(),
+  tags: z2.array(z2.string()).optional(),
+  input: z2.record(z2.unknown()),
+  persona: z2.union([z2.string().min(1), yamlPersonaSchema]).optional(),
+  maxTurns: z2.number().int().positive().optional(),
+  endWhen: yamlTerminationConditionSchema.optional(),
+  onConditionMet: z2.enum(["pass", "fail"]).optional(),
+  onMaxTurnsReached: z2.enum(["pass", "fail"]).optional(),
+  expectedOutput: z2.record(z2.unknown()).optional(),
+  expect: yamlExpectationSchema.optional()
+});
+var yamlEvalFileSchema = z2.object({
+  agent: z2.string().min(1, "Agent name is required"),
+  name: z2.string().optional(),
+  description: z2.string().optional(),
+  defaults: yamlTestCaseDefaultsSchema.optional(),
+  personas: z2.record(yamlPersonaSchema).optional(),
+  cases: z2.array(yamlTestCaseSchema).min(1, "At least one test case is required")
+});
+function validateYamlEvalFile(content) {
+  const result = yamlEvalFileSchema.safeParse(content);
+  if (!result.success) {
+    const errors = result.error.issues.map((issue) => {
+      const path3 = issue.path.join(".");
+      return path3 ? `  - ${path3}: ${issue.message}` : `  - ${issue.message}`;
+    }).join("\n");
+    throw new EvalError(`Invalid YAML eval file:
+${errors}`, {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */
+    });
+  }
+  return result.data;
+}
+
+// src/cli/yaml/loader.ts
+import { existsSync as existsSync2 } from "fs";
+import { readFile } from "fs/promises";
+import { isAbsolute, resolve as resolve2 } from "path";
+import { parse as parseYaml } from "yaml";
+
+// src/multi-turn/types.ts
+function isMaxTurnsCondition(condition) {
+  return condition.type === "maxTurns";
+}
+function isFieldSetCondition(condition) {
+  return condition.type === "fieldSet";
+}
+function isFieldValueCondition(condition) {
+  return condition.type === "fieldValue";
+}
+function isCustomCondition(condition) {
+  return condition.type === "custom";
+}
+function isMultiTurnTestCase(testCase2) {
+  return "multiTurn" in testCase2;
+}
+function isTerminated(result) {
+  return result.terminated === true;
+}
+
+// src/multi-turn/termination.ts
+function getFieldValue(obj, fieldPath) {
+  if (obj === null || obj === void 0) {
+    return void 0;
+  }
+  const parts = fieldPath.split(".");
+  let current = obj;
+  for (const part of parts) {
+    if (current === null || current === void 0) {
+      return void 0;
+    }
+    if (typeof current !== "object") {
+      return void 0;
+    }
+    current = current[part];
+  }
+  return current;
+}
+function isSet(value) {
+  return value !== null && value !== void 0;
+}
+function checkMaxTurns(condition, context) {
+  const shouldTerminate = context.currentTurn >= condition.count;
+  if (shouldTerminate) {
+    return {
+      terminated: true,
+      terminationType: "maxTurns",
+      matchedCondition: condition,
+      reason: `Maximum turns reached (${condition.count})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Turn ${context.currentTurn} of ${condition.count}`
+  };
+}
+function checkFieldSet(condition, context) {
+  const fieldValue = getFieldValue(context.lastOutput, condition.fieldPath);
+  const fieldIsSet2 = isSet(fieldValue);
+  if (fieldIsSet2) {
+    return {
+      terminated: true,
+      terminationType: "condition",
+      matchedCondition: condition,
+      reason: `Field "${condition.fieldPath}" is set (value: ${JSON.stringify(fieldValue)})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Field "${condition.fieldPath}" is not set`
+  };
+}
+function checkFieldValue(condition, context) {
+  const fieldValue = getFieldValue(context.lastOutput, condition.fieldPath);
+  const matches = fieldValue === condition.expectedValue;
+  if (matches) {
+    return {
+      terminated: true,
+      terminationType: "condition",
+      matchedCondition: condition,
+      reason: `Field "${condition.fieldPath}" equals expected value`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Field "${condition.fieldPath}" does not equal expected value (got: ${JSON.stringify(fieldValue)})`
+  };
+}
+async function checkCustom(condition, context) {
+  const description = condition.description ?? "Custom condition";
+  try {
+    const shouldTerminate = await condition.check(context);
+    if (shouldTerminate) {
+      return {
+        terminated: true,
+        terminationType: "condition",
+        matchedCondition: condition,
+        reason: `${description} met`
+      };
+    }
+    return {
+      terminated: false,
+      reason: `${description} not met`
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    return {
+      terminated: false,
+      reason: `${description} failed: ${errorMessage}`
+    };
+  }
+}
+async function checkCondition(condition, context) {
+  if (isMaxTurnsCondition(condition)) {
+    return checkMaxTurns(condition, context);
+  }
+  if (isFieldValueCondition(condition)) {
+    return checkFieldValue(condition, context);
+  }
+  if (isFieldSetCondition(condition)) {
+    return checkFieldSet(condition, context);
+  }
+  if (isCustomCondition(condition)) {
+    return checkCustom(condition, context);
+  }
+  const _exhaustive = condition;
+  throw new EvalError(`Unknown condition type: ${JSON.stringify(_exhaustive)}`, {
+    code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */,
+    context: { condition: _exhaustive }
+  });
+}
+async function checkTermination(conditions, context) {
+  if (conditions.length === 0) {
+    return {
+      terminated: false,
+      reason: "No termination conditions specified"
+    };
+  }
+  for (const condition of conditions) {
+    const result = await checkCondition(condition, context);
+    if (result.terminated) {
+      return result;
+    }
+  }
+  return {
+    terminated: false,
+    reason: "No termination conditions met"
+  };
+}
+
+// src/utils/json.ts
+function truncate(str, maxLength) {
+  if (!str) {
+    return "";
+  }
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.slice(0, maxLength) + "...";
+}
+
+// src/multi-turn/conditions.ts
+function naturalLanguage(options) {
+  const { provider, prompt, systemPrompt } = options;
+  const defaultSystemPrompt = `You are an assistant that evaluates whether a conversation should terminate.
+Analyze the conversation history and determine if the specified condition is met.
+Respond with ONLY "yes" or "no" - nothing else.`;
+  return {
+    type: "custom",
+    check: async (context) => {
+      const historyText = context.history.map(
+        (h) => `Turn ${h.turn}:
+Input: ${JSON.stringify(h.input)}
+Output: ${JSON.stringify(h.output)}`
+      ).join("\n\n");
+      const userPrompt = `## Termination Condition
+${prompt}
+
+## Conversation History
+${historyText || "(No history yet)"}
+
+## Current Turn
+Turn: ${context.currentTurn}
+Last Output: ${JSON.stringify(context.lastOutput)}
+
+Should the conversation terminate based on the condition above? Answer "yes" or "no" only.`;
+      const execution = provider.simpleExecution(async (session) => {
+        const result = await session.generateText({
+          messages: [
+            { role: "system", content: systemPrompt ?? defaultSystemPrompt },
+            { role: "user", content: userPrompt }
+          ]
+        });
+        return result.text;
+      });
+      const executionResult = await execution.result();
+      if (executionResult.status !== "succeeded") {
+        throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+      }
+      const responseText = executionResult.value;
+      const answer = responseText.toLowerCase().trim();
+      return answer === "yes" || answer.startsWith("yes");
+    },
+    description: `NL: ${truncate(prompt, 50)}`
+  };
+}
+function fieldEquals(fieldPath, expectedValue) {
+  return {
+    type: "custom",
+    check: async (context) => {
+      const result = await checkCondition(
+        { type: "fieldValue", fieldPath, expectedValue },
+        context
+      );
+      return result.terminated;
+    },
+    description: `fieldEquals(${fieldPath}, ${JSON.stringify(expectedValue)})`
+  };
+}
+function fieldIsSet(fieldPath) {
+  return {
+    type: "custom",
+    check: async (context) => {
+      const result = await checkCondition({ type: "fieldSet", fieldPath }, context);
+      return result.terminated;
+    },
+    description: `fieldIsSet(${fieldPath})`
+  };
+}
+
+// src/multi-turn/runner.ts
+import { resolveFileSourcesInInput } from "@agtlantis/core";
+var DEFAULT_MAX_TURNS = 10;
+var DEFAULT_ON_CONDITION_MET = "pass";
+var DEFAULT_ON_MAX_TURNS_REACHED = "fail";
+function aggregateTokenUsage(usages) {
+  return usages.reduce(
+    (acc, usage) => ({
+      inputTokens: acc.inputTokens + usage.inputTokens,
+      outputTokens: acc.outputTokens + usage.outputTokens,
+      totalTokens: acc.totalTokens + usage.totalTokens
+    }),
+    { inputTokens: 0, outputTokens: 0, totalTokens: 0 }
+  );
+}
+function getEffectiveMaxTurns(conditions, safetyLimit) {
+  const maxTurnsCondition = conditions.find((c2) => c2.type === "maxTurns");
+  if (maxTurnsCondition && maxTurnsCondition.type === "maxTurns") {
+    return Math.min(maxTurnsCondition.count, safetyLimit);
+  }
+  return safetyLimit;
+}
+async function resolveInput(followUpInput, context) {
+  const inputValue = followUpInput.input;
+  if (typeof inputValue === "function") {
+    const result = inputValue(context);
+    return result instanceof Promise ? await result : result;
+  }
+  return inputValue;
+}
+function buildContext(currentTurn, history) {
+  return {
+    currentTurn,
+    history,
+    lastOutput: history.length > 0 ? history[history.length - 1].output : void 0
+  };
+}
+function getFollowUpInput(followUpInputs, followUpIndex) {
+  let currentIndex = 0;
+  for (const followUp of followUpInputs) {
+    const repeatCount = followUp.turns ?? 1;
+    if (!Number.isFinite(repeatCount) && followUpIndex >= currentIndex) {
+      return followUp;
+    }
+    if (followUpIndex < currentIndex + repeatCount) {
+      return followUp;
+    }
+    currentIndex += repeatCount;
+  }
+  return null;
+}
+function validateFollowUpInputs(followUpInputs) {
+  for (let i = 0; i < followUpInputs.length; i++) {
+    const followUp = followUpInputs[i];
+    if (followUp.turns === void 0) {
+      continue;
+    }
+    if (typeof followUp.turns !== "number" || followUp.turns < 1) {
+      throw new EvalError("turns must be a positive number or Infinity", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: {
+          description: followUp.description,
+          turns: followUp.turns
+        }
+      });
+    }
+    if (!Number.isFinite(followUp.turns) && i < followUpInputs.length - 1) {
+      throw new EvalError(
+        "turns: Infinity must be the last followUpInput (subsequent items would be unreachable)",
+        {
+          code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+          context: {
+            description: followUp.description,
+            position: i,
+            totalItems: followUpInputs.length
+          }
+        }
+      );
+    }
+  }
+}
+async function getTurnInput(turn, testCaseInput, followUpInputs, conversationHistory) {
+  if (turn === 1) {
+    return { type: "success", input: testCaseInput };
+  }
+  const followUpIndex = turn - 2;
+  const followUp = getFollowUpInput(followUpInputs, followUpIndex);
+  if (!followUp) {
+    return { type: "exhausted" };
+  }
+  const ctx = buildContext(turn, conversationHistory);
+  const input = await resolveInput(followUp, ctx);
+  return { type: "success", input };
+}
+function isFileResolutionError(result) {
+  return "type" in result && result.type === "fileResolutionError";
+}
+async function executeSingleTurn(input, agent, testCaseId, turn) {
+  let resolvedInput;
+  try {
+    resolvedInput = await resolveFileSourcesInInput(input, {
+      basePath: process.cwd()
+    });
+  } catch (e) {
+    return {
+      type: "fileResolutionError",
+      reason: `FileSource resolution failed on turn ${turn}: ${e instanceof Error ? e.message : String(e)}`
+    };
+  }
+  const startTime = performance.now();
+  let output;
+  let metadata;
+  let error;
+  try {
+    const agentResult = await agent.execute(resolvedInput);
+    output = agentResult.result;
+    metadata = agentResult.metadata;
+  } catch (e) {
+    error = EvalError.from(e, "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */, {
+      testCaseId,
+      turn,
+      agentName: agent.config.name
+    });
+  }
+  const latencyMs = performance.now() - startTime;
+  return { output, metadata, latencyMs, error };
+}
+function determinePassFromTermination(termination, onConditionMet, onMaxTurnsReached) {
+  if (!isTerminated(termination)) {
+    return true;
+  }
+  switch (termination.terminationType) {
+    case "error":
+    case "exhausted":
+      return false;
+    case "maxTurns":
+      return onMaxTurnsReached === "pass";
+    case "condition":
+      return onConditionMet === "pass";
+    default:
+      return true;
+  }
+}
+async function executeMultiTurnTestCase(testCase2, context, options) {
+  const { agent, judge, agentDescription } = context;
+  const { multiTurn } = testCase2;
+  const signal = options?.signal;
+  const maxTurns = getEffectiveMaxTurns(
+    multiTurn.terminateWhen,
+    multiTurn.maxTurns ?? DEFAULT_MAX_TURNS
+  );
+  const onConditionMet = multiTurn.onConditionMet ?? DEFAULT_ON_CONDITION_MET;
+  const onMaxTurnsReached = multiTurn.onMaxTurnsReached ?? DEFAULT_ON_MAX_TURNS_REACHED;
+  const followUpInputs = multiTurn.followUpInputs ?? [];
+  validateFollowUpInputs(followUpInputs);
+  const conversationHistory = [];
+  const tokenUsages = [];
+  let totalLatencyMs = 0;
+  let termination = {
+    terminated: false,
+    reason: "Execution not started"
+  };
+  for (let turn = 1; turn <= maxTurns; turn++) {
+    if (signal?.aborted) {
+      throw new EvalError("Multi-turn test execution aborted", {
+        code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+        context: { testCaseId: testCase2.id, turn, reason: "aborted" }
+      });
+    }
+    const inputResult = await getTurnInput(
+      turn,
+      testCase2.input,
+      followUpInputs,
+      conversationHistory
+    );
+    if (inputResult.type === "exhausted") {
+      termination = {
+        terminated: true,
+        terminationType: "exhausted",
+        reason: "All follow-up inputs exhausted"
+      };
+      break;
+    }
+    const input = inputResult.input;
+    const turnResult = await executeSingleTurn(input, agent, testCase2.id ?? "unknown", turn);
+    if (isFileResolutionError(turnResult)) {
+      termination = {
+        terminated: true,
+        terminationType: "error",
+        reason: turnResult.reason
+      };
+      break;
+    }
+    const {
+      output: agentOutput,
+      metadata: agentMetadata,
+      latencyMs,
+      error: agentError
+    } = turnResult;
+    totalLatencyMs += latencyMs;
+    const turnUsage = agentMetadata?.tokenUsage ?? {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0
+    };
+    tokenUsages.push(turnUsage);
+    conversationHistory.push({
+      turn,
+      input,
+      output: agentOutput,
+      metadata: agentMetadata
+    });
+    if (agentError) {
+      termination = {
+        terminated: true,
+        terminationType: "error",
+        reason: `Agent execution failed on turn ${turn}: ${agentError.message}`
+      };
+      break;
+    }
+    const ctx = buildContext(turn, conversationHistory);
+    termination = await checkTermination(multiTurn.terminateWhen, ctx);
+    if (termination.terminated) {
+      break;
+    }
+    if (turn >= maxTurns) {
+      termination = {
+        terminated: true,
+        terminationType: "maxTurns",
+        matchedCondition: { type: "maxTurns", count: maxTurns },
+        reason: `Maximum turns reached (${maxTurns})`
+      };
+      break;
+    }
+  }
+  const aggregatedTokenUsage = aggregateTokenUsage(tokenUsages);
+  const metrics = {
+    latencyMs: totalLatencyMs,
+    tokenUsage: aggregatedTokenUsage
+  };
+  const lastTurn = conversationHistory[conversationHistory.length - 1];
+  const finalOutput = lastTurn?.output;
+  const judgeResult = await judge.evaluate({
+    input: testCase2.input,
+    output: finalOutput,
+    agentDescription,
+    files: testCase2.files
+  });
+  const passedTermination = determinePassFromTermination(
+    termination,
+    onConditionMet,
+    onMaxTurnsReached
+  );
+  const passed = passedTermination && judgeResult.passed;
+  return {
+    testCase: testCase2,
+    output: finalOutput,
+    metrics,
+    verdicts: judgeResult.verdicts,
+    overallScore: judgeResult.overallScore,
+    passed,
+    judgeMetadata: judgeResult.metadata,
+    conversationHistory,
+    termination,
+    totalTurns: conversationHistory.length
+  };
+}
+
+// src/multi-turn/ai-user.ts
+var DEFAULT_SYSTEM_PROMPT = `You are simulating a realistic user in a conversation with an AI assistant.
+
+## Your Role
+Generate natural, context-appropriate user messages based on the conversation history.
+
+## Guidelines
+
+1. **Stay in Character**: Respond as a real user would - with natural language, occasional typos, or casual phrasing when appropriate.
+
+2. **Be Goal-Oriented**: Users have objectives. Pursue them logically based on the conversation context:
+   - If the assistant asks a question, provide a reasonable answer
+   - If clarification is needed, ask for it naturally
+   - If a task is progressing, guide it toward completion
+
+3. **React Appropriately**: Respond to what the assistant says:
+   - Acknowledge when the assistant is helpful
+   - Express confusion if the response is unclear
+   - Correct misunderstandings if they occur
+
+4. **Keep It Realistic**: Real users:
+   - Don't always provide perfect information upfront
+   - May change their mind or add requirements
+   - Sometimes need time to think or decide
+
+## Output Format
+Respond with ONLY the user's message. No additional formatting, explanation, or meta-commentary.`;
+function aiUser(options) {
+  const { provider, systemPrompt, formatHistory, buildInput } = options;
+  const defaultFormatHistory = (ctx) => ctx.history.map(
+    (h, i) => `[Turn ${i + 1}]
+User: ${JSON.stringify(h.input)}
+Assistant: ${JSON.stringify(h.output)}`
+  ).join("\n\n");
+  return async (context) => {
+    const historyText = (formatHistory ?? defaultFormatHistory)(context);
+    const resolvedSystemPrompt = typeof systemPrompt === "function" ? systemPrompt(context) : systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    const userPrompt = historyText ? `## Conversation History
+${historyText}
+
+## Your Task
+Generate the next user message based on the conversation above:` : `## Your Task
+This is the start of a new conversation. Generate an appropriate opening message from the user:`;
+    const execution = provider.simpleExecution(async (session) => {
+      const result = await session.generateText({
+        messages: [
+          { role: "system", content: resolvedSystemPrompt },
+          { role: "user", content: userPrompt }
+        ]
+      });
+      return result.text;
+    });
+    const executionResult = await execution.result();
+    if (executionResult.status !== "succeeded") {
+      throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+    }
+    const responseText = executionResult.value;
+    return buildInput(responseText, context);
+  };
+}
+
+// src/cli/yaml/loader.ts
+async function loadYamlEvalFile(path3, options = {}) {
+  const { basePath = process.cwd(), skipValidation = false } = options;
+  const absolutePath = isAbsolute(path3) ? path3 : resolve2(basePath, path3);
+  if (!existsSync2(absolutePath)) {
+    throw new EvalError(`YAML eval file not found: ${absolutePath}`, {
+      code: "FILE_READ_ERROR" /* FILE_READ_ERROR */,
+      context: { path: path3, absolutePath }
+    });
+  }
+  let content;
+  try {
+    content = await readFile(absolutePath, "utf-8");
+  } catch (error) {
+    throw EvalError.from(error, "FILE_READ_ERROR" /* FILE_READ_ERROR */, {
+      path: path3,
+      absolutePath
+    });
+  }
+  let parsed;
+  try {
+    parsed = parseYaml(content);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new EvalError(`Failed to parse YAML: ${message}`, {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { path: path3, absolutePath },
+      cause: error instanceof Error ? error : void 0
+    });
+  }
+  if (skipValidation) {
+    return parsed;
+  }
+  return validateYamlEvalFile(parsed);
+}
+async function loadYamlEvalFiles(paths, options = {}) {
+  const results = [];
+  for (const path3 of paths) {
+    const content = await loadYamlEvalFile(path3, options);
+    results.push({ path: path3, content });
+  }
+  return results;
+}
+function convertToTestCases(yaml, context) {
+  const { defaults, personas, cases } = yaml;
+  return cases.map((testCase2) => {
+    const merged = mergeWithDefaults(testCase2, defaults);
+    if (isMultiTurnCase(merged)) {
+      return convertToMultiTurnTestCase(merged, personas, context);
+    }
+    return convertToSimpleTestCase(merged);
+  });
+}
+function mergeWithDefaults(testCase2, defaults) {
+  if (!defaults) {
+    return testCase2;
+  }
+  return {
+    ...testCase2,
+    maxTurns: testCase2.maxTurns ?? defaults.maxTurns,
+    endWhen: testCase2.endWhen ?? defaults.endWhen,
+    onConditionMet: testCase2.onConditionMet ?? defaults.onConditionMet,
+    onMaxTurnsReached: testCase2.onMaxTurnsReached ?? defaults.onMaxTurnsReached,
+    tags: [...defaults.tags ?? [], ...testCase2.tags ?? []]
+  };
+}
+function isMultiTurnCase(testCase2) {
+  return testCase2.persona !== void 0 || testCase2.endWhen !== void 0;
+}
+function resolvePersona(ref, personas) {
+  if (ref === void 0) {
+    return void 0;
+  }
+  if (typeof ref === "object") {
+    return ref;
+  }
+  if (!personas || !(ref in personas)) {
+    throw new EvalError(`Persona not found: "${ref}"`, {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: {
+        personaRef: ref,
+        availablePersonas: personas ? Object.keys(personas) : []
+      }
+    });
+  }
+  return personas[ref];
+}
+function convertTerminationCondition(condition, provider) {
+  if (condition.naturalLanguage) {
+    return naturalLanguage({
+      provider,
+      prompt: condition.naturalLanguage
+    });
+  }
+  if (condition.field) {
+    if (condition.equals !== void 0) {
+      return fieldEquals(condition.field, condition.equals);
+    }
+    return fieldIsSet(condition.field);
+  }
+  throw new EvalError("Invalid termination condition: no field or naturalLanguage specified", {
+    code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+    context: { condition }
+  });
+}
+function convertToSimpleTestCase(merged) {
+  return {
+    id: merged.id,
+    description: merged.name ?? merged.description,
+    tags: merged.tags,
+    input: merged.input,
+    expectedOutput: merged.expectedOutput
+  };
+}
+function convertToMultiTurnTestCase(merged, personas, context) {
+  const { provider, buildInput, formatHistory } = context;
+  const persona = resolvePersona(merged.persona, personas);
+  const terminateWhen = [];
+  if (merged.endWhen) {
+    terminateWhen.push(convertTerminationCondition(merged.endWhen, provider));
+  }
+  const followUpInputs = [];
+  if (persona) {
+    const defaultBuildInput = (response) => ({ message: response });
+    const aiUserInput = aiUser({
+      provider,
+      systemPrompt: persona.systemPrompt,
+      formatHistory,
+      buildInput: buildInput ?? defaultBuildInput
+    });
+    const maxTurns = merged.maxTurns ?? 10;
+    for (let i = 0; i < maxTurns - 1; i++) {
+      followUpInputs.push({
+        input: aiUserInput,
+        description: `AI User (${persona.name}) - Turn ${i + 2}`
+      });
+    }
+  }
+  return {
+    id: merged.id,
+    description: merged.name ?? merged.description,
+    tags: merged.tags,
+    input: merged.input,
+    expectedOutput: merged.expectedOutput,
+    multiTurn: {
+      followUpInputs,
+      terminateWhen,
+      maxTurns: merged.maxTurns ?? 10,
+      onConditionMet: merged.onConditionMet ?? "pass",
+      onMaxTurnsReached: merged.onMaxTurnsReached ?? "fail"
+    }
+  };
+}
+
+// src/cli/utils/env.ts
+import { existsSync as existsSync3 } from "fs";
+import { readFile as readFile2 } from "fs/promises";
+import { resolve as resolve3 } from "path";
+async function loadEnvFile(filePath = ".env", cwd = process.cwd()) {
+  const absolutePath = resolve3(cwd, filePath);
+  if (!existsSync3(absolutePath)) {
+    return;
+  }
+  try {
+    const content = await readFile2(absolutePath, "utf-8");
+    const vars = parseEnvFile(content);
+    for (const [key, value] of Object.entries(vars)) {
+      if (process.env[key] === void 0) {
+        process.env[key] = value;
+      }
+    }
+  } catch {
+  }
+}
+function parseEnvFile(content) {
+  const result = {};
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) {
+      continue;
+    }
+    const eqIndex = trimmed.indexOf("=");
+    if (eqIndex === -1) {
+      continue;
+    }
+    const key = trimmed.slice(0, eqIndex).trim();
+    let value = trimmed.slice(eqIndex + 1).trim();
+    if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+      value = value.slice(1, -1);
+    }
+    if (value.includes("\\")) {
+      value = value.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "	").replace(/\\\\/g, "\\");
+    }
+    if (key) {
+      result[key] = value;
+    }
+  }
+  return result;
+}
+
+// src/cli/utils/provider-factory.ts
+import {
+  createOpenAIProvider,
+  createGoogleProvider
+} from "@agtlantis/core";
+import { mock } from "@agtlantis/core/testing";
+
+// src/cli/constants.ts
+var CLI_DEFAULTS = {
+  /** Width for console dividers (═ characters) */
+  DIVIDER_WIDTH: 60,
+  /** Default score used in mock mode for testing */
+  MOCK_DEFAULT_SCORE: 85
+};
+
+// src/cli/output/colors.ts
+var colors = {
+  reset: "\x1B[0m",
+  bold: "\x1B[1m",
+  dim: "\x1B[2m",
+  green: "\x1B[32m",
+  red: "\x1B[31m",
+  yellow: "\x1B[33m",
+  blue: "\x1B[34m",
+  cyan: "\x1B[36m",
+  gray: "\x1B[90m"
+};
+var isColorSupported = process.stdout.isTTY && !process.env.NO_COLOR;
+function c(color, text) {
+  return isColorSupported ? `${colors[color]}${text}${colors.reset}` : text;
+}
+
+// src/cli/output/console.ts
+function printBanner() {
+  console.log();
+  console.log(c("cyan", "  agent-eval"));
+  console.log(c("dim", "  LLM-as-Judge AI Agent Evaluation"));
+  console.log();
+}
+function printProgress(message) {
+  console.log(c("dim", `  ${message}`));
+}
+function printSummary(report, options = {}) {
+  const { summary, results } = report;
+  const { verbose, duration } = options;
+  const passRate = summary.totalTests > 0 ? (summary.passed / summary.totalTests * 100).toFixed(1) : "0.0";
+  const divider = "\u2550".repeat(CLI_DEFAULTS.DIVIDER_WIDTH);
+  console.log();
+  console.log(c("cyan", divider));
+  console.log(c("bold", "  Evaluation Results"));
+  console.log(c("cyan", divider));
+  console.log();
+  console.log(`  ${c("bold", "Total Tests:")}    ${summary.totalTests}`);
+  console.log(
+    `  ${c("bold", "Passed:")}         ${c("green", String(summary.passed))} (${passRate}%)`
+  );
+  console.log(
+    `  ${c("bold", "Failed:")}         ${summary.failed > 0 ? c("red", String(summary.failed)) : "0"}`
+  );
+  console.log(
+    `  ${c("bold", "Average Score:")}  ${summary.avgScore.toFixed(1)}/100`
+  );
+  console.log();
+  console.log(
+    `  ${c("bold", "Total Tokens:")}   ${formatNumber(summary.metrics.totalTokens)}`
+  );
+  console.log(
+    `  ${c("bold", "Avg Latency:")}    ${formatMs(summary.metrics.avgLatencyMs)}`
+  );
+  if (duration !== void 0) {
+    console.log();
+    console.log(`  ${c("bold", "Duration:")}       ${formatMs(duration)}`);
+  }
+  console.log();
+  console.log(c("cyan", divider));
+  if (verbose && results.length > 0) {
+    printVerboseResults(results);
+  }
+}
+function printVerboseResults(results) {
+  console.log();
+  console.log(c("bold", "  Test Results:"));
+  console.log();
+  for (const result of results) {
+    const status = result.passed ? c("green", "\u2713 PASS") : c("red", "\u2717 FAIL");
+    const testId = "testCase" in result && result.testCase?.id ? result.testCase.id : "unknown";
+    console.log(`  ${status}  ${testId}`);
+    console.log(`         Score: ${result.overallScore.toFixed(1)}/100`);
+    if ("criteriaScores" in result && result.criteriaScores) {
+      const scores = result.criteriaScores;
+      for (const score of scores) {
+        console.log(`         ${c("dim", score.criterionId)}: ${score.score.toFixed(1)}`);
+      }
+    }
+    console.log();
+  }
+}
+function printError(error) {
+  console.error();
+  console.error(c("red", "  \u2717 Error:"));
+  console.error();
+  console.error(`  ${error.message}`);
+  console.error();
+}
+function formatNumber(num) {
+  return num.toLocaleString("en-US");
+}
+function formatMs(ms) {
+  if (ms < 1e3) {
+    return `${ms.toFixed(0)}ms`;
+  }
+  return `${(ms / 1e3).toFixed(2)}s`;
+}
+
+// src/cli/utils/provider-factory.ts
+function getApiKeyFromEnv(provider) {
+  if (provider === "openai") {
+    return process.env.OPENAI_API_KEY;
+  }
+  return process.env.GOOGLE_API_KEY;
+}
+function createProviderFromConfig(config) {
+  const { llm } = config;
+  const apiKey = llm.apiKey ?? getApiKeyFromEnv(llm.provider);
+  if (!apiKey) {
+    const envVar = llm.provider === "openai" ? "OPENAI_API_KEY" : "GOOGLE_API_KEY";
+    throw new ConfigError(
+      `API key not found for ${llm.provider}.
+
+Set the ${envVar} environment variable or provide apiKey in config.`,
+      "CONFIG_VALIDATION_ERROR"
+    );
+  }
+  if (llm.provider === "openai") {
+    return createOpenAIProvider({
+      apiKey
+    }).withDefaultModel(llm.defaultModel ?? "gpt-4o-mini");
+  }
+  return createGoogleProvider({
+    apiKey
+  }).withDefaultModel(llm.defaultModel ?? "gemini-1.5-flash");
+}
+function initializeProviders(config, options) {
+  if (options.mock) {
+    printProgress("Using mock Provider (--mock mode)");
+    const mockVerdicts = config.judge.criteria.map((criterion) => ({
+      criterionId: criterion.id,
+      score: CLI_DEFAULTS.MOCK_DEFAULT_SCORE,
+      reasoning: "Mock evaluation - test mode",
+      passed: true
+    }));
+    const mockProvider = mock.provider(mock.json({ verdicts: mockVerdicts }));
+    return { mainProvider: mockProvider, judgeProvider: mockProvider, improverProvider: mockProvider };
+  }
+  const mainProvider = createProviderFromConfig(config);
+  const judgeProvider = config.judge.llm ? createProviderFromConfig({ ...config, llm: config.judge.llm }) : mainProvider;
+  const improverProvider = config.improver?.llm ? createProviderFromConfig({ ...config, llm: config.improver.llm }) : mainProvider;
+  return { mainProvider, judgeProvider, improverProvider };
+}
+
+// src/cli/output/report.ts
+import { mkdir, writeFile as writeFile2 } from "fs/promises";
+import { existsSync as existsSync4 } from "fs";
+import { join } from "path";
+
+// src/reporter/markdown.ts
+import { writeFile } from "fs/promises";
+import { getFileSourcesDisplayInfo } from "@agtlantis/core";
+var PASS_ICON = "\u2705";
+var FAIL_ICON = "\u274C";
+var PRIORITY_ORDER = { high: 0, medium: 1, low: 2 };
+function reportToMarkdown(report, options = {}) {
+  const {
+    expandPassedTests = false,
+    includeRawOutput = false,
+    outputPreviewLength = 200
+  } = options;
+  const { summary, results, suggestions, generatedAt, promptVersion } = report;
+  const passRate = summary.totalTests > 0 ? (summary.passed / summary.totalTests * 100).toFixed(1) : "0.0";
+  const lines = [];
+  lines.push("# Evaluation Report");
+  lines.push("");
+  lines.push(`> Generated: ${generatedAt.toISOString()}`);
+  lines.push(`> Prompt Version: ${promptVersion}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push("");
+  lines.push(`| Metric | Value |`);
+  lines.push(`|--------|-------|`);
+  lines.push(`| Total Tests | ${summary.totalTests} |`);
+  if (summary.iterations && summary.iterations > 1) {
+    lines.push(`| **Iterations** | **${summary.iterations}** |`);
+  }
+  lines.push(`| Passed | ${summary.passed} (${passRate}%) |`);
+  lines.push(`| Failed | ${summary.failed} |`);
+  if (summary.avgStdDev !== void 0) {
+    lines.push(
+      `| Average Score | ${summary.avgScore.toFixed(1)} \xB1 ${summary.avgStdDev.toFixed(1)} |`
+    );
+  } else {
+    lines.push(`| Average Score | ${summary.avgScore.toFixed(1)} |`);
+  }
+  if (summary.avgPassRate !== void 0) {
+    lines.push(`| Avg Pass Rate | ${(summary.avgPassRate * 100).toFixed(1)}% |`);
+  }
+  lines.push(`| Avg Latency | ${summary.metrics.avgLatencyMs.toFixed(0)}ms |`);
+  lines.push(`| Total Tokens | ${summary.metrics.totalTokens} |`);
+  if (summary.costSummary?.total !== void 0) {
+    lines.push(`| Est. Cost | $${summary.costSummary.total.toFixed(4)} |`);
+  }
+  lines.push("");
+  const failedResults = results.filter((r) => !r.passed);
+  if (failedResults.length > 0) {
+    lines.push(`## ${FAIL_ICON} Failed Tests`);
+    lines.push("");
+    for (const result of failedResults) {
+      lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+    }
+  }
+  const passedResults = results.filter((r) => r.passed);
+  if (passedResults.length > 0) {
+    lines.push(`## ${PASS_ICON} Passed Tests`);
+    lines.push("");
+    if (expandPassedTests) {
+      for (const result of passedResults) {
+        lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+      }
+    } else {
+      lines.push("<details>");
+      lines.push("<summary>Click to expand passed tests</summary>");
+      lines.push("");
+      for (const result of passedResults) {
+        lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+      }
+      lines.push("</details>");
+      lines.push("");
+    }
+  }
+  if (suggestions.length > 0) {
+    lines.push("## \u{1F4A1} Improvement Suggestions");
+    lines.push("");
+    const sortedSuggestions = [...suggestions].sort(
+      (a, b) => PRIORITY_ORDER[a.priority] - PRIORITY_ORDER[b.priority]
+    );
+    for (const suggestion of sortedSuggestions) {
+      lines.push(formatSuggestion(suggestion));
+    }
+  }
+  return lines.join("\n");
+}
+function jsonCodeBlock(value, maxLength) {
+  const json = JSON.stringify(value, null, 2);
+  const content = maxLength !== void 0 ? truncate(json, maxLength) : json;
+  return ["```json", content, "```"];
+}
+function passFailIcon(passed) {
+  return passed ? PASS_ICON : FAIL_ICON;
+}
+function formatTestResult(result, previewLength, includeRaw) {
+  const lines = [];
+  const testId = result.testCase.id ?? "unnamed";
+  const scoreDisplay = result.iterationStats ? `${result.overallScore.toFixed(1)} \xB1 ${result.iterationStats.stdDev.toFixed(1)}` : result.overallScore.toFixed(1);
+  lines.push(`### ${testId} (Score: ${scoreDisplay})`);
+  lines.push("");
+  if (result.testCase.description) {
+    lines.push(`> ${result.testCase.description}`);
+    lines.push("");
+  }
+  const fileDisplayInfos = getFileSourcesDisplayInfo(result.testCase.input);
+  if (fileDisplayInfos.length > 0) {
+    lines.push("**Files:**");
+    for (const info of fileDisplayInfos) {
+      const namePrefix = info.filename ? `${info.filename} - ` : "";
+      lines.push(`- ${namePrefix}${info.source}: ${info.description} (${info.mediaType})`);
+    }
+    lines.push("");
+  }
+  if (result.totalTurns !== void 0) {
+    lines.push(
+      `**Multi-turn:** ${result.totalTurns} turns | Termination: ${result.terminationReason ?? "unknown"}`
+    );
+    lines.push("");
+  }
+  if (result.multiTurnIterationStats) {
+    lines.push(...formatMultiTurnIterationStats(result.multiTurnIterationStats));
+  }
+  if (result.iterationStats && result.iterationResults) {
+    lines.push(...formatIterationResults(result.iterationStats, result.iterationResults));
+  }
+  if (result.conversationHistory && result.conversationHistory.length > 0) {
+    lines.push(...formatConversationHistory(result.conversationHistory, previewLength));
+  } else {
+    lines.push(
+      ...formatSingleTurnInputOutput(result.testCase.input, result.output, previewLength)
+    );
+  }
+  lines.push("**Verdicts:**");
+  for (const verdict of result.verdicts) {
+    lines.push(
+      `- ${passFailIcon(verdict.passed)} **${verdict.criterionId}**: ${verdict.score} - ${verdict.reasoning}`
+    );
+  }
+  lines.push("");
+  if (includeRaw) {
+    lines.push("<details>");
+    lines.push("<summary>Raw Output</summary>");
+    lines.push("");
+    lines.push(...jsonCodeBlock(result.output));
+    lines.push("</details>");
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatMultiTurnIterationStats(stats) {
+  const terminationSummary = Object.entries(stats.terminationCounts).map(([type, count]) => `${type}: ${count}`).join(", ") || "none";
+  return [
+    "**Multi-turn Iteration Statistics:**",
+    "",
+    "| Metric | Value |",
+    "|--------|-------|",
+    `| Avg Turns | ${stats.avgTurns.toFixed(1)} |`,
+    `| Min/Max Turns | ${stats.minTurns} / ${stats.maxTurns} |`,
+    `| Termination Distribution | ${terminationSummary} |`,
+    ""
+  ];
+}
+function formatIterationResults(stats, results) {
+  const lines = [
+    "**Iteration Results:**",
+    "",
+    "| # | Score | Passed | Latency |",
+    "|---|-------|--------|---------|"
+  ];
+  results.forEach((iter, idx) => {
+    lines.push(
+      `| ${idx + 1} | ${iter.overallScore.toFixed(1)} | ${passFailIcon(iter.passed)} | ${iter.metrics.latencyMs.toFixed(0)}ms |`
+    );
+  });
+  lines.push("");
+  lines.push(
+    `**Stats:** ${stats.mean.toFixed(1)} \xB1 ${stats.stdDev.toFixed(1)} (min: ${stats.min.toFixed(0)}, max: ${stats.max.toFixed(0)}, pass rate: ${(stats.passRate * 100).toFixed(0)}%)`
+  );
+  lines.push("");
+  return lines;
+}
+function formatConversationHistory(history, previewLength) {
+  const lines = ["**Conversation History:**", ""];
+  for (const turn of history) {
+    lines.push("<details>");
+    lines.push(`<summary>Turn ${turn.turn}</summary>`);
+    lines.push("");
+    lines.push("**Input:**");
+    lines.push(...jsonCodeBlock(turn.input, previewLength));
+    lines.push("");
+    lines.push("**Output:**");
+    lines.push(...jsonCodeBlock(turn.output, previewLength));
+    lines.push("</details>");
+    lines.push("");
+  }
+  return lines;
+}
+function formatSingleTurnInputOutput(input, output, previewLength) {
+  return [
+    "**Input:**",
+    ...jsonCodeBlock(input, previewLength),
+    "",
+    "**Output:**",
+    ...jsonCodeBlock(output, previewLength),
+    ""
+  ];
+}
+function formatSuggestion(suggestion) {
+  const lines = [];
+  const priorityIcon = { high: "\u{1F534}", medium: "\u{1F7E1}", low: "\u{1F7E2}" }[suggestion.priority] ?? "\u26AA";
+  lines.push(`### ${priorityIcon} [${suggestion.priority.toUpperCase()}] ${suggestion.type}`);
+  lines.push("");
+  lines.push(`**Reasoning:** ${suggestion.reasoning}`);
+  lines.push("");
+  lines.push(`**Expected Improvement:** ${suggestion.expectedImprovement}`);
+  lines.push("");
+  lines.push("**Diff:**");
+  lines.push("```diff");
+  lines.push(`- ${suggestion.currentValue.split("\n").join("\n- ")}`);
+  lines.push(`+ ${suggestion.suggestedValue.split("\n").join("\n+ ")}`);
+  lines.push("```");
+  lines.push("");
+  return lines.join("\n");
+}
+
+// src/cli/output/report.ts
+async function generateReport(report, options = {}) {
+  const {
+    dir = "./reports",
+    filename = generateFilename(),
+    markdown
+  } = options;
+  if (!existsSync4(dir)) {
+    await mkdir(dir, { recursive: true });
+  }
+  const content = reportToMarkdown(report, markdown);
+  const outputPath = join(dir, filename);
+  await writeFile2(outputPath, content, "utf-8");
+  return outputPath;
+}
+function generateFilename() {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/:/g, "-").replace(/\.\d{3}Z$/, "");
+  return `eval-${timestamp}.md`;
+}
+
+// src/core/runner.ts
+import { resolveFileSourcesInInput as resolveFileSourcesInInput2 } from "@agtlantis/core";
+
+// src/utils/semaphore.ts
+function createSemaphore(limit) {
+  let running = 0;
+  const waiting = [];
+  return {
+    async acquire() {
+      if (running < limit) {
+        running++;
+        return;
+      }
+      return new Promise((resolve4) => {
+        waiting.push(resolve4);
+      });
+    },
+    release() {
+      running--;
+      const next = waiting.shift();
+      if (next) {
+        running++;
+        next();
+      }
+    }
+  };
+}
+
+// src/core/constants.ts
+var SCORE = {
+  /** Minimum possible score */
+  MIN: 0,
+  /** Maximum possible score */
+  MAX: 100,
+  /** Default threshold for passing evaluation */
+  DEFAULT_PASS_THRESHOLD: 70,
+  /** Threshold for majority-based pass determination (50%) */
+  MAJORITY_PASS_THRESHOLD: 0.5
+};
+var ZERO_TOKEN_USAGE = {
+  inputTokens: 0,
+  outputTokens: 0,
+  totalTokens: 0
+};
+
+// src/core/runner.ts
+async function executeTestCase(testCase2, context, signal) {
+  const { agent, judge, agentDescription } = context;
+  if (signal?.aborted) {
+    throw new EvalError("Test execution aborted", {
+      code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+      context: { testCaseId: testCase2.id, reason: "aborted" }
+    });
+  }
+  let resolvedInput;
+  try {
+    resolvedInput = await resolveFileSourcesInInput2(testCase2.input, {
+      basePath: process.cwd()
+    });
+  } catch (e) {
+    const error2 = EvalError.from(e, "FILE_READ_ERROR" /* FILE_READ_ERROR */, {
+      testCaseId: testCase2.id,
+      agentName: agent.config.name
+    });
+    return createFailedResult(testCase2, error2);
+  }
+  const startTime = performance.now();
+  let output;
+  let tokenUsage = ZERO_TOKEN_USAGE;
+  let error;
+  try {
+    const agentResult = await agent.execute(resolvedInput);
+    output = agentResult.result;
+    if (agentResult.metadata?.tokenUsage) {
+      tokenUsage = agentResult.metadata.tokenUsage;
+    }
+  } catch (e) {
+    error = EvalError.from(e, "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */, {
+      testCaseId: testCase2.id,
+      agentName: agent.config.name
+    });
+    output = void 0;
+  }
+  const latencyMs = performance.now() - startTime;
+  const metrics = { latencyMs, tokenUsage };
+  const testResult = { testCase: testCase2, output, metrics, error };
+  if (error) {
+    return {
+      kind: "single-turn",
+      ...testResult,
+      verdicts: [],
+      overallScore: 0,
+      passed: false,
+      judgeMetadata: void 0
+    };
+  }
+  if (signal?.aborted) {
+    throw new EvalError("Test execution aborted before evaluation", {
+      code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+      context: { testCaseId: testCase2.id, reason: "aborted" }
+    });
+  }
+  const judgeResult = await judge.evaluate({
+    input: testCase2.input,
+    output,
+    agentDescription,
+    files: testCase2.files
+  });
+  return {
+    kind: "single-turn",
+    ...testResult,
+    verdicts: judgeResult.verdicts,
+    overallScore: judgeResult.overallScore,
+    passed: judgeResult.passed,
+    judgeMetadata: judgeResult.metadata
+  };
+}
+function createFailedResult(testCase2, error) {
+  return {
+    kind: "single-turn",
+    testCase: testCase2,
+    output: void 0,
+    metrics: { latencyMs: 0, tokenUsage: ZERO_TOKEN_USAGE },
+    error,
+    verdicts: [],
+    overallScore: 0,
+    passed: false,
+    judgeMetadata: void 0
+  };
+}
+function toMultiTurnResult(result) {
+  return {
+    kind: "multi-turn",
+    testCase: result.testCase,
+    output: result.output,
+    metrics: result.metrics,
+    verdicts: result.verdicts,
+    overallScore: result.overallScore,
+    passed: result.passed,
+    judgeMetadata: result.judgeMetadata,
+    conversationHistory: result.conversationHistory,
+    totalTurns: result.totalTurns,
+    terminationReason: result.termination.reason,
+    termination: result.termination
+  };
+}
+async function runWithConcurrency(testCases2, context, options = {}) {
+  const { concurrency = 1, stopOnFirstFailure = false, signal } = options;
+  if (concurrency < 1) {
+    throw new EvalError("Concurrency must be at least 1", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { concurrency }
+    });
+  }
+  if (testCases2.length === 0) {
+    return [];
+  }
+  const semaphore = createSemaphore(concurrency);
+  const results = [];
+  let shouldStop = false;
+  let firstError;
+  const internalAbort = new AbortController();
+  const propagateExternalAbort = () => {
+    shouldStop = true;
+    internalAbort.abort();
+  };
+  signal?.addEventListener("abort", propagateExternalAbort);
+  if (signal?.aborted) {
+    shouldStop = true;
+  }
+  try {
+    const executeOne = async (testCase2, index) => {
+      if (shouldStop) return;
+      await semaphore.acquire();
+      try {
+        if (shouldStop) return;
+        const result = await executeTestCaseByType(testCase2, context, internalAbort.signal);
+        results[index] = result;
+        if (stopOnFirstFailure && !result.passed) {
+          shouldStop = true;
+          internalAbort.abort();
+        }
+      } catch (e) {
+        if (!firstError && !isAbortError(e)) {
+          firstError = e instanceof Error ? e : new Error(String(e));
+        }
+        shouldStop = true;
+        internalAbort.abort();
+      } finally {
+        semaphore.release();
+      }
+    };
+    const promises = testCases2.map((tc, i) => executeOne(tc, i));
+    await Promise.allSettled(promises);
+    if (firstError) {
+      throw firstError;
+    }
+    return results.filter((r) => r !== void 0);
+  } finally {
+    signal?.removeEventListener("abort", propagateExternalAbort);
+  }
+}
+function isAbortError(e) {
+  return e instanceof DOMException && e.name === "AbortError" || e instanceof EvalError && e.context?.reason === "aborted";
+}
+async function executeTestCaseByType(testCase2, context, signal) {
+  if (isMultiTurnTestCase(testCase2)) {
+    const multiTurnResult = await executeMultiTurnTestCase(testCase2, context, { signal });
+    return toMultiTurnResult(multiTurnResult);
+  }
+  return executeTestCase(testCase2, context, signal);
+}
+
+// src/core/types.ts
+function isMultiTurnResult(result) {
+  return result.kind === "multi-turn" || result.kind === "multi-turn-iterated";
+}
+
+// src/core/iteration.ts
+function calculateIterationStats(results) {
+  if (results.length === 0) {
+    return {
+      iterations: 0,
+      scores: [],
+      mean: 0,
+      stdDev: 0,
+      min: 0,
+      max: 0,
+      passRate: 0,
+      passCount: 0
+    };
+  }
+  const scores = results.map((r) => r.overallScore);
+  const passCount = results.filter((r) => r.passed).length;
+  const mean = scores.reduce((sum, s) => sum + s, 0) / scores.length;
+  const variance = scores.reduce((sum, s) => sum + Math.pow(s - mean, 2), 0) / scores.length;
+  const stdDev = Math.sqrt(variance);
+  return {
+    iterations: results.length,
+    scores,
+    mean,
+    stdDev,
+    min: Math.min(...scores),
+    max: Math.max(...scores),
+    passRate: passCount / results.length,
+    passCount
+  };
+}
+function calculateMultiTurnIterationStats(results) {
+  const baseStats = calculateIterationStats(results);
+  const turns = results.map((r) => r.totalTurns);
+  const terminationCounts = {};
+  for (const r of results) {
+    const type = r.termination.terminationType;
+    if (type) {
+      terminationCounts[type] = (terminationCounts[type] || 0) + 1;
+    }
+  }
+  return {
+    ...baseStats,
+    avgTurns: turns.length > 0 ? turns.reduce((a, b) => a + b, 0) / turns.length : 0,
+    minTurns: turns.length > 0 ? Math.min(...turns) : 0,
+    maxTurns: turns.length > 0 ? Math.max(...turns) : 0,
+    terminationCounts
+  };
+}
+function selectRepresentativeResult(results, mean) {
+  if (results.length === 0) {
+    throw new EvalError("Cannot select representative result from empty array", {
+      code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */
+    });
+  }
+  return results.reduce((closest, current) => {
+    const closestDiff = Math.abs(closest.overallScore - mean);
+    const currentDiff = Math.abs(current.overallScore - mean);
+    return currentDiff < closestDiff ? current : closest;
+  });
+}
+function aggregateIterationResults(allIterationResults) {
+  if (allIterationResults.length === 0) {
+    return [];
+  }
+  const testCount = allIterationResults[0].length;
+  const aggregated = [];
+  for (let i = 0; i < testCount; i++) {
+    const resultsForTestCase = allIterationResults.map((iteration) => iteration[i]);
+    const stats = calculateIterationStats(resultsForTestCase);
+    const representative = selectRepresentativeResult(resultsForTestCase, stats.mean);
+    const isMultiTurn = resultsForTestCase.some((r) => isMultiTurnResult(r));
+    const passedByMajority = stats.passRate >= SCORE.MAJORITY_PASS_THRESHOLD;
+    if (isMultiTurn) {
+      const multiTurnResults = resultsForTestCase.filter(
+        (r) => isMultiTurnResult(r)
+      );
+      const multiTurnRep = representative;
+      const aggregatedResult = {
+        kind: "multi-turn-iterated",
+        testCase: multiTurnRep.testCase,
+        output: multiTurnRep.output,
+        metrics: multiTurnRep.metrics,
+        verdicts: multiTurnRep.verdicts,
+        error: multiTurnRep.error,
+        overallScore: stats.mean,
+        passed: passedByMajority,
+        iterationStats: stats,
+        iterationResults: resultsForTestCase,
+        conversationHistory: multiTurnRep.conversationHistory,
+        totalTurns: multiTurnRep.totalTurns,
+        terminationReason: multiTurnRep.terminationReason,
+        termination: multiTurnRep.termination,
+        multiTurnIterationStats: calculateMultiTurnIterationStats(multiTurnResults)
+      };
+      aggregated.push(aggregatedResult);
+    } else {
+      const aggregatedResult = {
+        kind: "single-turn-iterated",
+        testCase: representative.testCase,
+        output: representative.output,
+        metrics: representative.metrics,
+        verdicts: representative.verdicts,
+        error: representative.error,
+        overallScore: stats.mean,
+        passed: passedByMajority,
+        iterationStats: stats,
+        iterationResults: resultsForTestCase
+      };
+      aggregated.push(aggregatedResult);
+    }
+  }
+  return aggregated;
+}
+function filterIteratedResults(results) {
+  return results.filter(
+    (r) => r.kind === "single-turn-iterated" || r.kind === "multi-turn-iterated"
+  );
+}
+function averageIterationStat(results, selector) {
+  const iteratedResults = filterIteratedResults(results);
+  if (iteratedResults.length === 0) {
+    return void 0;
+  }
+  const total = iteratedResults.reduce((sum, r) => sum + selector(r.iterationStats), 0);
+  return total / iteratedResults.length;
+}
+function calculateAvgStdDev(results) {
+  return averageIterationStat(results, (stats) => stats.stdDev);
+}
+function calculateAvgPassRate(results) {
+  return averageIterationStat(results, (stats) => stats.passRate);
+}
+
+// src/core/suite.ts
+function calculateAggregatedMetrics(results) {
+  if (results.length === 0) {
+    return { avgLatencyMs: 0, totalTokens: 0 };
+  }
+  const totalLatencyMs = sumBy(results, (r) => r.metrics.latencyMs);
+  const totalTokens = sumBy(results, (r) => r.metrics.tokenUsage.totalTokens);
+  return {
+    avgLatencyMs: totalLatencyMs / results.length,
+    totalTokens
+  };
+}
+function sumBy(items, selector) {
+  return items.reduce((sum, item) => sum + selector(item), 0);
+}
+function calculateSummary(results, iterations) {
+  const metrics = calculateAggregatedMetrics(results);
+  const passedCount = results.filter((r) => r.passed).length;
+  const failedCount = results.length - passedCount;
+  const avgScore = results.length > 0 ? sumBy(results, (r) => r.overallScore) / results.length : 0;
+  const summary = {
+    totalTests: results.length,
+    passed: passedCount,
+    failed: failedCount,
+    avgScore,
+    metrics
+  };
+  const hasMultipleIterations = iterations && iterations > 1;
+  if (hasMultipleIterations) {
+    summary.iterations = iterations;
+    summary.avgStdDev = calculateAvgStdDev(results);
+    summary.avgPassRate = calculateAvgPassRate(results);
+  }
+  return summary;
+}
+function createEvalSuite(config) {
+  const { agent, agentDescription, judge, improver } = config;
+  const description = agentDescription ?? agent.config.description ?? agent.config.name;
+  const suite = {
+    async run(testCases2, options) {
+      const iterations = options?.iterations ?? 1;
+      validateIterations(iterations);
+      const executeContext = { agent, judge, agentDescription: description };
+      const results = iterations <= 1 ? await runWithConcurrency(testCases2, executeContext, options) : await runMultipleIterations(testCases2, executeContext, options, iterations);
+      const summary = calculateSummary(results, iterations > 1 ? iterations : void 0);
+      const suggestions = improver ? (await improver.improve(agent.prompt, results)).suggestions : [];
+      return {
+        summary,
+        results,
+        suggestions,
+        generatedAt: /* @__PURE__ */ new Date(),
+        promptVersion: agent.prompt.version
+      };
+    },
+    withAgent(newAgent) {
+      return createEvalSuite({
+        ...config,
+        agent: newAgent,
+        agentDescription: void 0
+      });
+    }
+  };
+  return suite;
+}
+function validateIterations(iterations) {
+  if (iterations < 1 || !Number.isInteger(iterations)) {
+    throw new EvalError(
+      `Invalid iterations value: ${iterations}. Must be a positive integer.`,
+      { code: "INVALID_CONFIG" /* INVALID_CONFIG */, context: { iterations } }
+    );
+  }
+}
+async function runMultipleIterations(testCases2, executeContext, options, iterations) {
+  const allIterationResults = [];
+  for (let i = 0; i < iterations; i++) {
+    const iterationResults = await runWithConcurrency(
+      testCases2,
+      executeContext,
+      { ...options, iterations: void 0 }
+    );
+    allIterationResults.push(iterationResults);
+  }
+  return aggregateIterationResults(allIterationResults);
+}
+
+// src/index.ts
+import {
+  resolveFileSource,
+  resolveFileSourcesInInput as resolveFileSourcesInInput3,
+  scanForFileSources,
+  getFileSourceDisplayInfo,
+  getFileSourcesDisplayInfo as getFileSourcesDisplayInfo2,
+  inferMediaType,
+  isFileSource,
+  isFileSourcePath,
+  isFileSourceData,
+  isFileSourceBase64,
+  isFileSourceUrl
+} from "@agtlantis/core";
+
+// src/judge/llm-judge.ts
+import { Output } from "ai";
+import { z as z3 } from "zod";
+
+// src/judge/prompts/default.ts
+var defaultJudgePrompt = {
+  id: "default-judge",
+  version: "2.0.0",
+  system: `You are an expert evaluator specializing in assessing AI Agent outputs.
+
+Your role is to fairly and thoroughly evaluate the agent's output against the provided criteria.
+
+## Evaluation Principles
+
+1. **Scoring**: Assign a score between 0-100 for each criterion
+   - 90-100: Exceptional - Exceeds expectations with no significant issues
+   - 70-89: Good - Meets expectations with minor issues
+   - 50-69: Acceptable - Partially meets expectations, notable issues present
+   - 30-49: Poor - Falls short of expectations, significant issues
+   - 0-29: Failing - Does not meet minimum requirements
+
+2. **Reasoning**: Always provide specific, evidence-based reasoning
+   - Quote or reference specific parts of the output
+   - Explain both strengths and weaknesses
+   - Be constructive and actionable in feedback
+
+3. **Objectivity**: Evaluate based solely on the criteria provided
+   - Avoid personal preferences or unstated requirements
+   - Consider the agent's intended purpose and context
+   - Weight severity of issues proportionally
+
+## Response Format
+
+You MUST respond with valid JSON only. No additional text or explanation outside the JSON structure.
+
+{
+  "verdicts": [
+    {
+      "criterionId": "criterion-id",
+      "score": 0-100,
+      "reasoning": "Detailed explanation with specific evidence from the output",
+      "passed": true/false
+    }
+  ]
+}`,
+  renderUserPrompt: (ctx) => {
+    const fileSection = buildFileSection(ctx.files);
+    return `
+## Agent Under Evaluation
+${ctx.agentDescription}
+
+## Input Provided to Agent
+\`\`\`json
+${JSON.stringify(ctx.input, null, 2)}
+\`\`\`
+${fileSection}
+## Agent Output
+\`\`\`json
+${JSON.stringify(ctx.output, null, 2)}
+\`\`\`
+
+## Evaluation Criteria
+${ctx.criteria.map((c2) => `- **${c2.name}** (id: ${c2.id}, weight: ${c2.weight ?? 1}): ${c2.description}`).join("\n")}
+
+Please evaluate the agent's output against each criterion listed above.`.trim();
+  }
+};
+function buildFileSection(files) {
+  if (!files || files.length === 0) {
+    return "";
+  }
+  return `
+## Reference Files
+${files.map((f) => `### ${f.path}
+\`\`\`
+${f.content}
+\`\`\``).join("\n\n")}
+`;
+}
+
+// src/judge/llm-judge.ts
+function toEvalTokenUsage(usage) {
+  return {
+    inputTokens: usage.inputTokens ?? 0,
+    outputTokens: usage.outputTokens ?? 0,
+    totalTokens: usage.totalTokens ?? 0
+  };
+}
+function hasValidator(criterion) {
+  return "validator" in criterion && typeof criterion.validator === "function";
+}
+var JudgeResponseSchema = z3.object({
+  verdicts: z3.array(
+    z3.object({
+      criterionId: z3.string(),
+      score: z3.number().min(SCORE.MIN).max(SCORE.MAX),
+      reasoning: z3.string(),
+      passed: z3.boolean().optional()
+    })
+  )
+});
+function validateAllCriteriaHaveVerdicts(verdicts, criteriaIds) {
+  const providedIds = new Set(verdicts.map((v) => v.criterionId));
+  const missingIds = criteriaIds.filter((id) => !providedIds.has(id));
+  if (missingIds.length > 0) {
+    throw new EvalError("Judge response missing verdicts for some criteria", {
+      code: "VERDICT_PARSE_ERROR" /* VERDICT_PARSE_ERROR */,
+      context: { missingCriteriaIds: missingIds, providedIds: [...providedIds] }
+    });
+  }
+}
+function calculateOverallScore(verdicts, criteriaWeights) {
+  let totalWeight = 0;
+  let weightedSum = 0;
+  for (const verdict of verdicts) {
+    const weight = criteriaWeights.get(verdict.criterionId) ?? 1;
+    weightedSum += verdict.score * weight;
+    totalWeight += weight;
+  }
+  if (totalWeight === 0) {
+    return 0;
+  }
+  return Math.round(weightedSum / totalWeight * 100) / 100;
+}
+function runValidatorCriteria(validatorCriteria, output) {
+  return validatorCriteria.map((criterion) => {
+    const result = criterion.validator(output);
+    if (result.valid) {
+      return {
+        criterionId: criterion.id,
+        score: 100,
+        reasoning: `${criterion.name} \uD1B5\uACFC`,
+        passed: true
+      };
+    }
+    return {
+      criterionId: criterion.id,
+      score: 0,
+      reasoning: `${criterion.name} \uC2E4\uD328:
+${result.errorSummary ?? "\uC720\uD6A8\uC131 \uAC80\uC99D \uC624\uB958"}`,
+      passed: false
+    };
+  });
+}
+async function runLLMEvaluation(provider, prompt, context, llmCriteriaIds, passThreshold) {
+  const messages = [
+    { role: "system", content: prompt.system },
+    { role: "user", content: prompt.renderUserPrompt(context) }
+  ];
+  let response;
+  let usage;
+  try {
+    const execution = provider.simpleExecution(async (session) => {
+      const result = await session.generateText({
+        messages,
+        output: Output.object({ schema: JudgeResponseSchema })
+      });
+      return result.output;
+    });
+    const executionResult = await execution.result();
+    if (executionResult.status !== "succeeded") {
+      throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+    }
+    response = executionResult.value;
+    usage = executionResult.summary.totalLLMUsage;
+  } catch (cause) {
+    throw EvalError.from(cause, "LLM_API_ERROR" /* LLM_API_ERROR */, {
+      promptId: prompt.id,
+      promptVersion: prompt.version
+    });
+  }
+  validateAllCriteriaHaveVerdicts(response.verdicts, llmCriteriaIds);
+  const verdicts = response.verdicts.map((v) => ({
+    criterionId: v.criterionId,
+    score: v.score,
+    reasoning: v.reasoning,
+    passed: v.passed ?? v.score >= passThreshold
+  }));
+  return { verdicts, usage };
+}
+function createJudge(config) {
+  const {
+    provider,
+    prompt = defaultJudgePrompt,
+    criteria,
+    passThreshold = SCORE.DEFAULT_PASS_THRESHOLD,
+    model
+  } = config;
+  const validatorCriteria = [];
+  const llmCriteria = [];
+  const criteriaWeights = /* @__PURE__ */ new Map();
+  const llmCriteriaIds = [];
+  for (const c2 of criteria) {
+    criteriaWeights.set(c2.id, c2.weight ?? 1);
+    if (hasValidator(c2)) {
+      validatorCriteria.push(c2);
+    } else {
+      llmCriteria.push(c2);
+      llmCriteriaIds.push(c2.id);
+    }
+  }
+  return {
+    async evaluate(evalContext) {
+      const { input, output, agentDescription, files } = evalContext;
+      const validatorVerdicts = runValidatorCriteria(validatorCriteria, output);
+      let llmVerdicts = [];
+      let llmUsage;
+      if (llmCriteria.length > 0) {
+        const context = {
+          agentDescription,
+          input,
+          output,
+          criteria: llmCriteria,
+          files
+        };
+        const llmResult = await runLLMEvaluation(
+          provider,
+          prompt,
+          context,
+          llmCriteriaIds,
+          passThreshold
+        );
+        llmVerdicts = llmResult.verdicts;
+        llmUsage = llmResult.usage;
+      }
+      const allVerdicts = [...validatorVerdicts, ...llmVerdicts];
+      const overallScore = calculateOverallScore(allVerdicts, criteriaWeights);
+      const passed = overallScore >= passThreshold;
+      const metadata = llmUsage ? { tokenUsage: toEvalTokenUsage(llmUsage), model } : void 0;
+      return {
+        verdicts: allVerdicts,
+        overallScore,
+        passed,
+        metadata
+      };
+    }
+  };
+}
+
+// src/reporter/json-reporter.ts
+import { writeFileSync } from "fs";
+
+// src/reporter/cost-helpers.ts
+import {
+  calculateCostFromUsage
+} from "@agtlantis/core";
+function toLanguageModelUsage(usage) {
+  return {
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    totalTokens: usage.totalTokens
+  };
+}
+var PROVIDER_MAPPING = {
+  gemini: "google",
+  openai: "openai",
+  anthropic: "anthropic",
+  google: "google"
+};
+function detectProvider(model) {
+  if (!model) return "google";
+  if (model.startsWith("gpt-") || model.startsWith("o1") || model.startsWith("o3")) {
+    return "openai";
+  }
+  if (model.startsWith("gemini-")) {
+    return "google";
+  }
+  if (model.startsWith("claude-")) {
+    return "anthropic";
+  }
+  return "google";
+}
+function normalizeProvider(provider) {
+  if (!provider) return "google";
+  return PROVIDER_MAPPING[provider] ?? provider;
+}
+function calculateComponentCost(tokenUsage, model, provider, config) {
+  if (!tokenUsage) return void 0;
+  const normalizedProvider = provider ? normalizeProvider(provider) : detectProvider(model);
+  const providerPricing = config?.providerPricing?.[normalizedProvider];
+  const result = calculateCostFromUsage(
+    toLanguageModelUsage(tokenUsage),
+    model ?? "unknown",
+    normalizedProvider,
+    providerPricing
+  );
+  return result.total;
+}
+function buildCostBreakdown(costs) {
+  const total = (costs.agent ?? 0) + (costs.judge ?? 0) + (costs.improver ?? 0);
+  return {
+    ...costs,
+    total: total > 0 ? total : void 0
+  };
+}
+function calculateResultCost(result, config) {
+  const agentCost = calculateComponentCost(
+    result.metrics.tokenUsage,
+    result.agentMetadata?.model,
+    result.agentMetadata?.provider,
+    config
+  );
+  const judgeCost = result.judgeMetadata?.tokenUsage ? calculateComponentCost(
+    result.judgeMetadata.tokenUsage,
+    result.judgeMetadata.model,
+    result.judgeMetadata.provider,
+    config
+  ) : void 0;
+  return buildCostBreakdown({
+    agent: agentCost,
+    judge: judgeCost
+  });
+}
+function calculateReportCosts(report, config) {
+  let totalAgent = 0;
+  let totalJudge = 0;
+  for (const result of report.results) {
+    const breakdown = calculateResultCost(result, config);
+    totalAgent += breakdown.agent ?? 0;
+    totalJudge += breakdown.judge ?? 0;
+  }
+  return {
+    total: totalAgent + totalJudge,
+    byComponent: {
+      agent: totalAgent,
+      judge: totalJudge
+    }
+  };
+}
+
+// src/reporter/format-utils.ts
+import { mkdirSync } from "fs";
+import path from "path";
+
+// src/reporter/markdown-reporter.ts
+import { writeFileSync as writeFileSync2 } from "fs";
+
+// src/reporter/cycle-json.ts
+import { writeFileSync as writeFileSync3, mkdirSync as mkdirSync2 } from "fs";
+import path2 from "path";
+
+// src/reporter/cycle-markdown.ts
+import { writeFileSync as writeFileSync4 } from "fs";
+
+// src/improver/utils.ts
+import { compileTemplate } from "@agtlantis/core";
+function safeReplace(str, search, replacement) {
+  return str.replace(search, () => replacement);
+}
+function bumpVersion(version, bump) {
+  const parts = version.split(".").map((n) => parseInt(n, 10));
+  if (parts.length !== 3 || parts.some(isNaN)) {
+    throw new EvalError(
+      `Invalid version format: "${version}". Expected semver format (x.y.z)`,
+      {
+        code: "SUGGESTION_APPLY_ERROR" /* SUGGESTION_APPLY_ERROR */,
+        context: { version, expectedFormat: "x.y.z" }
+      }
+    );
+  }
+  const [major, minor, patch] = parts;
+  switch (bump) {
+    case "major":
+      return `${major + 1}.0.0`;
+    case "minor":
+      return `${major}.${minor + 1}.0`;
+    case "patch":
+      return `${major}.${minor}.${patch + 1}`;
+  }
+}
+function applyPromptSuggestions(currentPrompt, suggestions, options) {
+  const approvedSuggestions = suggestions.filter((s) => s.approved);
+  if (approvedSuggestions.length === 0) {
+    return {
+      prompt: currentPrompt,
+      appliedCount: 0,
+      skipped: []
+    };
+  }
+  let newPrompt = { ...currentPrompt };
+  let appliedCount = 0;
+  const skipped = [];
+  for (const suggestion of approvedSuggestions) {
+    const applyResult = applySingleSuggestion(newPrompt, suggestion);
+    if (applyResult.success) {
+      newPrompt = applyResult.prompt;
+      appliedCount++;
+    } else {
+      skipped.push({ suggestion, reason: applyResult.reason });
+    }
+  }
+  if (options?.bumpVersion && appliedCount > 0) {
+    newPrompt = {
+      ...newPrompt,
+      version: bumpVersion(currentPrompt.version, options.bumpVersion)
+    };
+  }
+  return {
+    prompt: newPrompt,
+    appliedCount,
+    skipped
+  };
+}
+var AGENT_PROMPT_CORE_FIELDS = [
+  "id",
+  "version",
+  "system",
+  "renderUserPrompt",
+  "userTemplate"
+];
+function applySingleSuggestion(prompt, suggestion) {
+  switch (suggestion.type) {
+    case "system_prompt": {
+      if (!prompt.system.includes(suggestion.currentValue)) {
+        return {
+          success: false,
+          reason: `currentValue not found in system prompt: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      return {
+        success: true,
+        prompt: {
+          ...prompt,
+          system: safeReplace(
+            prompt.system,
+            suggestion.currentValue,
+            suggestion.suggestedValue
+          )
+        }
+      };
+    }
+    case "user_prompt": {
+      const userTemplate = prompt.userTemplate;
+      if (typeof userTemplate !== "string") {
+        throw new EvalError(
+          `Cannot apply user_prompt suggestion: prompt does not have a userTemplate field. The renderUserPrompt is a function and cannot be modified directly.`,
+          {
+            code: "SUGGESTION_APPLY_ERROR" /* SUGGESTION_APPLY_ERROR */,
+            context: {
+              suggestionType: suggestion.type,
+              hasUserTemplate: "userTemplate" in prompt
+            }
+          }
+        );
+      }
+      if (!userTemplate.includes(suggestion.currentValue)) {
+        return {
+          success: false,
+          reason: `currentValue not found in userTemplate: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      const newTemplate = safeReplace(
+        userTemplate,
+        suggestion.currentValue,
+        suggestion.suggestedValue
+      );
+      return {
+        success: true,
+        prompt: {
+          ...prompt,
+          userTemplate: newTemplate,
+          renderUserPrompt: compileTemplate(newTemplate, prompt.id)
+        }
+      };
+    }
+    case "parameters": {
+      const updatedPrompt = { ...prompt };
+      let found = false;
+      for (const [key, value] of Object.entries(updatedPrompt)) {
+        if (AGENT_PROMPT_CORE_FIELDS.includes(
+          key
+        )) {
+          continue;
+        }
+        if (typeof value === "string" && value.includes(suggestion.currentValue)) {
+          updatedPrompt[key] = safeReplace(
+            value,
+            suggestion.currentValue,
+            suggestion.suggestedValue
+          );
+          found = true;
+          break;
+        }
+      }
+      if (!found) {
+        return {
+          success: false,
+          reason: `currentValue not found in any parameter field: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      return {
+        success: true,
+        prompt: updatedPrompt
+      };
+    }
+    default: {
+      const _exhaustive = suggestion.type;
+      return {
+        success: false,
+        reason: `Unknown suggestion type: ${suggestion.type}`
+      };
+    }
+  }
+}
+
+// src/improver/llm-improver.ts
+import { Output as Output2 } from "ai";
+import { z as z4 } from "zod";
+
+// src/improver/prompts/default.ts
+var defaultImproverPrompt = {
+  id: "default-improver",
+  version: "2.0.0",
+  system: `You are an expert prompt engineer specializing in optimizing AI Agent prompts.
+
+Your role is to analyze test results and evaluation feedback to propose targeted improvements.
+
+## Improvement Principles
+
+1. **Focus on Impact**: Prioritize changes that address the lowest-scoring criteria
+   - Target specific failure patterns, not general improvements
+   - One well-crafted change is better than many superficial ones
+
+2. **Be Specific and Actionable**: Provide concrete changes, not vague suggestions
+   - Show exact text to add, modify, or remove
+   - Explain the mechanism by which the change will help
+
+3. **Consider Trade-offs**: Evaluate side effects of each change
+   - Will this fix break other test cases?
+   - Does it increase prompt length/cost significantly?
+   - Could it introduce new failure modes?
+
+4. **Maintain Prompt Quality**: Preserve clarity and structure
+   - Keep prompts readable and maintainable
+   - Avoid over-engineering or excessive constraints
+   - Ensure changes align with the agent's core purpose
+
+## Suggestion Priority Levels
+- **high**: Critical issues causing test failures, should be addressed immediately
+- **medium**: Issues affecting quality scores, recommended for next iteration
+- **low**: Minor optimizations, nice-to-have improvements
+
+## Response Format
+
+You MUST respond with valid JSON only. No additional text outside the JSON structure.
+
+{
+  "suggestions": [
+    {
+      "type": "system_prompt" | "user_prompt" | "parameters",
+      "priority": "high" | "medium" | "low",
+      "currentValue": "The specific text or value being changed",
+      "suggestedValue": "The proposed replacement text or value",
+      "reasoning": "Why this change addresses the identified issue",
+      "expectedImprovement": "Predicted impact on scores and behavior"
+    }
+  ]
+}`,
+  renderUserPrompt: (ctx) => {
+    const failedDetails = buildFailedCaseDetails(ctx.evaluatedResults);
+    return `
+## Current Agent Prompt
+
+### System Prompt
+\`\`\`
+${ctx.agentPrompt.system}
+\`\`\`
+
+## Test Results Summary
+- Total tests: ${ctx.evaluatedResults.length}
+- Passed: ${ctx.evaluatedResults.filter((r) => r.passed).length}
+- Failed: ${ctx.evaluatedResults.filter((r) => !r.passed).length}
+
+## Performance Metrics
+- Average latency: ${ctx.aggregatedMetrics.avgLatencyMs}ms
+- Total tokens used: ${ctx.aggregatedMetrics.totalTokens}
+
+## Failed/Low-Score Cases Details
+${failedDetails}
+
+Based on the above results, please propose specific prompt improvements.`.trim();
+  }
+};
+function buildFailedCaseDetails(results) {
+  const failedOrLowScore = results.filter((r) => !r.passed || r.overallScore < 70);
+  if (failedOrLowScore.length === 0) {
+    return "(None - all tests passed with acceptable scores)";
+  }
+  return failedOrLowScore.map(
+    (r) => `
+### ${r.testCase.id ?? "unnamed"} (Score: ${r.overallScore})
+**Input:** ${truncate(JSON.stringify(r.testCase.input), 200)}
+**Output:** ${truncate(JSON.stringify(r.output), 200)}
+**Evaluation:**
+${r.verdicts.map((v) => `- ${v.criterionId}: ${v.score}/100 - ${v.reasoning}`).join("\n")}`
+  ).join("\n");
+}
+
+// src/improver/llm-improver.ts
+function toEvalTokenUsage2(usage) {
+  return {
+    inputTokens: usage.inputTokens ?? 0,
+    outputTokens: usage.outputTokens ?? 0,
+    totalTokens: usage.totalTokens ?? 0
+  };
+}
+var ImproverResponseSchema = z4.object({
+  suggestions: z4.array(
+    z4.object({
+      type: z4.enum(["system_prompt", "user_prompt", "parameters"]),
+      priority: z4.enum(["high", "medium", "low"]),
+      currentValue: z4.string(),
+      suggestedValue: z4.string(),
+      reasoning: z4.string(),
+      expectedImprovement: z4.string()
+    })
+  )
+});
+function aggregateMetrics(results) {
+  if (results.length === 0) {
+    return {
+      avgLatencyMs: 0,
+      totalTokens: 0
+    };
+  }
+  let totalLatency = 0;
+  let totalTokens = 0;
+  for (const result of results) {
+    totalLatency += result.metrics.latencyMs;
+    totalTokens += result.metrics.tokenUsage.totalTokens;
+  }
+  return {
+    avgLatencyMs: Math.round(totalLatency / results.length),
+    totalTokens
+  };
+}
+function createImprover(config) {
+  const { provider, prompt = defaultImproverPrompt, model } = config;
+  return {
+    async improve(agentPrompt, results) {
+      const context = {
+        agentPrompt,
+        evaluatedResults: results,
+        aggregatedMetrics: aggregateMetrics(results)
+      };
+      const messages = [
+        { role: "system", content: prompt.system },
+        { role: "user", content: prompt.renderUserPrompt(context) }
+      ];
+      let response;
+      let llmUsage;
+      try {
+        const execution = provider.simpleExecution(async (session) => {
+          const result = await session.generateText({
+            messages,
+            output: Output2.object({ schema: ImproverResponseSchema })
+          });
+          return result.output;
+        });
+        const executionResult = await execution.result();
+        if (executionResult.status !== "succeeded") {
+          throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+        }
+        response = executionResult.value;
+        llmUsage = executionResult.summary.totalLLMUsage;
+      } catch (cause) {
+        throw EvalError.from(cause, "LLM_API_ERROR" /* LLM_API_ERROR */, {
+          promptId: prompt.id,
+          promptVersion: prompt.version
+        });
+      }
+      const suggestions = response.suggestions.map((s) => ({
+        ...s,
+        approved: void 0,
+        modified: void 0
+      }));
+      const metadata = llmUsage ? { tokenUsage: toEvalTokenUsage2(llmUsage), model } : void 0;
+      return { suggestions, metadata };
+    }
+  };
+}
+
+// src/index.ts
+import { mock as mock2, MockProvider } from "@agtlantis/core/testing";
+import {
+  compileTemplate as compileTemplate3,
+  createFilePromptRepository
+} from "@agtlantis/core";
+import {
+  calculateCostFromUsage as calculateCostFromUsage3,
+  OPENAI_PRICING,
+  GOOGLE_PRICING,
+  ANTHROPIC_PRICING,
+  DEFAULT_PRICING_CONFIG
+} from "@agtlantis/core";
+
+// src/improvement-cycle/types.ts
+function isTargetScoreCondition(condition) {
+  return condition.type === "targetScore";
+}
+function isMaxRoundsCondition(condition) {
+  return condition.type === "maxRounds";
+}
+function isNoImprovementCondition(condition) {
+  return condition.type === "noImprovement";
+}
+function isMaxCostCondition(condition) {
+  return condition.type === "maxCost";
+}
+function isCustomCycleCondition(condition) {
+  return condition.type === "custom";
+}
+
+// src/improvement-cycle/conditions.ts
+function targetScore(threshold) {
+  if (!Number.isFinite(threshold)) {
+    throw new EvalError("threshold must be a finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { threshold }
+    });
+  }
+  if (threshold < 0 || threshold > 100) {
+    throw new EvalError("threshold must be between 0 and 100", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { threshold }
+    });
+  }
+  return { type: "targetScore", threshold };
+}
+function maxRounds(count) {
+  if (!Number.isInteger(count) || count < 1) {
+    throw new EvalError("count must be a positive integer", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { count }
+    });
+  }
+  return { type: "maxRounds", count };
+}
+function noImprovement(consecutiveRounds, minDelta) {
+  if (!Number.isInteger(consecutiveRounds) || consecutiveRounds < 1) {
+    throw new EvalError("consecutiveRounds must be a positive integer", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { consecutiveRounds }
+    });
+  }
+  if (minDelta !== void 0 && (!Number.isFinite(minDelta) || minDelta < 0)) {
+    throw new EvalError("minDelta must be a non-negative finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { minDelta }
+    });
+  }
+  return {
+    type: "noImprovement",
+    consecutiveRounds,
+    ...minDelta !== void 0 && { minDelta }
+  };
+}
+function maxCost(maxUSD) {
+  if (!Number.isFinite(maxUSD) || maxUSD <= 0) {
+    throw new EvalError("maxUSD must be a positive finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { maxUSD }
+    });
+  }
+  return { type: "maxCost", maxUSD };
+}
+function checkTargetScore(condition, ctx) {
+  if (ctx.latestScore >= condition.threshold) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Target score ${condition.threshold} reached (current: ${ctx.latestScore})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Score ${ctx.latestScore} below target ${condition.threshold}`
+  };
+}
+function checkMaxRounds(condition, ctx) {
+  if (ctx.currentRound >= condition.count) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Maximum rounds reached (${condition.count})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Round ${ctx.currentRound} of ${condition.count}`
+  };
+}
+function checkNoImprovement(condition, ctx) {
+  const { consecutiveRounds, minDelta = 0 } = condition;
+  const { history } = ctx;
+  let noImprovementCount = 0;
+  for (let i = history.length - 1; i >= 0; i--) {
+    const round = history[i];
+    if (round.scoreDelta === null) break;
+    if (round.scoreDelta <= minDelta) {
+      noImprovementCount++;
+    } else {
+      break;
+    }
+  }
+  if (noImprovementCount >= consecutiveRounds) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `No improvement for ${noImprovementCount} consecutive round${noImprovementCount === 1 ? "" : "s"}`
+    };
+  }
+  const roundWord = noImprovementCount === 1 ? "round" : "rounds";
+  return {
+    terminated: false,
+    reason: `${noImprovementCount} ${roundWord} without improvement (need ${consecutiveRounds})`
+  };
+}
+function checkMaxCost(condition, ctx) {
+  if (ctx.totalCost >= condition.maxUSD) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Cost limit exceeded ($${ctx.totalCost.toFixed(2)} >= $${condition.maxUSD.toFixed(2)})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Cost $${ctx.totalCost.toFixed(2)} under limit $${condition.maxUSD.toFixed(2)}`
+  };
+}
+async function checkCustomCondition(condition, ctx) {
+  const description = condition.description ?? "Custom condition";
+  try {
+    const shouldTerminate = await condition.check(ctx);
+    if (shouldTerminate) {
+      return {
+        terminated: true,
+        matchedCondition: condition,
+        reason: `${description} met`
+      };
+    }
+    return {
+      terminated: false,
+      reason: `${description} not met`
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      terminated: false,
+      reason: `${description} check failed: ${message}`
+    };
+  }
+}
+async function checkCycleCondition(condition, context) {
+  if (isTargetScoreCondition(condition)) {
+    return checkTargetScore(condition, context);
+  }
+  if (isMaxRoundsCondition(condition)) {
+    return checkMaxRounds(condition, context);
+  }
+  if (isNoImprovementCondition(condition)) {
+    return checkNoImprovement(condition, context);
+  }
+  if (isMaxCostCondition(condition)) {
+    return checkMaxCost(condition, context);
+  }
+  if (isCustomCycleCondition(condition)) {
+    return checkCustomCondition(condition, context);
+  }
+  const _exhaustive = condition;
+  throw new EvalError(`Unknown condition type: ${JSON.stringify(_exhaustive)}`, {
+    code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */,
+    context: { condition: _exhaustive }
+  });
+}
+async function checkCycleTermination(conditions, context) {
+  if (conditions.length === 0) {
+    return {
+      terminated: false,
+      reason: "No termination conditions specified"
+    };
+  }
+  for (const condition of conditions) {
+    const result = await checkCycleCondition(condition, context);
+    if (result.terminated) {
+      return result;
+    }
+  }
+  return {
+    terminated: false,
+    reason: "No termination conditions met"
+  };
+}
+
+// src/improvement-cycle/runner.ts
+import { calculateCostFromUsage as calculateCostFromUsage2 } from "@agtlantis/core";
+
+// src/improvement-cycle/history.ts
+import crypto from "crypto";
+import { existsSync as existsSync5 } from "fs";
+import { mkdir as mkdir2, readFile as readFile3, writeFile as writeFile3 } from "fs/promises";
+import { dirname } from "path";
+import { compileTemplate as compileTemplate2 } from "@agtlantis/core";
+var defaultHistoryStorage = {
+  readFile: (path3) => readFile3(path3, "utf-8"),
+  writeFile: (path3, content) => writeFile3(path3, content, "utf-8"),
+  exists: existsSync5,
+  mkdir: (path3, options) => mkdir2(path3, options)
+};
+function hasUserTemplate(prompt) {
+  return typeof prompt.userTemplate === "string";
+}
+function serializePrompt(prompt) {
+  const p = prompt;
+  if (!hasUserTemplate(p)) {
+    throw new EvalError("Cannot serialize prompt: userTemplate field is required", {
+      code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+      context: { promptId: p.id }
+    });
+  }
+  const { id, version, system, userTemplate, renderUserPrompt, ...rest } = p;
+  const customFields = Object.keys(rest).length > 0 ? rest : void 0;
+  return {
+    id,
+    version,
+    system,
+    userTemplate,
+    ...customFields && { customFields }
+  };
+}
+function validateDeserializedPrompt(obj, promptId) {
+  const requiredStrings = ["id", "version", "system", "userTemplate"];
+  for (const field of requiredStrings) {
+    if (typeof obj[field] !== "string") {
+      throw new EvalError(`Invalid deserialized prompt: ${field} must be a string`, {
+        code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+        context: { promptId, field, actual: typeof obj[field] }
+      });
+    }
+  }
+  if (typeof obj.renderUserPrompt !== "function") {
+    throw new EvalError("Invalid deserialized prompt: renderUserPrompt must be a function", {
+      code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+      context: { promptId, actual: typeof obj.renderUserPrompt }
+    });
+  }
+}
+function deserializePrompt(serialized) {
+  const { id, version, system, userTemplate, customFields } = serialized;
+  let renderUserPrompt;
+  try {
+    renderUserPrompt = compileTemplate2(userTemplate, id);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new EvalError(`Failed to compile userTemplate: ${message}`, {
+      code: "TEMPLATE_COMPILE_ERROR" /* TEMPLATE_COMPILE_ERROR */,
+      context: { promptId: id, userTemplate }
+    });
+  }
+  const result = {
+    ...customFields,
+    id,
+    version,
+    system,
+    userTemplate,
+    renderUserPrompt
+  };
+  validateDeserializedPrompt(result, id);
+  return result;
+}
+function serializeRoundResult(result) {
+  const { summary } = result.report;
+  return {
+    round: result.round,
+    completedAt: result.completedAt.toISOString(),
+    avgScore: summary.avgScore,
+    passed: summary.passed,
+    failed: summary.failed,
+    totalTests: summary.totalTests,
+    suggestionsGenerated: result.suggestionsGenerated,
+    suggestionsApproved: result.suggestionsApproved,
+    promptSnapshot: result.promptSnapshot,
+    promptVersionAfter: result.promptVersionAfter,
+    cost: result.cost,
+    scoreDelta: result.scoreDelta
+  };
+}
+function validateHistorySchema(data) {
+  if (typeof data !== "object" || data === null) {
+    throw new EvalError("Invalid history: not an object", {
+      code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */
+    });
+  }
+  const h = data;
+  if (h.schemaVersion !== "1.1.0") {
+    throw new EvalError(`Unsupported schema version: ${String(h.schemaVersion)}`, {
+      code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */,
+      context: { schemaVersion: h.schemaVersion }
+    });
+  }
+  const requiredFields = [
+    "sessionId",
+    "startedAt",
+    "initialPrompt",
+    "currentPrompt",
+    "rounds",
+    "totalCost"
+  ];
+  for (const field of requiredFields) {
+    if (!(field in h)) {
+      throw new EvalError(`Invalid history: missing field "${field}"`, {
+        code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */,
+        context: { missingField: field }
+      });
+    }
+  }
+}
+var ImprovementSessionImpl = class {
+  _history;
+  _isUpdating = false;
+  _savePromise = Promise.resolve();
+  config;
+  constructor(history, config = {}) {
+    this._history = history;
+    this.config = {
+      autoSave: config.autoSave ?? false,
+      ...config
+    };
+  }
+  get sessionId() {
+    return this._history.sessionId;
+  }
+  get history() {
+    return this._history;
+  }
+  get canSave() {
+    return this.config.path !== void 0;
+  }
+  addRound(roundResult, updatedPrompt) {
+    if (this._isUpdating) {
+      throw new EvalError("Session is being updated", {
+        code: "CONCURRENT_MODIFICATION" /* CONCURRENT_MODIFICATION */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    if (this._history.completedAt) {
+      throw new EvalError("Cannot add round to completed session", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    this._isUpdating = true;
+    try {
+      const serializedRound = serializeRoundResult(roundResult);
+      this._history = {
+        ...this._history,
+        currentPrompt: updatedPrompt,
+        rounds: [...this._history.rounds, serializedRound],
+        totalCost: this._history.totalCost + roundResult.cost.total
+      };
+      if (this.config.autoSave && this.canSave) {
+        this.save().catch((err) => this.handleAutoSaveError(err));
+      }
+    } finally {
+      this._isUpdating = false;
+    }
+  }
+  complete(terminationReason) {
+    this._history = {
+      ...this._history,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      terminationReason
+    };
+    if (this.config.autoSave && this.canSave) {
+      this.save().catch((err) => this.handleAutoSaveError(err));
+    }
+  }
+  handleAutoSaveError(error) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    if (this.config.onAutoSaveError) {
+      this.config.onAutoSaveError(err);
+    } else {
+      console.error("Auto-save failed:", err);
+    }
+  }
+  async save() {
+    if (!this.config.path) {
+      throw new EvalError("Cannot save: no path configured", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    this._savePromise = this._savePromise.then(async () => {
+      await saveHistory(this._history, this.config.path, this.config.storage);
+    });
+    return this._savePromise;
+  }
+  async flush() {
+    return this._savePromise;
+  }
+};
+function createSession(initialPrompt, config) {
+  const serializedPrompt = serializePrompt(initialPrompt);
+  const history = {
+    schemaVersion: "1.1.0",
+    sessionId: crypto.randomUUID(),
+    startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    initialPrompt: serializedPrompt,
+    currentPrompt: serializedPrompt,
+    rounds: [],
+    totalCost: 0
+  };
+  return new ImprovementSessionImpl(history, config);
+}
+async function resumeSession(path3, config) {
+  const history = await loadHistory(path3, config?.storage);
+  const reopenedHistory = {
+    ...history,
+    completedAt: void 0,
+    terminationReason: void 0
+  };
+  return new ImprovementSessionImpl(reopenedHistory, { ...config, path: path3 });
+}
+async function saveHistory(history, path3, storage = defaultHistoryStorage) {
+  try {
+    const dir = dirname(path3);
+    if (dir && dir !== "." && dir !== "/" && !storage.exists(dir)) {
+      await storage.mkdir(dir, { recursive: true });
+    }
+    await storage.writeFile(path3, JSON.stringify(history, null, 2));
+  } catch (error) {
+    if (error instanceof EvalError) throw error;
+    throw EvalError.from(error, "FILE_WRITE_ERROR" /* FILE_WRITE_ERROR */, { path: path3 });
+  }
+}
+async function loadHistory(path3, storage = defaultHistoryStorage) {
+  try {
+    if (!storage.exists(path3)) {
+      throw new EvalError(`History file not found: ${path3}`, {
+        code: "FILE_READ_ERROR" /* FILE_READ_ERROR */,
+        context: { path: path3 }
+      });
+    }
+    const content = await storage.readFile(path3);
+    const history = JSON.parse(content);
+    validateHistorySchema(history);
+    return history;
+  } catch (error) {
+    if (error instanceof EvalError) throw error;
+    throw EvalError.from(error, "FILE_READ_ERROR" /* FILE_READ_ERROR */, { path: path3 });
+  }
+}
+
+// src/improvement-cycle/runner.ts
+function initializeCycleState(initialPrompt, existingSession) {
+  const resumeFromRound = existingSession ? existingSession.history.rounds.length : 0;
+  return {
+    currentPrompt: initialPrompt,
+    currentRound: resumeFromRound,
+    previousScores: existingSession ? existingSession.history.rounds.map((r) => r.avgScore) : [],
+    totalCost: existingSession ? existingSession.history.totalCost : 0,
+    completedRounds: []
+  };
+}
+function calculateScoreDelta(currentScore, previousScores) {
+  if (previousScores.length === 0) {
+    return null;
+  }
+  const previousScore = previousScores[previousScores.length - 1];
+  return currentScore - previousScore;
+}
+function buildCycleContext(state, currentScore) {
+  return {
+    currentRound: state.currentRound,
+    latestScore: currentScore,
+    previousScores: [...state.previousScores],
+    totalCost: state.totalCost,
+    history: state.completedRounds
+  };
+}
+function createRoundResult(state, report, improveResult, cost, scoreDelta, promptSnapshot) {
+  return {
+    round: state.currentRound,
+    report,
+    completedAt: /* @__PURE__ */ new Date(),
+    suggestionsGenerated: improveResult.suggestions,
+    suggestionsApproved: [],
+    // Will be updated after decision
+    promptSnapshot,
+    promptVersionAfter: state.currentPrompt.version,
+    cost,
+    scoreDelta
+  };
+}
+async function handleStopDecision(state, session, roundResult, promptSnapshot, terminatedByCondition, conditionReason) {
+  const reason = terminatedByCondition ? conditionReason : "User requested stop";
+  session.addRound(roundResult, promptSnapshot);
+  session.complete(reason);
+  await session.flush();
+  state.completedRounds.push(roundResult);
+  return {
+    rounds: state.completedRounds,
+    finalPrompt: deserializePrompt(session.history.currentPrompt),
+    terminationReason: reason,
+    totalCost: state.totalCost,
+    history: session.history
+  };
+}
+function handleRollbackDecision(state, rollbackToRound) {
+  const targetRoundIndex = rollbackToRound - 1;
+  if (targetRoundIndex < 0 || targetRoundIndex >= state.completedRounds.length) {
+    throw new Error(`Cannot rollback to round ${rollbackToRound}: round not found`);
+  }
+  const targetRound = state.completedRounds[targetRoundIndex];
+  state.currentPrompt = deserializePrompt(targetRound.promptSnapshot);
+  state.previousScores = state.previousScores.slice(0, rollbackToRound - 1);
+}
+function handleContinueDecision(state, session, roundResult, approvedSuggestions, versionBump) {
+  const updatedRoundResult = {
+    ...roundResult,
+    suggestionsApproved: approvedSuggestions
+  };
+  if (approvedSuggestions.length > 0) {
+    const applyResult = applyPromptSuggestions(state.currentPrompt, approvedSuggestions, {
+      bumpVersion: versionBump
+    });
+    state.currentPrompt = applyResult.prompt;
+    updatedRoundResult.promptVersionAfter = state.currentPrompt.version;
+  }
+  const updatedPromptSnapshot = serializePrompt(state.currentPrompt);
+  session.addRound(updatedRoundResult, updatedPromptSnapshot);
+  state.completedRounds.push(updatedRoundResult);
+  return updatedRoundResult;
+}
+async function executeRound(config, state, pricingConfig) {
+  const { createAgent, judge, improver, testCases: testCases2, options = {} } = config;
+  const agent = createAgent(state.currentPrompt);
+  const suite = createEvalSuite({
+    agent,
+    judge,
+    agentDescription: options.agentDescription
+  });
+  const report = await suite.run(testCases2, options.runOptions);
+  const improveResult = improver ? await improver.improve(state.currentPrompt, report.results) : { suggestions: [] };
+  const cost = calculateRoundCost(report, improveResult, pricingConfig);
+  return { report, improveResult, cost };
+}
+function detectProviderForImprover(model) {
+  if (!model) return "anthropic";
+  if (model.startsWith("claude-")) return "anthropic";
+  if (model.startsWith("gpt-") || model.startsWith("o1") || model.startsWith("o3")) return "openai";
+  if (model.startsWith("gemini-")) return "google";
+  return "anthropic";
+}
+function toLanguageModelUsage2(usage) {
+  return {
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    totalTokens: usage.totalTokens
+  };
+}
+function calculateImproverCost(improveResult, pricingConfig) {
+  const usage = improveResult.metadata?.tokenUsage;
+  if (!usage) return 0;
+  const model = improveResult.metadata?.model ?? "unknown";
+  const provider = detectProviderForImprover(model);
+  const providerPricing = pricingConfig?.providerPricing?.[provider];
+  const result = calculateCostFromUsage2(
+    toLanguageModelUsage2(usage),
+    model,
+    provider,
+    providerPricing
+  );
+  return result.total;
+}
+function calculateRoundCost(report, improveResult, pricingConfig) {
+  const reportCosts = pricingConfig ? calculateReportCosts(report, pricingConfig) : { total: 0, byComponent: { agent: 0, judge: 0 } };
+  const improverCost = calculateImproverCost(improveResult, pricingConfig);
+  return {
+    agent: reportCosts.byComponent.agent ?? 0,
+    judge: reportCosts.byComponent.judge ?? 0,
+    improver: improverCost,
+    total: reportCosts.total + improverCost
+  };
+}
+async function* runImprovementCycle(config) {
+  const { initialPrompt, terminateWhen = [], options = {} } = config;
+  const { pricingConfig, versionBump = "patch", history: historyConfig, session: existingSession } = options;
+  const session = existingSession ?? createSession(
+    initialPrompt,
+    historyConfig ? { path: historyConfig.path, autoSave: historyConfig.autoSave } : void 0
+  );
+  const state = initializeCycleState(initialPrompt, existingSession);
+  try {
+    while (true) {
+      state.currentRound++;
+      const { report, improveResult, cost } = await executeRound(config, state, pricingConfig);
+      state.totalCost += cost.total;
+      const currentScore = report.summary.avgScore;
+      const scoreDelta = calculateScoreDelta(currentScore, state.previousScores);
+      const promptSnapshot = serializePrompt(state.currentPrompt);
+      const roundResult = createRoundResult(state, report, improveResult, cost, scoreDelta, promptSnapshot);
+      const context = buildCycleContext(state, currentScore);
+      state.previousScores.push(currentScore);
+      const terminationCheck = await checkCycleTermination(terminateWhen, context);
+      const pendingSuggestions = improveResult.suggestions.map((s) => ({
+        ...s,
+        approved: false
+      }));
+      const roundYield = {
+        roundResult,
+        pendingSuggestions,
+        terminationCheck,
+        context
+      };
+      const decision = yield roundYield;
+      if (!decision || decision.action === "stop") {
+        return await handleStopDecision(
+          state,
+          session,
+          roundResult,
+          promptSnapshot,
+          terminationCheck.terminated,
+          terminationCheck.reason
+        );
+      }
+      if (decision.action === "rollback" && decision.rollbackToRound !== void 0) {
+        handleRollbackDecision(state, decision.rollbackToRound);
+        continue;
+      }
+      handleContinueDecision(
+        state,
+        session,
+        roundResult,
+        decision.approvedSuggestions ?? [],
+        versionBump
+      );
+    }
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    session.complete(`Error: ${errorMessage}`);
+    throw error;
+  }
+}
+async function runImprovementCycleAuto(config) {
+  const cycle = runImprovementCycle(config);
+  let iteratorResult = await cycle.next();
+  while (!iteratorResult.done) {
+    const roundYield = iteratorResult.value;
+    let decision;
+    if (roundYield.terminationCheck.terminated) {
+      decision = { action: "stop" };
+    } else {
+      const approvedSuggestions = roundYield.pendingSuggestions.map((s) => ({
+        ...s,
+        approved: true
+      }));
+      decision = { action: "continue", approvedSuggestions };
+    }
+    iteratorResult = await cycle.next(decision);
+  }
+  return iteratorResult.value;
+}
+
+// src/cli/commands/run.ts
+async function runCommand(configPath, options) {
+  const startTime = Date.now();
+  try {
+    printBanner();
+    printProgress("Loading environment...");
+    await loadEnvFile(options.envFile);
+    printProgress("Loading configuration...");
+    const config = await loadConfigWithDefaults(configPath);
+    printProgress("Initializing providers...");
+    const { mainProvider, judgeProvider, improverProvider } = initializeProviders(config, options);
+    const judge = createJudge({
+      provider: judgeProvider,
+      prompt: config.judge.prompt,
+      criteria: config.judge.criteria,
+      passThreshold: config.judge.passThreshold
+    });
+    const improver = config.improver ? createImprover({
+      provider: improverProvider,
+      prompt: config.improver.prompt
+    }) : void 0;
+    const concurrency = options.concurrency ? parseInt(options.concurrency, 10) : config.run?.concurrency ?? 1;
+    const iterations = options.iterations ? parseInt(options.iterations, 10) : config.run?.iterations ?? 1;
+    const verbose = options.verbose ?? config.output?.verbose ?? false;
+    const allReports = [];
+    const includePatterns = options.include ?? config.include;
+    if (includePatterns && includePatterns.length > 0) {
+      const yamlReports = await runYamlTests({
+        config,
+        options,
+        includePatterns,
+        mainProvider,
+        judge,
+        improver,
+        concurrency,
+        iterations
+      });
+      allReports.push(...yamlReports);
+    }
+    if (config.testCases && config.testCases.length > 0) {
+      const inlineReports = await runInlineTests({
+        config,
+        options,
+        judge,
+        improver,
+        concurrency,
+        iterations
+      });
+      allReports.push(...inlineReports);
+    }
+    const report = mergeReports(allReports, resolvePromptVersion(config));
+    if (report.summary.totalTests === 0) {
+      printError(
+        new Error(
+          "No test cases to run after filtering.\n" + (options.tags ? `  Tags filter: ${options.tags.join(", ")}
+` : "") + (options.agent ? `  Agent filter: ${options.agent}
+` : "")
+        )
+      );
+      process.exit(1);
+    }
+    const duration = Date.now() - startTime;
+    printSummary(report, { verbose, duration });
+    if (options.report !== false) {
+      const outputPath = await generateReport(report, {
+        dir: config.output?.dir,
+        filename: options.output ?? config.output?.filename
+      });
+      console.log(`
+Report saved to: ${outputPath}`);
+    }
+    const hasFailures = report.summary.failed > 0;
+    process.exit(hasFailures ? 1 : 0);
+  } catch (error) {
+    printError(error instanceof Error ? error : new Error(String(error)));
+    process.exit(1);
+  }
+}
+async function runYamlTests(params) {
+  const { config, options, includePatterns, mainProvider, judge, improver, concurrency, iterations } = params;
+  const reports = [];
+  printProgress("Discovering YAML eval files...");
+  const filePaths = await discoverEvalFiles(config, { include: includePatterns });
+  if (filePaths.length === 0) {
+    printProgress("No YAML files found matching patterns");
+    return reports;
+  }
+  printProgress(`Discovered ${filePaths.length} YAML file(s)`);
+  const yamlFiles = await loadYamlEvalFiles(filePaths);
+  const filteredFiles = options.agent ? yamlFiles.filter((f) => f.content.agent === options.agent) : yamlFiles;
+  if (options.agent && filteredFiles.length === 0) {
+    const availableAgents = [...new Set(yamlFiles.map((f) => f.content.agent))];
+    throw new ConfigError(
+      `No YAML files found for agent "${options.agent}".
+Available agents: ${availableAgents.join(", ")}`,
+      "CONFIG_VALIDATION_ERROR"
+    );
+  }
+  const yamlByAgent = groupYamlByAgent(filteredFiles);
+  for (const [agentName, agentFiles] of yamlByAgent) {
+    printProgress(`Running tests for agent: ${agentName}`);
+    const agent = lookupAgent(config, agentName);
+    const yamlContext = {
+      provider: mainProvider,
+      buildInput: (response, _ctx) => ({ message: response })
+    };
+    const yamlTestCases = [];
+    for (const file of agentFiles) {
+      const cases = convertToTestCases(file.content, yamlContext);
+      yamlTestCases.push(...cases);
+    }
+    const filteredCases = filterByTags(yamlTestCases, options.tags);
+    if (filteredCases.length === 0) {
+      printProgress(`  No matching tests after tag filter for ${agentName}`);
+      continue;
+    }
+    const { singleTurnCases, multiTurnCases } = splitTestCases(filteredCases);
+    const suite = createEvalSuite({
+      agent,
+      judge,
+      improver,
+      agentDescription: resolveAgentDescription(config, agent)
+    });
+    if (singleTurnCases.length > 0) {
+      printProgress(`  Running ${singleTurnCases.length} single-turn test(s)...`);
+      const report = await suite.run(singleTurnCases, {
+        concurrency,
+        iterations,
+        stopOnFirstFailure: config.run?.stopOnFirstFailure
+      });
+      reports.push(report);
+    }
+    if (multiTurnCases.length > 0) {
+      printProgress(`  Running ${multiTurnCases.length} multi-turn test(s)...`);
+      const multiTurnResults = await runMultiTurnCases(multiTurnCases, {
+        agent,
+        judge,
+        agentDescription: resolveAgentDescription(config, agent)
+      });
+      const multiTurnReport = createMultiTurnReport(multiTurnResults, { ...config, agent });
+      reports.push(multiTurnReport);
+    }
+  }
+  return reports;
+}
+async function runInlineTests(params) {
+  const { config, options, judge, improver, concurrency, iterations } = params;
+  const reports = [];
+  const filteredInline = filterByTags(config.testCases, options.tags);
+  if (filteredInline.length === 0) {
+    return reports;
+  }
+  const { singleTurnCases, multiTurnCases } = splitTestCases(filteredInline);
+  if (singleTurnCases.length > 0) {
+    printProgress(`Running ${singleTurnCases.length} inline single-turn test(s)...`);
+    const suite = createEvalSuite({
+      agent: config.agent,
+      judge,
+      improver,
+      agentDescription: resolveAgentDescription(config, config.agent)
+    });
+    const report = await suite.run(singleTurnCases, {
+      concurrency,
+      iterations,
+      stopOnFirstFailure: config.run?.stopOnFirstFailure
+    });
+    reports.push(report);
+  }
+  if (multiTurnCases.length > 0) {
+    printProgress(`Running ${multiTurnCases.length} inline multi-turn test(s)...`);
+    const multiTurnResults = await runMultiTurnCases(multiTurnCases, {
+      agent: config.agent,
+      judge,
+      agentDescription: resolveAgentDescription(config, config.agent)
+    });
+    const multiTurnReport = createMultiTurnReport(multiTurnResults, config);
+    reports.push(multiTurnReport);
+  }
+  return reports;
+}
+async function runMultiTurnCases(testCases2, context) {
+  const results = [];
+  for (const testCase2 of testCases2) {
+    const result = await executeMultiTurnTestCase(
+      testCase2,
+      context
+    );
+    results.push(result);
+  }
+  return results;
+}
+function createMultiTurnReport(results, config) {
+  const totalTests = results.length;
+  const passed = results.filter((r) => r.passed).length;
+  const failed = totalTests - passed;
+  const avgScore = totalTests > 0 ? results.reduce((sum, r) => sum + r.overallScore, 0) / totalTests : 0;
+  const totalLatency = results.reduce((sum, r) => sum + r.metrics.latencyMs, 0);
+  const totalTokens = results.reduce((sum, r) => sum + r.metrics.tokenUsage.totalTokens, 0);
+  return {
+    summary: {
+      totalTests,
+      passed,
+      failed,
+      avgScore,
+      metrics: {
+        avgLatencyMs: totalTests > 0 ? totalLatency / totalTests : 0,
+        totalTokens,
+        totalEstimatedCost: 0
+      }
+    },
+    results,
+    suggestions: [],
+    generatedAt: /* @__PURE__ */ new Date(),
+    promptVersion: resolvePromptVersion(config)
+  };
+}
+function groupYamlByAgent(files) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const file of files) {
+    const agentName = file.content.agent;
+    if (!groups.has(agentName)) {
+      groups.set(agentName, []);
+    }
+    groups.get(agentName).push(file);
+  }
+  return groups;
+}
+function lookupAgent(config, agentName) {
+  if (!config.agents || !(agentName in config.agents)) {
+    const available = config.agents ? Object.keys(config.agents) : [];
+    throw new ConfigError(
+      `Agent "${agentName}" not found in config.agents registry.
+Available agents: ${available.length > 0 ? available.join(", ") : "(none)"}`,
+      "CONFIG_VALIDATION_ERROR"
+    );
+  }
+  return config.agents[agentName];
+}
+function filterByTags(testCases2, tags) {
+  if (!tags || tags.length === 0) {
+    return testCases2;
+  }
+  return testCases2.filter((tc) => {
+    if (!tc.tags || tc.tags.length === 0) {
+      return false;
+    }
+    return tc.tags.some((tag) => tags.includes(tag));
+  });
+}
+function splitTestCases(testCases2) {
+  const singleTurnCases = [];
+  const multiTurnCases = [];
+  for (const testCase2 of testCases2) {
+    if (isMultiTurnConfig(testCase2)) {
+      multiTurnCases.push(testCase2);
+    } else {
+      singleTurnCases.push(testCase2);
+    }
+  }
+  return { singleTurnCases, multiTurnCases };
+}
+function resolveAgentDescription(config, agent) {
+  return config.agentDescription ?? agent.config.description ?? "";
+}
+function resolvePromptVersion(config) {
+  if (config.agent?.prompt?.version) {
+    return config.agent.prompt.version;
+  }
+  if (config.agents) {
+    const firstAgent = Object.values(config.agents)[0];
+    if (firstAgent?.prompt?.version) {
+      return firstAgent.prompt.version;
+    }
+  }
+  return "unknown";
+}
+function mergeReports(reports, promptVersion) {
+  if (reports.length === 0) {
+    return createEmptyReport(promptVersion);
+  }
+  if (reports.length === 1) {
+    return reports[0];
+  }
+  const allResults = [];
+  const suggestionMap = /* @__PURE__ */ new Map();
+  let totalTests = 0;
+  let passed = 0;
+  let failed = 0;
+  let totalScore = 0;
+  let totalLatency = 0;
+  let totalTokens = 0;
+  let totalCost = 0;
+  for (const report of reports) {
+    allResults.push(...report.results);
+    deduplicateSuggestions(report.suggestions, suggestionMap);
+    totalTests += report.summary.totalTests;
+    passed += report.summary.passed;
+    failed += report.summary.failed;
+    totalScore += report.summary.avgScore * report.summary.totalTests;
+    totalLatency += report.summary.metrics.avgLatencyMs * report.summary.totalTests;
+    totalTokens += report.summary.metrics.totalTokens;
+    totalCost += report.summary.metrics.totalEstimatedCost ?? 0;
+  }
+  return {
+    summary: {
+      totalTests,
+      passed,
+      failed,
+      avgScore: totalTests > 0 ? totalScore / totalTests : 0,
+      metrics: {
+        avgLatencyMs: totalTests > 0 ? totalLatency / totalTests : 0,
+        totalTokens,
+        totalEstimatedCost: totalCost
+      }
+    },
+    results: allResults,
+    suggestions: [...suggestionMap.values()],
+    generatedAt: /* @__PURE__ */ new Date(),
+    promptVersion
+  };
+}
+function createEmptyReport(promptVersion) {
+  return {
+    summary: {
+      totalTests: 0,
+      passed: 0,
+      failed: 0,
+      avgScore: 0,
+      metrics: {
+        avgLatencyMs: 0,
+        totalTokens: 0,
+        totalEstimatedCost: 0
+      }
+    },
+    results: [],
+    suggestions: [],
+    generatedAt: /* @__PURE__ */ new Date(),
+    promptVersion
+  };
+}
+function deduplicateSuggestions(suggestions, map) {
+  for (const suggestion of suggestions) {
+    const key = `${suggestion.type}:${suggestion.suggestedValue}`;
+    if (!map.has(key)) {
+      map.set(key, suggestion);
+    }
+  }
+}
+
+// src/cli/output/improve-report.ts
+function printImprovementSummary(result, options = {}) {
+  const { rounds, terminationReason, totalCost, finalPrompt } = result;
+  const { verbose, duration } = options;
+  const divider = "\u2550".repeat(CLI_DEFAULTS.DIVIDER_WIDTH);
+  console.log();
+  console.log(c("cyan", divider));
+  console.log(c("bold", "  Improvement Cycle Results"));
+  console.log(c("cyan", divider));
+  console.log();
+  const finalScore = getFinalScore(rounds);
+  const scoreChange = getScoreChange(rounds);
+  console.log(`  ${c("bold", "Total Rounds:")}    ${rounds.length}`);
+  console.log(`  ${c("bold", "Final Score:")}     ${finalScore.toFixed(1)}/100`);
+  console.log(`  ${c("bold", "Score Change:")}    ${scoreChange}`);
+  console.log(`  ${c("bold", "Total Cost:")}      $${totalCost.toFixed(2)}`);
+  console.log(`  ${c("bold", "Final Version:")}   ${finalPrompt.version}`);
+  console.log();
+  console.log(`  ${c("bold", "Termination:")}     ${terminationReason}`);
+  if (duration !== void 0) {
+    console.log();
+    console.log(`  ${c("bold", "Duration:")}        ${formatDuration(duration)}`);
+  }
+  console.log();
+  console.log(c("cyan", divider));
+  if (verbose && rounds.length > 0) {
+    printRoundsDetail(rounds);
+  }
+}
+function getFinalScore(rounds) {
+  if (rounds.length === 0) return 0;
+  return rounds[rounds.length - 1].report.summary.avgScore;
+}
+function getScoreChange(rounds) {
+  if (rounds.length < 1) return "N/A";
+  const firstScore = rounds[0].report.summary.avgScore;
+  const lastScore = rounds[rounds.length - 1].report.summary.avgScore;
+  const delta = lastScore - firstScore;
+  if (rounds.length === 1) {
+    return c("dim", "N/A (first round)");
+  }
+  if (delta > 0) {
+    return c("green", `+${delta.toFixed(1)}`);
+  } else if (delta < 0) {
+    return c("red", `${delta.toFixed(1)}`);
+  }
+  return "0.0";
+}
+function formatDuration(ms) {
+  if (ms < 1e3) return `${ms}ms`;
+  if (ms < 6e4) return `${(ms / 1e3).toFixed(1)}s`;
+  return `${(ms / 6e4).toFixed(1)}m`;
+}
+function printRoundsDetail(rounds) {
+  console.log();
+  console.log(c("bold", "  Round History:"));
+  console.log();
+  for (const round of rounds) {
+    const scoreStr = round.report.summary.avgScore.toFixed(1);
+    const deltaStr = round.scoreDelta !== null ? ` (${round.scoreDelta >= 0 ? "+" : ""}${round.scoreDelta.toFixed(1)})` : "";
+    const costStr = `$${round.cost.total.toFixed(2)}`;
+    console.log(
+      `  Round ${round.round}: Score ${scoreStr}${deltaStr} | Cost ${costStr}`
+    );
+    console.log(
+      `         Suggestions: ${round.suggestionsGenerated.length} generated, ${round.suggestionsApproved.length} applied`
+    );
+  }
+}
+
+// src/cli/commands/improve.ts
+async function improveCommand(configPath, options) {
+  const startTime = Date.now();
+  try {
+    printBanner();
+    validateImproveOptions(options);
+    printProgress("Loading environment...");
+    await loadEnvFile(options.envFile);
+    printProgress("Loading configuration...");
+    const config = await loadConfigWithDefaults(configPath);
+    printProgress("Initializing providers...");
+    const { mainProvider, judgeProvider, improverProvider } = initializeProviders(config, options);
+    const conditions = buildTerminationConditions(options);
+    if (conditions.length === 0) {
+      throw new Error(
+        "At least one termination condition is required.\nUse --target-score, --max-rounds, --max-cost, or --stale-rounds"
+      );
+    }
+    const judge = createJudge({
+      provider: judgeProvider,
+      prompt: config.judge.prompt,
+      criteria: config.judge.criteria,
+      passThreshold: config.judge.passThreshold
+    });
+    if (!config.improver) {
+      throw new Error(
+        "Improver configuration is required for improvement cycles.\nAdd an `improver` section to your config file."
+      );
+    }
+    const improver = createImprover({
+      provider: improverProvider,
+      prompt: config.improver.prompt
+    });
+    if (options.resume) {
+      await runResumeMode(
+        options,
+        config,
+        conditions,
+        judge,
+        improver,
+        mainProvider,
+        startTime
+      );
+    } else {
+      await runFreshMode(
+        options,
+        config,
+        conditions,
+        judge,
+        improver,
+        mainProvider,
+        startTime
+      );
+    }
+  } catch (error) {
+    printError(error instanceof Error ? error : new Error(String(error)));
+    process.exit(1);
+  }
+}
+function validateImproveOptions(options) {
+  if (!options.history && !options.resume) {
+    throw new Error(
+      "--history <path> is required to save improvement history.\nOr use --resume <path> to continue from existing history."
+    );
+  }
+}
+function buildTerminationConditions(options) {
+  const conditions = [];
+  if (options.targetScore) {
+    const score = parseInt(options.targetScore, 10);
+    if (isNaN(score) || score < 0 || score > 100) {
+      throw new Error(`Invalid target score: ${options.targetScore}. Must be 0-100.`);
+    }
+    conditions.push(targetScore(score));
+  }
+  if (options.maxRounds) {
+    const rounds = parseInt(options.maxRounds, 10);
+    if (isNaN(rounds) || rounds < 1) {
+      throw new Error(`Invalid max rounds: ${options.maxRounds}. Must be >= 1.`);
+    }
+    conditions.push(maxRounds(rounds));
+  }
+  if (options.maxCost) {
+    const cost = parseFloat(options.maxCost);
+    if (isNaN(cost) || cost <= 0) {
+      throw new Error(`Invalid max cost: ${options.maxCost}. Must be > 0.`);
+    }
+    conditions.push(maxCost(cost));
+  }
+  if (options.staleRounds) {
+    const rounds = parseInt(options.staleRounds, 10);
+    if (isNaN(rounds) || rounds < 1) {
+      throw new Error(`Invalid stale-rounds: ${options.staleRounds}. Must be >= 1.`);
+    }
+    conditions.push(noImprovement(rounds));
+  }
+  return conditions;
+}
+function createAgentFactory(baseAgent) {
+  return (prompt) => ({
+    ...baseAgent,
+    prompt
+  });
+}
+async function runFreshMode(options, config, conditions, judge, improver, _mainProvider, startTime) {
+  printProgress("Starting improvement cycle...");
+  const testCases2 = config.testCases ?? [];
+  if (testCases2.length === 0) {
+    throw new Error(
+      "No test cases found. Add testCases to your config or use include patterns."
+    );
+  }
+  const cycleConfig = {
+    createAgent: createAgentFactory(config.agent),
+    initialPrompt: config.agent.prompt,
+    testCases: testCases2,
+    judge,
+    improver,
+    terminateWhen: conditions,
+    options: {
+      pricingConfig: config.pricing,
+      agentDescription: config.agentDescription,
+      history: options.history ? {
+        path: options.history,
+        autoSave: true
+      } : void 0,
+      runOptions: {
+        concurrency: options.concurrency ? parseInt(options.concurrency, 10) : config.run?.concurrency,
+        iterations: options.iterations ? parseInt(options.iterations, 10) : config.run?.iterations
+      }
+    }
+  };
+  printProgress(`Running with ${testCases2.length} test case(s)...`);
+  printProgress(`Termination: ${formatConditions(conditions)}`);
+  console.log();
+  const result = await runImprovementCycleAuto(cycleConfig);
+  const duration = Date.now() - startTime;
+  printImprovementSummary(result, { verbose: options.verbose, duration });
+  if (options.history) {
+    console.log(`
+History saved to: ${options.history}`);
+  }
+  process.exit(0);
+}
+async function runResumeMode(options, config, conditions, judge, improver, _mainProvider, startTime) {
+  printProgress(`Resuming from ${options.resume}...`);
+  const session = await resumeSession(options.resume, { autoSave: true });
+  const currentPrompt = deserializePrompt(session.history.currentPrompt);
+  printProgress(`Resumed session ${session.sessionId}`);
+  printProgress(`Continuing from round ${session.history.rounds.length + 1}`);
+  const testCases2 = config.testCases ?? [];
+  if (testCases2.length === 0) {
+    throw new Error(
+      "No test cases found. Add testCases to your config or use include patterns."
+    );
+  }
+  const cycleConfig = {
+    createAgent: createAgentFactory(config.agent),
+    initialPrompt: currentPrompt,
+    testCases: testCases2,
+    judge,
+    improver,
+    terminateWhen: conditions,
+    options: {
+      pricingConfig: config.pricing,
+      agentDescription: config.agentDescription,
+      history: {
+        path: options.resume,
+        autoSave: true
+      },
+      session,
+      // Pass the resumed session to preserve session ID and accumulated state
+      runOptions: {
+        concurrency: options.concurrency ? parseInt(options.concurrency, 10) : config.run?.concurrency,
+        iterations: options.iterations ? parseInt(options.iterations, 10) : config.run?.iterations
+      }
+    }
+  };
+  printProgress(`Running with ${testCases2.length} test case(s)...`);
+  printProgress(`Termination: ${formatConditions(conditions)}`);
+  console.log();
+  const result = await runImprovementCycleAuto(cycleConfig);
+  const duration = Date.now() - startTime;
+  printImprovementSummary(result, { verbose: options.verbose, duration });
+  console.log(`
+History saved to: ${options.resume}`);
+  process.exit(0);
+}
+function formatConditions(conditions) {
+  return conditions.map((c2) => {
+    switch (c2.type) {
+      case "targetScore":
+        return `score >= ${c2.threshold}`;
+      case "maxRounds":
+        return `max ${c2.count} rounds`;
+      case "maxCost":
+        return `max $${c2.maxUSD}`;
+      case "noImprovement":
+        return `no improvement for ${c2.consecutiveRounds} rounds`;
+      case "custom":
+        return c2.description ?? "custom condition";
+    }
+  }).join(" OR ");
+}
+
+// src/cli/commands/rollback.ts
+import { existsSync as existsSync6 } from "fs";
+import { mkdir as mkdir3, writeFile as writeFile4 } from "fs/promises";
+import { dirname as dirname2 } from "path";
+async function rollbackCommand(historyPath, options) {
+  try {
+    printBanner();
+    validateRollbackOptions(historyPath, options);
+    printProgress(`Loading history from ${historyPath}...`);
+    const history = await loadHistory(historyPath);
+    const { prompt: serializedPrompt, sourceLabel } = extractPromptSnapshot(history, options);
+    printProgress(`Extracting ${sourceLabel}...`);
+    const format = options.format ?? "json";
+    const output = formatPromptOutput(serializedPrompt, format);
+    await writeOutputFile(options.output, output);
+    console.log();
+    console.log(`  Prompt extracted to: ${options.output}`);
+    console.log(`    Prompt ID: ${serializedPrompt.id}`);
+    console.log(`    Version: ${serializedPrompt.version}`);
+    console.log();
+  } catch (error) {
+    printError(error instanceof Error ? error : new Error(String(error)));
+    process.exit(1);
+  }
+}
+function extractPromptSnapshot(history, options) {
+  if (options.initial) {
+    return {
+      prompt: history.initialPrompt,
+      sourceLabel: "initial prompt"
+    };
+  }
+  const roundNumber = parseInt(options.round, 10);
+  return {
+    prompt: extractPromptFromRound(history, roundNumber),
+    sourceLabel: `round ${roundNumber}`
+  };
+}
+function validateRollbackOptions(historyPath, options) {
+  if (!historyPath) {
+    throw new Error("History file path is required");
+  }
+  const hasRound = options.round !== void 0;
+  const hasInitial = options.initial === true;
+  if (!hasRound && !hasInitial) {
+    throw new Error("Either --round <n> or --initial is required");
+  }
+  if (hasRound && hasInitial) {
+    throw new Error("Cannot use both --round and --initial");
+  }
+  if (!options.output) {
+    throw new Error("--output <path> is required");
+  }
+  if (hasRound) {
+    const roundNum = parseInt(options.round, 10);
+    if (!Number.isInteger(roundNum) || roundNum < 1) {
+      throw new Error(`Invalid round number: ${options.round}. Must be 1 or greater.`);
+    }
+  }
+  if (options.format && !["json", "ts"].includes(options.format)) {
+    throw new Error(`Invalid format: ${options.format}. Use 'json' or 'ts'`);
+  }
+}
+function extractPromptFromRound(history, roundNumber) {
+  const roundIndex = roundNumber - 1;
+  if (roundIndex < 0 || roundIndex >= history.rounds.length) {
+    const availableRounds = history.rounds.length > 0 ? `1-${history.rounds.length}` : "none";
+    throw new Error(
+      `Round ${roundNumber} not found. Available rounds: ${availableRounds}. Use --initial for the original prompt.`
+    );
+  }
+  return history.rounds[roundIndex].promptSnapshot;
+}
+function formatPromptOutput(prompt, format) {
+  if (format === "json") {
+    return JSON.stringify(prompt, null, 2);
+  }
+  return generateTypeScriptPrompt(prompt);
+}
+function generateTypeScriptPrompt(prompt) {
+  const escapedSystem = escapeTemplateString(prompt.system);
+  const escapedUserTemplate = escapeTemplateString(prompt.userTemplate);
+  const customFieldsComment = prompt.customFields ? `
+ * Custom fields: ${Object.keys(prompt.customFields).join(", ")}` : "";
+  return `import { compileTemplate } from 'agent-eval'
+import type { AgentPrompt } from 'agent-eval'
+
+/**
+ * Extracted from improvement cycle
+ * Original ID: ${prompt.id}
+ * Version: ${prompt.version}${customFieldsComment}
+ */
+export const prompt: AgentPrompt<YourInputType> = {
+  id: '${prompt.id}',
+  version: '${prompt.version}',
+  system: \`${escapedSystem}\`,
+  userTemplate: \`${escapedUserTemplate}\`,
+  renderUserPrompt: compileTemplate(\`${escapedUserTemplate}\`),
+}
+`;
+}
+function escapeTemplateString(str) {
+  return str.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\${/g, "\\${");
+}
+async function writeOutputFile(path3, content) {
+  const dir = dirname2(path3);
+  if (dir && dir !== "." && dir !== "/" && !existsSync6(dir)) {
+    await mkdir3(dir, { recursive: true });
+  }
+  await writeFile4(path3, content, "utf-8");
+}
+
+// src/cli/index.ts
+var VERSION = "0.1.0";
+var cli = dist_default("agent-eval");
+cli.command("run [config]", "Run evaluation suite").option("-o, --output <path>", "Output path for markdown report").option("-e, --env-file <path>", "Path to env file", { default: ".env" }).option("-v, --verbose", "Enable verbose output").option("-c, --concurrency <n>", "Concurrency level").option("-i, --iterations <n>", "Number of iterations per test").option("--no-report", "Skip saving markdown report").option("--mock", "Use mock LLM for testing (no API calls)").option("--include <pattern>", "Glob patterns for YAML files (can be repeated)").option("--tags <tag>", "Filter test cases by tags, OR logic (can be repeated)").option("--agent <name>", "Filter to specific agent name").action(async (configPath, options) => {
+  try {
+    await runCommand(configPath, options);
+  } catch {
+    process.exit(1);
+  }
+});
+cli.command("improve [config]", "Run improvement cycle on prompts").option("-e, --env-file <path>", "Path to env file", { default: ".env" }).option("--history <path>", "Path to save history JSON").option("--target-score <n>", "Target score to reach (0-100)").option("--max-rounds <n>", "Maximum improvement rounds").option("--max-cost <usd>", "Maximum cost in USD").option("--stale-rounds <n>", "Stop after N rounds without improvement").option("--resume <path>", "Resume from existing history file").option("-c, --concurrency <n>", "Concurrency level").option("-i, --iterations <n>", "Iterations per test").option("-v, --verbose", "Enable verbose output").option("--mock", "Use mock LLM for testing (no API calls)").action(async (configPath, options) => {
+  try {
+    await improveCommand(configPath, options);
+  } catch {
+    process.exit(1);
+  }
+});
+cli.command("rollback <history>", "Extract prompt from improvement history").option("-r, --round <n>", "Round number to extract (1, 2, ...)").option("--initial", "Extract the initial prompt (before any improvements)").option("-o, --output <path>", "Output file path").option("-f, --format <type>", "Output format: json or ts", { default: "json" }).action(async (historyPath, options) => {
+  try {
+    await rollbackCommand(historyPath, options);
+  } catch {
+    process.exit(1);
+  }
+});
+cli.help();
+cli.version(VERSION);
+cli.parse();
+//# sourceMappingURL=cli.js.map