@agtlantis/eval 0.1.0

package/dist/index.cjs

@@ -0,0 +1,3998 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ANTHROPIC_PRICING: () => import_core10.ANTHROPIC_PRICING,
+  CompositeReporter: () => CompositeReporter,
+  ConsoleReporter: () => ConsoleReporter,
+  DEFAULT_PRICING_CONFIG: () => import_core10.DEFAULT_PRICING_CONFIG,
+  EvalError: () => EvalError,
+  EvalErrorCode: () => EvalErrorCode,
+  GOOGLE_PRICING: () => import_core10.GOOGLE_PRICING,
+  JsonReporter: () => JsonReporter,
+  MarkdownReporter: () => MarkdownReporter,
+  MockProvider: () => import_testing.MockProvider,
+  OPENAI_PRICING: () => import_core10.OPENAI_PRICING,
+  TestCaseCollection: () => TestCaseCollection,
+  accuracy: () => accuracy,
+  addCostsToResults: () => addCostsToResults,
+  afterTurns: () => afterTurns,
+  aggregateIterationResults: () => aggregateIterationResults,
+  aiUser: () => aiUser,
+  and: () => and,
+  applyPromptSuggestions: () => applyPromptSuggestions,
+  bumpVersion: () => bumpVersion,
+  calculateAvgPassRate: () => calculateAvgPassRate,
+  calculateAvgStdDev: () => calculateAvgStdDev,
+  calculateCostFromUsage: () => import_core10.calculateCostFromUsage,
+  calculateIterationStats: () => calculateIterationStats,
+  calculateMultiTurnIterationStats: () => calculateMultiTurnIterationStats,
+  calculateReportCosts: () => calculateReportCosts,
+  calculateResultCost: () => calculateResultCost,
+  checkCondition: () => checkCondition,
+  checkCycleCondition: () => checkCycleCondition,
+  checkCycleTermination: () => checkCycleTermination,
+  checkTermination: () => checkTermination,
+  compareReports: () => compareReports,
+  compileTemplate: () => import_core9.compileTemplate,
+  consistency: () => consistency,
+  createCompositeReporter: () => createCompositeReporter,
+  createConsoleReporter: () => createConsoleReporter,
+  createDefaultReporter: () => createDefaultReporter,
+  createEvalSuite: () => createEvalSuite,
+  createFilePromptRepository: () => import_core9.createFilePromptRepository,
+  createImprover: () => createImprover,
+  createJsonReporter: () => createJsonReporter,
+  createJudge: () => createJudge,
+  createMarkdownReporter: () => createMarkdownReporter,
+  createMockAgent: () => createMockAgent,
+  createMockImprover: () => createMockImprover,
+  createMockJudge: () => createMockJudge,
+  createReportRunner: () => createReportRunner,
+  createSession: () => createSession,
+  customCondition: () => customCondition,
+  cycleAnd: () => and2,
+  cycleNot: () => not2,
+  cycleOr: () => or2,
+  cycleToMarkdown: () => cycleToMarkdown,
+  defaultHistoryStorage: () => defaultHistoryStorage,
+  defineConfig: () => defineConfig,
+  deserializePrompt: () => deserializePrompt,
+  discoverEvalFiles: () => discoverEvalFiles,
+  executeMultiTurnTestCase: () => executeMultiTurnTestCase,
+  executeTestCase: () => executeTestCase,
+  fieldEquals: () => fieldEquals,
+  fieldIsSet: () => fieldIsSet,
+  getFieldValue: () => getFieldValue,
+  getFileSourceDisplayInfo: () => import_core8.getFileSourceDisplayInfo,
+  getFileSourcesDisplayInfo: () => import_core8.getFileSourcesDisplayInfo,
+  inferMediaType: () => import_core8.inferMediaType,
+  isCustomCondition: () => isCustomCondition,
+  isCustomCycleCondition: () => isCustomCycleCondition,
+  isCycleTerminated: () => isCycleTerminated,
+  isFieldSetCondition: () => isFieldSetCondition,
+  isFieldValueCondition: () => isFieldValueCondition,
+  isFileSource: () => import_core8.isFileSource,
+  isFileSourceBase64: () => import_core8.isFileSourceBase64,
+  isFileSourceData: () => import_core8.isFileSourceData,
+  isFileSourcePath: () => import_core8.isFileSourcePath,
+  isFileSourceUrl: () => import_core8.isFileSourceUrl,
+  isIteratedResult: () => isIteratedResult,
+  isMaxCostCondition: () => isMaxCostCondition,
+  isMaxRoundsCondition: () => isMaxRoundsCondition,
+  isMaxTurnsCondition: () => isMaxTurnsCondition,
+  isMultiTurnResult: () => isMultiTurnResult,
+  isMultiTurnTestCase: () => isMultiTurnTestCase,
+  isNoImprovementCondition: () => isNoImprovementCondition,
+  isSingleTurnResult: () => isSingleTurnResult,
+  isTargetScoreCondition: () => isTargetScoreCondition,
+  isTerminated: () => isTerminated,
+  loadHistory: () => loadHistory,
+  logCycle: () => logCycle,
+  maxCost: () => maxCost,
+  maxRounds: () => maxRounds,
+  mock: () => import_testing.mock,
+  naturalLanguage: () => naturalLanguage,
+  noImprovement: () => noImprovement,
+  not: () => not,
+  or: () => or,
+  relevance: () => relevance,
+  reportToMarkdown: () => reportToMarkdown,
+  resolveFileSource: () => import_core8.resolveFileSource,
+  resolveFileSourcesInInput: () => import_core8.resolveFileSourcesInInput,
+  resumeSession: () => resumeSession,
+  runImprovementCycle: () => runImprovementCycle,
+  runImprovementCycleAuto: () => runImprovementCycleAuto,
+  runWithConcurrency: () => runWithConcurrency,
+  saveCycleJson: () => saveCycleJson,
+  saveCycleMarkdown: () => saveCycleMarkdown,
+  saveHistory: () => saveHistory,
+  saveReportMarkdown: () => saveReportMarkdown,
+  scanForFileSources: () => import_core8.scanForFileSources,
+  schema: () => schema,
+  selectRepresentativeResult: () => selectRepresentativeResult,
+  serializePrompt: () => serializePrompt,
+  suggestionDiff: () => suggestionDiff,
+  suggestionPreview: () => suggestionPreview,
+  suggestionSummary: () => suggestionSummary,
+  targetScore: () => targetScore,
+  testCase: () => testCase,
+  testCases: () => testCases,
+  toEvalAgent: () => toEvalAgent
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/core/runner.ts
+var import_core2 = require("@agtlantis/core");
+
+// src/multi-turn/types.ts
+function isMaxTurnsCondition(condition) {
+  return condition.type === "maxTurns";
+}
+function isFieldSetCondition(condition) {
+  return condition.type === "fieldSet";
+}
+function isFieldValueCondition(condition) {
+  return condition.type === "fieldValue";
+}
+function isCustomCondition(condition) {
+  return condition.type === "custom";
+}
+function isMultiTurnTestCase(testCase2) {
+  return "multiTurn" in testCase2;
+}
+function isTerminated(result) {
+  return result.terminated === true;
+}
+
+// src/core/errors.ts
+var EvalErrorCode = /* @__PURE__ */ ((EvalErrorCode2) => {
+  EvalErrorCode2["LLM_API_ERROR"] = "LLM_API_ERROR";
+  EvalErrorCode2["LLM_RATE_LIMIT"] = "LLM_RATE_LIMIT";
+  EvalErrorCode2["LLM_TIMEOUT"] = "LLM_TIMEOUT";
+  EvalErrorCode2["JSON_PARSE_ERROR"] = "JSON_PARSE_ERROR";
+  EvalErrorCode2["VERDICT_PARSE_ERROR"] = "VERDICT_PARSE_ERROR";
+  EvalErrorCode2["TEMPLATE_COMPILE_ERROR"] = "TEMPLATE_COMPILE_ERROR";
+  EvalErrorCode2["AGENT_EXECUTION_ERROR"] = "AGENT_EXECUTION_ERROR";
+  EvalErrorCode2["INVALID_CONFIG"] = "INVALID_CONFIG";
+  EvalErrorCode2["MISSING_API_KEY"] = "MISSING_API_KEY";
+  EvalErrorCode2["PROMPT_NOT_FOUND"] = "PROMPT_NOT_FOUND";
+  EvalErrorCode2["PROMPT_INVALID_FORMAT"] = "PROMPT_INVALID_FORMAT";
+  EvalErrorCode2["PROMPT_WRITE_ERROR"] = "PROMPT_WRITE_ERROR";
+  EvalErrorCode2["PROMPT_READ_ERROR"] = "PROMPT_READ_ERROR";
+  EvalErrorCode2["SUGGESTION_APPLY_ERROR"] = "SUGGESTION_APPLY_ERROR";
+  EvalErrorCode2["SCHEMA_VALIDATION_ERROR"] = "SCHEMA_VALIDATION_ERROR";
+  EvalErrorCode2["SCHEMA_GENERATION_ERROR"] = "SCHEMA_GENERATION_ERROR";
+  EvalErrorCode2["FILE_READ_ERROR"] = "FILE_READ_ERROR";
+  EvalErrorCode2["FILE_WRITE_ERROR"] = "FILE_WRITE_ERROR";
+  EvalErrorCode2["FILE_TOO_LARGE"] = "FILE_TOO_LARGE";
+  EvalErrorCode2["CONCURRENT_MODIFICATION"] = "CONCURRENT_MODIFICATION";
+  EvalErrorCode2["UNKNOWN_ERROR"] = "UNKNOWN_ERROR";
+  return EvalErrorCode2;
+})(EvalErrorCode || {});
+var EvalError = class _EvalError extends Error {
+  code;
+  cause;
+  context;
+  constructor(message, options) {
+    super(message);
+    this.name = "EvalError";
+    this.code = options.code;
+    this.cause = options.cause;
+    this.context = options.context;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, _EvalError);
+    }
+  }
+  /**
+   * Creates an EvalError from an unknown error with a specific code.
+   */
+  static from(error, code, context) {
+    if (error instanceof _EvalError) {
+      return error;
+    }
+    const cause = error instanceof Error ? error : new Error(String(error));
+    return new _EvalError(cause.message, { code, cause, context });
+  }
+  toJSON() {
+    return {
+      name: this.name,
+      message: this.message,
+      code: this.code,
+      context: this.context,
+      cause: this.cause?.message
+    };
+  }
+};
+
+// src/multi-turn/termination.ts
+function getFieldValue(obj, fieldPath) {
+  if (obj === null || obj === void 0) {
+    return void 0;
+  }
+  const parts = fieldPath.split(".");
+  let current = obj;
+  for (const part of parts) {
+    if (current === null || current === void 0) {
+      return void 0;
+    }
+    if (typeof current !== "object") {
+      return void 0;
+    }
+    current = current[part];
+  }
+  return current;
+}
+function isSet(value) {
+  return value !== null && value !== void 0;
+}
+function checkMaxTurns(condition, context) {
+  const shouldTerminate = context.currentTurn >= condition.count;
+  if (shouldTerminate) {
+    return {
+      terminated: true,
+      terminationType: "maxTurns",
+      matchedCondition: condition,
+      reason: `Maximum turns reached (${condition.count})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Turn ${context.currentTurn} of ${condition.count}`
+  };
+}
+function checkFieldSet(condition, context) {
+  const fieldValue = getFieldValue(context.lastOutput, condition.fieldPath);
+  const fieldIsSet2 = isSet(fieldValue);
+  if (fieldIsSet2) {
+    return {
+      terminated: true,
+      terminationType: "condition",
+      matchedCondition: condition,
+      reason: `Field "${condition.fieldPath}" is set (value: ${JSON.stringify(fieldValue)})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Field "${condition.fieldPath}" is not set`
+  };
+}
+function checkFieldValue(condition, context) {
+  const fieldValue = getFieldValue(context.lastOutput, condition.fieldPath);
+  const matches = fieldValue === condition.expectedValue;
+  if (matches) {
+    return {
+      terminated: true,
+      terminationType: "condition",
+      matchedCondition: condition,
+      reason: `Field "${condition.fieldPath}" equals expected value`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Field "${condition.fieldPath}" does not equal expected value (got: ${JSON.stringify(fieldValue)})`
+  };
+}
+async function checkCustom(condition, context) {
+  const description = condition.description ?? "Custom condition";
+  try {
+    const shouldTerminate = await condition.check(context);
+    if (shouldTerminate) {
+      return {
+        terminated: true,
+        terminationType: "condition",
+        matchedCondition: condition,
+        reason: `${description} met`
+      };
+    }
+    return {
+      terminated: false,
+      reason: `${description} not met`
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    return {
+      terminated: false,
+      reason: `${description} failed: ${errorMessage}`
+    };
+  }
+}
+async function checkCondition(condition, context) {
+  if (isMaxTurnsCondition(condition)) {
+    return checkMaxTurns(condition, context);
+  }
+  if (isFieldValueCondition(condition)) {
+    return checkFieldValue(condition, context);
+  }
+  if (isFieldSetCondition(condition)) {
+    return checkFieldSet(condition, context);
+  }
+  if (isCustomCondition(condition)) {
+    return checkCustom(condition, context);
+  }
+  const _exhaustive = condition;
+  throw new EvalError(`Unknown condition type: ${JSON.stringify(_exhaustive)}`, {
+    code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */,
+    context: { condition: _exhaustive }
+  });
+}
+async function checkTermination(conditions, context) {
+  if (conditions.length === 0) {
+    return {
+      terminated: false,
+      reason: "No termination conditions specified"
+    };
+  }
+  for (const condition of conditions) {
+    const result = await checkCondition(condition, context);
+    if (result.terminated) {
+      return result;
+    }
+  }
+  return {
+    terminated: false,
+    reason: "No termination conditions met"
+  };
+}
+
+// src/utils/json.ts
+function truncate(str, maxLength) {
+  if (!str) {
+    return "";
+  }
+  if (str.length <= maxLength) {
+    return str;
+  }
+  return str.slice(0, maxLength) + "...";
+}
+
+// src/utils/condition-composites.ts
+function createAndCheck(conditions, checkFn) {
+  return async (context) => {
+    for (const condition of conditions) {
+      const result = await checkFn(condition, context);
+      if (!result.terminated) {
+        return false;
+      }
+    }
+    return true;
+  };
+}
+function createOrCheck(conditions, checkFn) {
+  return async (context) => {
+    for (const condition of conditions) {
+      const result = await checkFn(condition, context);
+      if (result.terminated) {
+        return true;
+      }
+    }
+    return false;
+  };
+}
+function createNotCheck(condition, checkFn) {
+  return async (context) => {
+    const result = await checkFn(condition, context);
+    return !result.terminated;
+  };
+}
+function formatCompositeDescription(type, conditions) {
+  if (conditions.length === 0) {
+    return `${type}() - empty, never terminates`;
+  }
+  return `${type}(${conditions.map((c) => c.type).join(", ")})`;
+}
+
+// src/multi-turn/conditions.ts
+function naturalLanguage(options) {
+  const { provider, prompt, systemPrompt } = options;
+  const defaultSystemPrompt = `You are an assistant that evaluates whether a conversation should terminate.
+Analyze the conversation history and determine if the specified condition is met.
+Respond with ONLY "yes" or "no" - nothing else.`;
+  return {
+    type: "custom",
+    check: async (context) => {
+      const historyText = context.history.map(
+        (h) => `Turn ${h.turn}:
+Input: ${JSON.stringify(h.input)}
+Output: ${JSON.stringify(h.output)}`
+      ).join("\n\n");
+      const userPrompt = `## Termination Condition
+${prompt}
+
+## Conversation History
+${historyText || "(No history yet)"}
+
+## Current Turn
+Turn: ${context.currentTurn}
+Last Output: ${JSON.stringify(context.lastOutput)}
+
+Should the conversation terminate based on the condition above? Answer "yes" or "no" only.`;
+      const execution = provider.simpleExecution(async (session) => {
+        const result = await session.generateText({
+          messages: [
+            { role: "system", content: systemPrompt ?? defaultSystemPrompt },
+            { role: "user", content: userPrompt }
+          ]
+        });
+        return result.text;
+      });
+      const executionResult = await execution.result();
+      if (executionResult.status !== "succeeded") {
+        throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+      }
+      const responseText = executionResult.value;
+      const answer = responseText.toLowerCase().trim();
+      return answer === "yes" || answer.startsWith("yes");
+    },
+    description: `NL: ${truncate(prompt, 50)}`
+  };
+}
+function and(...conditions) {
+  if (conditions.length === 0) {
+    return {
+      type: "custom",
+      check: () => false,
+      description: formatCompositeDescription("and", [])
+    };
+  }
+  return {
+    type: "custom",
+    check: createAndCheck(conditions, checkCondition),
+    description: formatCompositeDescription("and", conditions)
+  };
+}
+function or(...conditions) {
+  if (conditions.length === 0) {
+    return {
+      type: "custom",
+      check: () => false,
+      description: formatCompositeDescription("or", [])
+    };
+  }
+  return {
+    type: "custom",
+    check: createOrCheck(conditions, checkCondition),
+    description: formatCompositeDescription("or", conditions)
+  };
+}
+function not(condition) {
+  return {
+    type: "custom",
+    check: createNotCheck(condition, checkCondition),
+    description: `not(${condition.type})`
+  };
+}
+function afterTurns(count) {
+  return {
+    type: "custom",
+    check: (context) => context.currentTurn >= count,
+    description: `afterTurns(${count})`
+  };
+}
+function fieldEquals(fieldPath, expectedValue) {
+  return {
+    type: "custom",
+    check: async (context) => {
+      const result = await checkCondition(
+        { type: "fieldValue", fieldPath, expectedValue },
+        context
+      );
+      return result.terminated;
+    },
+    description: `fieldEquals(${fieldPath}, ${JSON.stringify(expectedValue)})`
+  };
+}
+function fieldIsSet(fieldPath) {
+  return {
+    type: "custom",
+    check: async (context) => {
+      const result = await checkCondition({ type: "fieldSet", fieldPath }, context);
+      return result.terminated;
+    },
+    description: `fieldIsSet(${fieldPath})`
+  };
+}
+
+// src/multi-turn/runner.ts
+var import_core = require("@agtlantis/core");
+var DEFAULT_MAX_TURNS = 10;
+var DEFAULT_ON_CONDITION_MET = "pass";
+var DEFAULT_ON_MAX_TURNS_REACHED = "fail";
+function aggregateTokenUsage(usages) {
+  return usages.reduce(
+    (acc, usage) => ({
+      inputTokens: acc.inputTokens + usage.inputTokens,
+      outputTokens: acc.outputTokens + usage.outputTokens,
+      totalTokens: acc.totalTokens + usage.totalTokens
+    }),
+    { inputTokens: 0, outputTokens: 0, totalTokens: 0 }
+  );
+}
+function getEffectiveMaxTurns(conditions, safetyLimit) {
+  const maxTurnsCondition = conditions.find((c) => c.type === "maxTurns");
+  if (maxTurnsCondition && maxTurnsCondition.type === "maxTurns") {
+    return Math.min(maxTurnsCondition.count, safetyLimit);
+  }
+  return safetyLimit;
+}
+async function resolveInput(followUpInput, context) {
+  const inputValue = followUpInput.input;
+  if (typeof inputValue === "function") {
+    const result = inputValue(context);
+    return result instanceof Promise ? await result : result;
+  }
+  return inputValue;
+}
+function buildContext(currentTurn, history) {
+  return {
+    currentTurn,
+    history,
+    lastOutput: history.length > 0 ? history[history.length - 1].output : void 0
+  };
+}
+function getFollowUpInput(followUpInputs, followUpIndex) {
+  let currentIndex = 0;
+  for (const followUp of followUpInputs) {
+    const repeatCount = followUp.turns ?? 1;
+    if (!Number.isFinite(repeatCount) && followUpIndex >= currentIndex) {
+      return followUp;
+    }
+    if (followUpIndex < currentIndex + repeatCount) {
+      return followUp;
+    }
+    currentIndex += repeatCount;
+  }
+  return null;
+}
+function validateFollowUpInputs(followUpInputs) {
+  for (let i = 0; i < followUpInputs.length; i++) {
+    const followUp = followUpInputs[i];
+    if (followUp.turns === void 0) {
+      continue;
+    }
+    if (typeof followUp.turns !== "number" || followUp.turns < 1) {
+      throw new EvalError("turns must be a positive number or Infinity", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: {
+          description: followUp.description,
+          turns: followUp.turns
+        }
+      });
+    }
+    if (!Number.isFinite(followUp.turns) && i < followUpInputs.length - 1) {
+      throw new EvalError(
+        "turns: Infinity must be the last followUpInput (subsequent items would be unreachable)",
+        {
+          code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+          context: {
+            description: followUp.description,
+            position: i,
+            totalItems: followUpInputs.length
+          }
+        }
+      );
+    }
+  }
+}
+async function getTurnInput(turn, testCaseInput, followUpInputs, conversationHistory) {
+  if (turn === 1) {
+    return { type: "success", input: testCaseInput };
+  }
+  const followUpIndex = turn - 2;
+  const followUp = getFollowUpInput(followUpInputs, followUpIndex);
+  if (!followUp) {
+    return { type: "exhausted" };
+  }
+  const ctx = buildContext(turn, conversationHistory);
+  const input = await resolveInput(followUp, ctx);
+  return { type: "success", input };
+}
+function isFileResolutionError(result) {
+  return "type" in result && result.type === "fileResolutionError";
+}
+async function executeSingleTurn(input, agent, testCaseId, turn) {
+  let resolvedInput;
+  try {
+    resolvedInput = await (0, import_core.resolveFileSourcesInInput)(input, {
+      basePath: process.cwd()
+    });
+  } catch (e) {
+    return {
+      type: "fileResolutionError",
+      reason: `FileSource resolution failed on turn ${turn}: ${e instanceof Error ? e.message : String(e)}`
+    };
+  }
+  const startTime = performance.now();
+  let output;
+  let metadata;
+  let error;
+  try {
+    const agentResult = await agent.execute(resolvedInput);
+    output = agentResult.result;
+    metadata = agentResult.metadata;
+  } catch (e) {
+    error = EvalError.from(e, "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */, {
+      testCaseId,
+      turn,
+      agentName: agent.config.name
+    });
+  }
+  const latencyMs = performance.now() - startTime;
+  return { output, metadata, latencyMs, error };
+}
+function determinePassFromTermination(termination, onConditionMet, onMaxTurnsReached) {
+  if (!isTerminated(termination)) {
+    return true;
+  }
+  switch (termination.terminationType) {
+    case "error":
+    case "exhausted":
+      return false;
+    case "maxTurns":
+      return onMaxTurnsReached === "pass";
+    case "condition":
+      return onConditionMet === "pass";
+    default:
+      return true;
+  }
+}
+async function executeMultiTurnTestCase(testCase2, context, options) {
+  const { agent, judge, agentDescription } = context;
+  const { multiTurn } = testCase2;
+  const signal = options?.signal;
+  const maxTurns = getEffectiveMaxTurns(
+    multiTurn.terminateWhen,
+    multiTurn.maxTurns ?? DEFAULT_MAX_TURNS
+  );
+  const onConditionMet = multiTurn.onConditionMet ?? DEFAULT_ON_CONDITION_MET;
+  const onMaxTurnsReached = multiTurn.onMaxTurnsReached ?? DEFAULT_ON_MAX_TURNS_REACHED;
+  const followUpInputs = multiTurn.followUpInputs ?? [];
+  validateFollowUpInputs(followUpInputs);
+  const conversationHistory = [];
+  const tokenUsages = [];
+  let totalLatencyMs = 0;
+  let termination = {
+    terminated: false,
+    reason: "Execution not started"
+  };
+  for (let turn = 1; turn <= maxTurns; turn++) {
+    if (signal?.aborted) {
+      throw new EvalError("Multi-turn test execution aborted", {
+        code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+        context: { testCaseId: testCase2.id, turn, reason: "aborted" }
+      });
+    }
+    const inputResult = await getTurnInput(
+      turn,
+      testCase2.input,
+      followUpInputs,
+      conversationHistory
+    );
+    if (inputResult.type === "exhausted") {
+      termination = {
+        terminated: true,
+        terminationType: "exhausted",
+        reason: "All follow-up inputs exhausted"
+      };
+      break;
+    }
+    const input = inputResult.input;
+    const turnResult = await executeSingleTurn(input, agent, testCase2.id ?? "unknown", turn);
+    if (isFileResolutionError(turnResult)) {
+      termination = {
+        terminated: true,
+        terminationType: "error",
+        reason: turnResult.reason
+      };
+      break;
+    }
+    const {
+      output: agentOutput,
+      metadata: agentMetadata,
+      latencyMs,
+      error: agentError
+    } = turnResult;
+    totalLatencyMs += latencyMs;
+    const turnUsage = agentMetadata?.tokenUsage ?? {
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0
+    };
+    tokenUsages.push(turnUsage);
+    conversationHistory.push({
+      turn,
+      input,
+      output: agentOutput,
+      metadata: agentMetadata
+    });
+    if (agentError) {
+      termination = {
+        terminated: true,
+        terminationType: "error",
+        reason: `Agent execution failed on turn ${turn}: ${agentError.message}`
+      };
+      break;
+    }
+    const ctx = buildContext(turn, conversationHistory);
+    termination = await checkTermination(multiTurn.terminateWhen, ctx);
+    if (termination.terminated) {
+      break;
+    }
+    if (turn >= maxTurns) {
+      termination = {
+        terminated: true,
+        terminationType: "maxTurns",
+        matchedCondition: { type: "maxTurns", count: maxTurns },
+        reason: `Maximum turns reached (${maxTurns})`
+      };
+      break;
+    }
+  }
+  const aggregatedTokenUsage = aggregateTokenUsage(tokenUsages);
+  const metrics = {
+    latencyMs: totalLatencyMs,
+    tokenUsage: aggregatedTokenUsage
+  };
+  const lastTurn = conversationHistory[conversationHistory.length - 1];
+  const finalOutput = lastTurn?.output;
+  const judgeResult = await judge.evaluate({
+    input: testCase2.input,
+    output: finalOutput,
+    agentDescription,
+    files: testCase2.files
+  });
+  const passedTermination = determinePassFromTermination(
+    termination,
+    onConditionMet,
+    onMaxTurnsReached
+  );
+  const passed = passedTermination && judgeResult.passed;
+  return {
+    testCase: testCase2,
+    output: finalOutput,
+    metrics,
+    verdicts: judgeResult.verdicts,
+    overallScore: judgeResult.overallScore,
+    passed,
+    judgeMetadata: judgeResult.metadata,
+    conversationHistory,
+    termination,
+    totalTurns: conversationHistory.length
+  };
+}
+
+// src/multi-turn/ai-user.ts
+var DEFAULT_SYSTEM_PROMPT = `You are simulating a realistic user in a conversation with an AI assistant.
+
+## Your Role
+Generate natural, context-appropriate user messages based on the conversation history.
+
+## Guidelines
+
+1. **Stay in Character**: Respond as a real user would - with natural language, occasional typos, or casual phrasing when appropriate.
+
+2. **Be Goal-Oriented**: Users have objectives. Pursue them logically based on the conversation context:
+   - If the assistant asks a question, provide a reasonable answer
+   - If clarification is needed, ask for it naturally
+   - If a task is progressing, guide it toward completion
+
+3. **React Appropriately**: Respond to what the assistant says:
+   - Acknowledge when the assistant is helpful
+   - Express confusion if the response is unclear
+   - Correct misunderstandings if they occur
+
+4. **Keep It Realistic**: Real users:
+   - Don't always provide perfect information upfront
+   - May change their mind or add requirements
+   - Sometimes need time to think or decide
+
+## Output Format
+Respond with ONLY the user's message. No additional formatting, explanation, or meta-commentary.`;
+function aiUser(options) {
+  const { provider, systemPrompt, formatHistory, buildInput } = options;
+  const defaultFormatHistory = (ctx) => ctx.history.map(
+    (h, i) => `[Turn ${i + 1}]
+User: ${JSON.stringify(h.input)}
+Assistant: ${JSON.stringify(h.output)}`
+  ).join("\n\n");
+  return async (context) => {
+    const historyText = (formatHistory ?? defaultFormatHistory)(context);
+    const resolvedSystemPrompt = typeof systemPrompt === "function" ? systemPrompt(context) : systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    const userPrompt = historyText ? `## Conversation History
+${historyText}
+
+## Your Task
+Generate the next user message based on the conversation above:` : `## Your Task
+This is the start of a new conversation. Generate an appropriate opening message from the user:`;
+    const execution = provider.simpleExecution(async (session) => {
+      const result = await session.generateText({
+        messages: [
+          { role: "system", content: resolvedSystemPrompt },
+          { role: "user", content: userPrompt }
+        ]
+      });
+      return result.text;
+    });
+    const executionResult = await execution.result();
+    if (executionResult.status !== "succeeded") {
+      throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+    }
+    const responseText = executionResult.value;
+    return buildInput(responseText, context);
+  };
+}
+
+// src/utils/semaphore.ts
+function createSemaphore(limit) {
+  let running = 0;
+  const waiting = [];
+  return {
+    async acquire() {
+      if (running < limit) {
+        running++;
+        return;
+      }
+      return new Promise((resolve2) => {
+        waiting.push(resolve2);
+      });
+    },
+    release() {
+      running--;
+      const next = waiting.shift();
+      if (next) {
+        running++;
+        next();
+      }
+    }
+  };
+}
+
+// src/core/constants.ts
+var SCORE = {
+  /** Minimum possible score */
+  MIN: 0,
+  /** Maximum possible score */
+  MAX: 100,
+  /** Default threshold for passing evaluation */
+  DEFAULT_PASS_THRESHOLD: 70,
+  /** Threshold for majority-based pass determination (50%) */
+  MAJORITY_PASS_THRESHOLD: 0.5
+};
+var ZERO_TOKEN_USAGE = {
+  inputTokens: 0,
+  outputTokens: 0,
+  totalTokens: 0
+};
+
+// src/core/runner.ts
+async function executeTestCase(testCase2, context, signal) {
+  const { agent, judge, agentDescription } = context;
+  if (signal?.aborted) {
+    throw new EvalError("Test execution aborted", {
+      code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+      context: { testCaseId: testCase2.id, reason: "aborted" }
+    });
+  }
+  let resolvedInput;
+  try {
+    resolvedInput = await (0, import_core2.resolveFileSourcesInInput)(testCase2.input, {
+      basePath: process.cwd()
+    });
+  } catch (e) {
+    const error2 = EvalError.from(e, "FILE_READ_ERROR" /* FILE_READ_ERROR */, {
+      testCaseId: testCase2.id,
+      agentName: agent.config.name
+    });
+    return createFailedResult(testCase2, error2);
+  }
+  const startTime = performance.now();
+  let output;
+  let tokenUsage = ZERO_TOKEN_USAGE;
+  let error;
+  try {
+    const agentResult = await agent.execute(resolvedInput);
+    output = agentResult.result;
+    if (agentResult.metadata?.tokenUsage) {
+      tokenUsage = agentResult.metadata.tokenUsage;
+    }
+  } catch (e) {
+    error = EvalError.from(e, "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */, {
+      testCaseId: testCase2.id,
+      agentName: agent.config.name
+    });
+    output = void 0;
+  }
+  const latencyMs = performance.now() - startTime;
+  const metrics = { latencyMs, tokenUsage };
+  const testResult = { testCase: testCase2, output, metrics, error };
+  if (error) {
+    return {
+      kind: "single-turn",
+      ...testResult,
+      verdicts: [],
+      overallScore: 0,
+      passed: false,
+      judgeMetadata: void 0
+    };
+  }
+  if (signal?.aborted) {
+    throw new EvalError("Test execution aborted before evaluation", {
+      code: "AGENT_EXECUTION_ERROR" /* AGENT_EXECUTION_ERROR */,
+      context: { testCaseId: testCase2.id, reason: "aborted" }
+    });
+  }
+  const judgeResult = await judge.evaluate({
+    input: testCase2.input,
+    output,
+    agentDescription,
+    files: testCase2.files
+  });
+  return {
+    kind: "single-turn",
+    ...testResult,
+    verdicts: judgeResult.verdicts,
+    overallScore: judgeResult.overallScore,
+    passed: judgeResult.passed,
+    judgeMetadata: judgeResult.metadata
+  };
+}
+function createFailedResult(testCase2, error) {
+  return {
+    kind: "single-turn",
+    testCase: testCase2,
+    output: void 0,
+    metrics: { latencyMs: 0, tokenUsage: ZERO_TOKEN_USAGE },
+    error,
+    verdicts: [],
+    overallScore: 0,
+    passed: false,
+    judgeMetadata: void 0
+  };
+}
+function toMultiTurnResult(result) {
+  return {
+    kind: "multi-turn",
+    testCase: result.testCase,
+    output: result.output,
+    metrics: result.metrics,
+    verdicts: result.verdicts,
+    overallScore: result.overallScore,
+    passed: result.passed,
+    judgeMetadata: result.judgeMetadata,
+    conversationHistory: result.conversationHistory,
+    totalTurns: result.totalTurns,
+    terminationReason: result.termination.reason,
+    termination: result.termination
+  };
+}
+async function runWithConcurrency(testCases2, context, options = {}) {
+  const { concurrency = 1, stopOnFirstFailure = false, signal } = options;
+  if (concurrency < 1) {
+    throw new EvalError("Concurrency must be at least 1", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { concurrency }
+    });
+  }
+  if (testCases2.length === 0) {
+    return [];
+  }
+  const semaphore = createSemaphore(concurrency);
+  const results = [];
+  let shouldStop = false;
+  let firstError;
+  const internalAbort = new AbortController();
+  const propagateExternalAbort = () => {
+    shouldStop = true;
+    internalAbort.abort();
+  };
+  signal?.addEventListener("abort", propagateExternalAbort);
+  if (signal?.aborted) {
+    shouldStop = true;
+  }
+  try {
+    const executeOne = async (testCase2, index) => {
+      if (shouldStop) return;
+      await semaphore.acquire();
+      try {
+        if (shouldStop) return;
+        const result = await executeTestCaseByType(testCase2, context, internalAbort.signal);
+        results[index] = result;
+        if (stopOnFirstFailure && !result.passed) {
+          shouldStop = true;
+          internalAbort.abort();
+        }
+      } catch (e) {
+        if (!firstError && !isAbortError(e)) {
+          firstError = e instanceof Error ? e : new Error(String(e));
+        }
+        shouldStop = true;
+        internalAbort.abort();
+      } finally {
+        semaphore.release();
+      }
+    };
+    const promises = testCases2.map((tc, i) => executeOne(tc, i));
+    await Promise.allSettled(promises);
+    if (firstError) {
+      throw firstError;
+    }
+    return results.filter((r) => r !== void 0);
+  } finally {
+    signal?.removeEventListener("abort", propagateExternalAbort);
+  }
+}
+function isAbortError(e) {
+  return e instanceof DOMException && e.name === "AbortError" || e instanceof EvalError && e.context?.reason === "aborted";
+}
+async function executeTestCaseByType(testCase2, context, signal) {
+  if (isMultiTurnTestCase(testCase2)) {
+    const multiTurnResult = await executeMultiTurnTestCase(testCase2, context, { signal });
+    return toMultiTurnResult(multiTurnResult);
+  }
+  return executeTestCase(testCase2, context, signal);
+}
+
+// src/core/types.ts
+function toEvalAgent(agent) {
+  return {
+    config: {
+      name: agent.config.name,
+      description: agent.config.description
+    },
+    prompt: agent.prompt,
+    execute: async (input, options) => {
+      const result = await agent.execute(input, options);
+      return {
+        result: result.result,
+        metadata: result.metadata
+      };
+    }
+  };
+}
+function isSingleTurnResult(result) {
+  return result.kind === "single-turn" || result.kind === "single-turn-iterated";
+}
+function isMultiTurnResult(result) {
+  return result.kind === "multi-turn" || result.kind === "multi-turn-iterated";
+}
+function isIteratedResult(result) {
+  return result.kind === "single-turn-iterated" || result.kind === "multi-turn-iterated";
+}
+
+// src/core/iteration.ts
+function calculateIterationStats(results) {
+  if (results.length === 0) {
+    return {
+      iterations: 0,
+      scores: [],
+      mean: 0,
+      stdDev: 0,
+      min: 0,
+      max: 0,
+      passRate: 0,
+      passCount: 0
+    };
+  }
+  const scores = results.map((r) => r.overallScore);
+  const passCount = results.filter((r) => r.passed).length;
+  const mean = scores.reduce((sum, s) => sum + s, 0) / scores.length;
+  const variance = scores.reduce((sum, s) => sum + Math.pow(s - mean, 2), 0) / scores.length;
+  const stdDev = Math.sqrt(variance);
+  return {
+    iterations: results.length,
+    scores,
+    mean,
+    stdDev,
+    min: Math.min(...scores),
+    max: Math.max(...scores),
+    passRate: passCount / results.length,
+    passCount
+  };
+}
+function calculateMultiTurnIterationStats(results) {
+  const baseStats = calculateIterationStats(results);
+  const turns = results.map((r) => r.totalTurns);
+  const terminationCounts = {};
+  for (const r of results) {
+    const type = r.termination.terminationType;
+    if (type) {
+      terminationCounts[type] = (terminationCounts[type] || 0) + 1;
+    }
+  }
+  return {
+    ...baseStats,
+    avgTurns: turns.length > 0 ? turns.reduce((a, b) => a + b, 0) / turns.length : 0,
+    minTurns: turns.length > 0 ? Math.min(...turns) : 0,
+    maxTurns: turns.length > 0 ? Math.max(...turns) : 0,
+    terminationCounts
+  };
+}
+function selectRepresentativeResult(results, mean) {
+  if (results.length === 0) {
+    throw new EvalError("Cannot select representative result from empty array", {
+      code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */
+    });
+  }
+  return results.reduce((closest, current) => {
+    const closestDiff = Math.abs(closest.overallScore - mean);
+    const currentDiff = Math.abs(current.overallScore - mean);
+    return currentDiff < closestDiff ? current : closest;
+  });
+}
+function aggregateIterationResults(allIterationResults) {
+  if (allIterationResults.length === 0) {
+    return [];
+  }
+  const testCount = allIterationResults[0].length;
+  const aggregated = [];
+  for (let i = 0; i < testCount; i++) {
+    const resultsForTestCase = allIterationResults.map((iteration) => iteration[i]);
+    const stats = calculateIterationStats(resultsForTestCase);
+    const representative = selectRepresentativeResult(resultsForTestCase, stats.mean);
+    const isMultiTurn = resultsForTestCase.some((r) => isMultiTurnResult(r));
+    const passedByMajority = stats.passRate >= SCORE.MAJORITY_PASS_THRESHOLD;
+    if (isMultiTurn) {
+      const multiTurnResults = resultsForTestCase.filter(
+        (r) => isMultiTurnResult(r)
+      );
+      const multiTurnRep = representative;
+      const aggregatedResult = {
+        kind: "multi-turn-iterated",
+        testCase: multiTurnRep.testCase,
+        output: multiTurnRep.output,
+        metrics: multiTurnRep.metrics,
+        verdicts: multiTurnRep.verdicts,
+        error: multiTurnRep.error,
+        overallScore: stats.mean,
+        passed: passedByMajority,
+        iterationStats: stats,
+        iterationResults: resultsForTestCase,
+        conversationHistory: multiTurnRep.conversationHistory,
+        totalTurns: multiTurnRep.totalTurns,
+        terminationReason: multiTurnRep.terminationReason,
+        termination: multiTurnRep.termination,
+        multiTurnIterationStats: calculateMultiTurnIterationStats(multiTurnResults)
+      };
+      aggregated.push(aggregatedResult);
+    } else {
+      const aggregatedResult = {
+        kind: "single-turn-iterated",
+        testCase: representative.testCase,
+        output: representative.output,
+        metrics: representative.metrics,
+        verdicts: representative.verdicts,
+        error: representative.error,
+        overallScore: stats.mean,
+        passed: passedByMajority,
+        iterationStats: stats,
+        iterationResults: resultsForTestCase
+      };
+      aggregated.push(aggregatedResult);
+    }
+  }
+  return aggregated;
+}
+function filterIteratedResults(results) {
+  return results.filter(
+    (r) => r.kind === "single-turn-iterated" || r.kind === "multi-turn-iterated"
+  );
+}
+function averageIterationStat(results, selector) {
+  const iteratedResults = filterIteratedResults(results);
+  if (iteratedResults.length === 0) {
+    return void 0;
+  }
+  const total = iteratedResults.reduce((sum, r) => sum + selector(r.iterationStats), 0);
+  return total / iteratedResults.length;
+}
+function calculateAvgStdDev(results) {
+  return averageIterationStat(results, (stats) => stats.stdDev);
+}
+function calculateAvgPassRate(results) {
+  return averageIterationStat(results, (stats) => stats.passRate);
+}
+
+// src/core/suite.ts
+function calculateAggregatedMetrics(results) {
+  if (results.length === 0) {
+    return { avgLatencyMs: 0, totalTokens: 0 };
+  }
+  const totalLatencyMs = sumBy(results, (r) => r.metrics.latencyMs);
+  const totalTokens = sumBy(results, (r) => r.metrics.tokenUsage.totalTokens);
+  return {
+    avgLatencyMs: totalLatencyMs / results.length,
+    totalTokens
+  };
+}
+function sumBy(items, selector) {
+  return items.reduce((sum, item) => sum + selector(item), 0);
+}
+function calculateSummary(results, iterations) {
+  const metrics = calculateAggregatedMetrics(results);
+  const passedCount = results.filter((r) => r.passed).length;
+  const failedCount = results.length - passedCount;
+  const avgScore = results.length > 0 ? sumBy(results, (r) => r.overallScore) / results.length : 0;
+  const summary = {
+    totalTests: results.length,
+    passed: passedCount,
+    failed: failedCount,
+    avgScore,
+    metrics
+  };
+  const hasMultipleIterations = iterations && iterations > 1;
+  if (hasMultipleIterations) {
+    summary.iterations = iterations;
+    summary.avgStdDev = calculateAvgStdDev(results);
+    summary.avgPassRate = calculateAvgPassRate(results);
+  }
+  return summary;
+}
+function createEvalSuite(config) {
+  const { agent, agentDescription, judge, improver } = config;
+  const description = agentDescription ?? agent.config.description ?? agent.config.name;
+  const suite = {
+    async run(testCases2, options) {
+      const iterations = options?.iterations ?? 1;
+      validateIterations(iterations);
+      const executeContext = { agent, judge, agentDescription: description };
+      const results = iterations <= 1 ? await runWithConcurrency(testCases2, executeContext, options) : await runMultipleIterations(testCases2, executeContext, options, iterations);
+      const summary = calculateSummary(results, iterations > 1 ? iterations : void 0);
+      const suggestions = improver ? (await improver.improve(agent.prompt, results)).suggestions : [];
+      return {
+        summary,
+        results,
+        suggestions,
+        generatedAt: /* @__PURE__ */ new Date(),
+        promptVersion: agent.prompt.version
+      };
+    },
+    withAgent(newAgent) {
+      return createEvalSuite({
+        ...config,
+        agent: newAgent,
+        agentDescription: void 0
+      });
+    }
+  };
+  return suite;
+}
+function validateIterations(iterations) {
+  if (iterations < 1 || !Number.isInteger(iterations)) {
+    throw new EvalError(
+      `Invalid iterations value: ${iterations}. Must be a positive integer.`,
+      { code: "INVALID_CONFIG" /* INVALID_CONFIG */, context: { iterations } }
+    );
+  }
+}
+async function runMultipleIterations(testCases2, executeContext, options, iterations) {
+  const allIterationResults = [];
+  for (let i = 0; i < iterations; i++) {
+    const iterationResults = await runWithConcurrency(
+      testCases2,
+      executeContext,
+      { ...options, iterations: void 0 }
+    );
+    allIterationResults.push(iterationResults);
+  }
+  return aggregateIterationResults(allIterationResults);
+}
+
+// src/index.ts
+var import_core8 = require("@agtlantis/core");
+
+// src/judge/llm-judge.ts
+var import_ai = require("ai");
+var import_zod = require("zod");
+
+// src/judge/prompts/default.ts
+var defaultJudgePrompt = {
+  id: "default-judge",
+  version: "2.0.0",
+  system: `You are an expert evaluator specializing in assessing AI Agent outputs.
+
+Your role is to fairly and thoroughly evaluate the agent's output against the provided criteria.
+
+## Evaluation Principles
+
+1. **Scoring**: Assign a score between 0-100 for each criterion
+   - 90-100: Exceptional - Exceeds expectations with no significant issues
+   - 70-89: Good - Meets expectations with minor issues
+   - 50-69: Acceptable - Partially meets expectations, notable issues present
+   - 30-49: Poor - Falls short of expectations, significant issues
+   - 0-29: Failing - Does not meet minimum requirements
+
+2. **Reasoning**: Always provide specific, evidence-based reasoning
+   - Quote or reference specific parts of the output
+   - Explain both strengths and weaknesses
+   - Be constructive and actionable in feedback
+
+3. **Objectivity**: Evaluate based solely on the criteria provided
+   - Avoid personal preferences or unstated requirements
+   - Consider the agent's intended purpose and context
+   - Weight severity of issues proportionally
+
+## Response Format
+
+You MUST respond with valid JSON only. No additional text or explanation outside the JSON structure.
+
+{
+  "verdicts": [
+    {
+      "criterionId": "criterion-id",
+      "score": 0-100,
+      "reasoning": "Detailed explanation with specific evidence from the output",
+      "passed": true/false
+    }
+  ]
+}`,
+  renderUserPrompt: (ctx) => {
+    const fileSection = buildFileSection(ctx.files);
+    return `
+## Agent Under Evaluation
+${ctx.agentDescription}
+
+## Input Provided to Agent
+\`\`\`json
+${JSON.stringify(ctx.input, null, 2)}
+\`\`\`
+${fileSection}
+## Agent Output
+\`\`\`json
+${JSON.stringify(ctx.output, null, 2)}
+\`\`\`
+
+## Evaluation Criteria
+${ctx.criteria.map((c) => `- **${c.name}** (id: ${c.id}, weight: ${c.weight ?? 1}): ${c.description}`).join("\n")}
+
+Please evaluate the agent's output against each criterion listed above.`.trim();
+  }
+};
+function buildFileSection(files) {
+  if (!files || files.length === 0) {
+    return "";
+  }
+  return `
+## Reference Files
+${files.map((f) => `### ${f.path}
+\`\`\`
+${f.content}
+\`\`\``).join("\n\n")}
+`;
+}
+
+// src/judge/llm-judge.ts
+function toEvalTokenUsage(usage) {
+  return {
+    inputTokens: usage.inputTokens ?? 0,
+    outputTokens: usage.outputTokens ?? 0,
+    totalTokens: usage.totalTokens ?? 0
+  };
+}
+function hasValidator(criterion) {
+  return "validator" in criterion && typeof criterion.validator === "function";
+}
+var JudgeResponseSchema = import_zod.z.object({
+  verdicts: import_zod.z.array(
+    import_zod.z.object({
+      criterionId: import_zod.z.string(),
+      score: import_zod.z.number().min(SCORE.MIN).max(SCORE.MAX),
+      reasoning: import_zod.z.string(),
+      passed: import_zod.z.boolean().optional()
+    })
+  )
+});
+function validateAllCriteriaHaveVerdicts(verdicts, criteriaIds) {
+  const providedIds = new Set(verdicts.map((v) => v.criterionId));
+  const missingIds = criteriaIds.filter((id) => !providedIds.has(id));
+  if (missingIds.length > 0) {
+    throw new EvalError("Judge response missing verdicts for some criteria", {
+      code: "VERDICT_PARSE_ERROR" /* VERDICT_PARSE_ERROR */,
+      context: { missingCriteriaIds: missingIds, providedIds: [...providedIds] }
+    });
+  }
+}
+function calculateOverallScore(verdicts, criteriaWeights) {
+  let totalWeight = 0;
+  let weightedSum = 0;
+  for (const verdict of verdicts) {
+    const weight = criteriaWeights.get(verdict.criterionId) ?? 1;
+    weightedSum += verdict.score * weight;
+    totalWeight += weight;
+  }
+  if (totalWeight === 0) {
+    return 0;
+  }
+  return Math.round(weightedSum / totalWeight * 100) / 100;
+}
+function runValidatorCriteria(validatorCriteria, output) {
+  return validatorCriteria.map((criterion) => {
+    const result = criterion.validator(output);
+    if (result.valid) {
+      return {
+        criterionId: criterion.id,
+        score: 100,
+        reasoning: `${criterion.name} \uD1B5\uACFC`,
+        passed: true
+      };
+    }
+    return {
+      criterionId: criterion.id,
+      score: 0,
+      reasoning: `${criterion.name} \uC2E4\uD328:
+${result.errorSummary ?? "\uC720\uD6A8\uC131 \uAC80\uC99D \uC624\uB958"}`,
+      passed: false
+    };
+  });
+}
+async function runLLMEvaluation(provider, prompt, context, llmCriteriaIds, passThreshold) {
+  const messages = [
+    { role: "system", content: prompt.system },
+    { role: "user", content: prompt.renderUserPrompt(context) }
+  ];
+  let response;
+  let usage;
+  try {
+    const execution = provider.simpleExecution(async (session) => {
+      const result = await session.generateText({
+        messages,
+        output: import_ai.Output.object({ schema: JudgeResponseSchema })
+      });
+      return result.output;
+    });
+    const executionResult = await execution.result();
+    if (executionResult.status !== "succeeded") {
+      throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+    }
+    response = executionResult.value;
+    usage = executionResult.summary.totalLLMUsage;
+  } catch (cause) {
+    throw EvalError.from(cause, "LLM_API_ERROR" /* LLM_API_ERROR */, {
+      promptId: prompt.id,
+      promptVersion: prompt.version
+    });
+  }
+  validateAllCriteriaHaveVerdicts(response.verdicts, llmCriteriaIds);
+  const verdicts = response.verdicts.map((v) => ({
+    criterionId: v.criterionId,
+    score: v.score,
+    reasoning: v.reasoning,
+    passed: v.passed ?? v.score >= passThreshold
+  }));
+  return { verdicts, usage };
+}
+function createJudge(config) {
+  const {
+    provider,
+    prompt = defaultJudgePrompt,
+    criteria,
+    passThreshold = SCORE.DEFAULT_PASS_THRESHOLD,
+    model
+  } = config;
+  const validatorCriteria = [];
+  const llmCriteria = [];
+  const criteriaWeights = /* @__PURE__ */ new Map();
+  const llmCriteriaIds = [];
+  for (const c of criteria) {
+    criteriaWeights.set(c.id, c.weight ?? 1);
+    if (hasValidator(c)) {
+      validatorCriteria.push(c);
+    } else {
+      llmCriteria.push(c);
+      llmCriteriaIds.push(c.id);
+    }
+  }
+  return {
+    async evaluate(evalContext) {
+      const { input, output, agentDescription, files } = evalContext;
+      const validatorVerdicts = runValidatorCriteria(validatorCriteria, output);
+      let llmVerdicts = [];
+      let llmUsage;
+      if (llmCriteria.length > 0) {
+        const context = {
+          agentDescription,
+          input,
+          output,
+          criteria: llmCriteria,
+          files
+        };
+        const llmResult = await runLLMEvaluation(
+          provider,
+          prompt,
+          context,
+          llmCriteriaIds,
+          passThreshold
+        );
+        llmVerdicts = llmResult.verdicts;
+        llmUsage = llmResult.usage;
+      }
+      const allVerdicts = [...validatorVerdicts, ...llmVerdicts];
+      const overallScore = calculateOverallScore(allVerdicts, criteriaWeights);
+      const passed = overallScore >= passThreshold;
+      const metadata = llmUsage ? { tokenUsage: toEvalTokenUsage(llmUsage), model } : void 0;
+      return {
+        verdicts: allVerdicts,
+        overallScore,
+        passed,
+        metadata
+      };
+    }
+  };
+}
+
+// src/judge/criteria/validate-schema.ts
+function formatZodErrors(error) {
+  return error.errors.map((e) => {
+    const path3 = e.path.length > 0 ? `${e.path.join(".")}: ` : "";
+    return `- ${path3}${e.message}`;
+  }).join("\n");
+}
+function schema(options) {
+  const { schema: schema2, id, weight, name, description } = options;
+  return {
+    id: id ?? "schema-validation",
+    name: name ?? "\uC2A4\uD0A4\uB9C8 \uC720\uD6A8\uC131",
+    description: description ?? "\uCD9C\uB825\uC774 \uC9C0\uC815\uB41C \uC2A4\uD0A4\uB9C8(Zod)\uB97C \uC900\uC218\uD558\uB294\uC9C0 \uD504\uB85C\uADF8\uB798\uBC0D \uBC29\uC2DD\uC73C\uB85C \uAC80\uC99D\uD569\uB2C8\uB2E4.",
+    weight,
+    validator: (output) => {
+      const result = schema2.safeParse(output);
+      if (result.success) {
+        return { valid: true };
+      }
+      return {
+        valid: false,
+        errors: result.error.errors,
+        errorSummary: formatZodErrors(result.error)
+      };
+    }
+  };
+}
+
+// src/judge/criteria/index.ts
+function accuracy(options) {
+  return {
+    id: "accuracy",
+    name: "Accuracy",
+    description: "Evaluates whether the output is factually correct, free from errors, and avoids hallucinations. Check for incorrect facts, made-up information, or misrepresentation of the input data.",
+    weight: options?.weight
+  };
+}
+function consistency(options) {
+  return {
+    id: "consistency",
+    name: "Consistency",
+    description: "Evaluates whether the output is internally coherent and logically consistent. Check for self-contradictions, conflicting statements, or logical inconsistencies within the response.",
+    weight: options?.weight
+  };
+}
+function relevance(options) {
+  return {
+    id: "relevance",
+    name: "Relevance",
+    description: "Evaluates whether the output directly addresses the input and fulfills the user intent. Check for off-topic content, missing key requirements, or responses that fail to answer the actual question.",
+    weight: options?.weight
+  };
+}
+
+// src/reporter/markdown.ts
+var import_promises = require("fs/promises");
+var import_core3 = require("@agtlantis/core");
+var PASS_ICON = "\u2705";
+var FAIL_ICON = "\u274C";
+var PRIORITY_ORDER = { high: 0, medium: 1, low: 2 };
+function reportToMarkdown(report, options = {}) {
+  const {
+    expandPassedTests = false,
+    includeRawOutput = false,
+    outputPreviewLength = 200
+  } = options;
+  const { summary, results, suggestions, generatedAt, promptVersion } = report;
+  const passRate = summary.totalTests > 0 ? (summary.passed / summary.totalTests * 100).toFixed(1) : "0.0";
+  const lines = [];
+  lines.push("# Evaluation Report");
+  lines.push("");
+  lines.push(`> Generated: ${generatedAt.toISOString()}`);
+  lines.push(`> Prompt Version: ${promptVersion}`);
+  lines.push("");
+  lines.push("## Summary");
+  lines.push("");
+  lines.push(`| Metric | Value |`);
+  lines.push(`|--------|-------|`);
+  lines.push(`| Total Tests | ${summary.totalTests} |`);
+  if (summary.iterations && summary.iterations > 1) {
+    lines.push(`| **Iterations** | **${summary.iterations}** |`);
+  }
+  lines.push(`| Passed | ${summary.passed} (${passRate}%) |`);
+  lines.push(`| Failed | ${summary.failed} |`);
+  if (summary.avgStdDev !== void 0) {
+    lines.push(
+      `| Average Score | ${summary.avgScore.toFixed(1)} \xB1 ${summary.avgStdDev.toFixed(1)} |`
+    );
+  } else {
+    lines.push(`| Average Score | ${summary.avgScore.toFixed(1)} |`);
+  }
+  if (summary.avgPassRate !== void 0) {
+    lines.push(`| Avg Pass Rate | ${(summary.avgPassRate * 100).toFixed(1)}% |`);
+  }
+  lines.push(`| Avg Latency | ${summary.metrics.avgLatencyMs.toFixed(0)}ms |`);
+  lines.push(`| Total Tokens | ${summary.metrics.totalTokens} |`);
+  if (summary.costSummary?.total !== void 0) {
+    lines.push(`| Est. Cost | $${summary.costSummary.total.toFixed(4)} |`);
+  }
+  lines.push("");
+  const failedResults = results.filter((r) => !r.passed);
+  if (failedResults.length > 0) {
+    lines.push(`## ${FAIL_ICON} Failed Tests`);
+    lines.push("");
+    for (const result of failedResults) {
+      lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+    }
+  }
+  const passedResults = results.filter((r) => r.passed);
+  if (passedResults.length > 0) {
+    lines.push(`## ${PASS_ICON} Passed Tests`);
+    lines.push("");
+    if (expandPassedTests) {
+      for (const result of passedResults) {
+        lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+      }
+    } else {
+      lines.push("<details>");
+      lines.push("<summary>Click to expand passed tests</summary>");
+      lines.push("");
+      for (const result of passedResults) {
+        lines.push(formatTestResult(result, outputPreviewLength, includeRawOutput));
+      }
+      lines.push("</details>");
+      lines.push("");
+    }
+  }
+  if (suggestions.length > 0) {
+    lines.push("## \u{1F4A1} Improvement Suggestions");
+    lines.push("");
+    const sortedSuggestions = [...suggestions].sort(
+      (a, b) => PRIORITY_ORDER[a.priority] - PRIORITY_ORDER[b.priority]
+    );
+    for (const suggestion of sortedSuggestions) {
+      lines.push(formatSuggestion(suggestion));
+    }
+  }
+  return lines.join("\n");
+}
+async function saveReportMarkdown(report, path3, options) {
+  const markdown = reportToMarkdown(report, options);
+  await (0, import_promises.writeFile)(path3, markdown, "utf-8");
+}
+function jsonCodeBlock(value, maxLength) {
+  const json = JSON.stringify(value, null, 2);
+  const content = maxLength !== void 0 ? truncate(json, maxLength) : json;
+  return ["```json", content, "```"];
+}
+function passFailIcon(passed) {
+  return passed ? PASS_ICON : FAIL_ICON;
+}
+function formatTestResult(result, previewLength, includeRaw) {
+  const lines = [];
+  const testId = result.testCase.id ?? "unnamed";
+  const scoreDisplay = result.iterationStats ? `${result.overallScore.toFixed(1)} \xB1 ${result.iterationStats.stdDev.toFixed(1)}` : result.overallScore.toFixed(1);
+  lines.push(`### ${testId} (Score: ${scoreDisplay})`);
+  lines.push("");
+  if (result.testCase.description) {
+    lines.push(`> ${result.testCase.description}`);
+    lines.push("");
+  }
+  const fileDisplayInfos = (0, import_core3.getFileSourcesDisplayInfo)(result.testCase.input);
+  if (fileDisplayInfos.length > 0) {
+    lines.push("**Files:**");
+    for (const info of fileDisplayInfos) {
+      const namePrefix = info.filename ? `${info.filename} - ` : "";
+      lines.push(`- ${namePrefix}${info.source}: ${info.description} (${info.mediaType})`);
+    }
+    lines.push("");
+  }
+  if (result.totalTurns !== void 0) {
+    lines.push(
+      `**Multi-turn:** ${result.totalTurns} turns | Termination: ${result.terminationReason ?? "unknown"}`
+    );
+    lines.push("");
+  }
+  if (result.multiTurnIterationStats) {
+    lines.push(...formatMultiTurnIterationStats(result.multiTurnIterationStats));
+  }
+  if (result.iterationStats && result.iterationResults) {
+    lines.push(...formatIterationResults(result.iterationStats, result.iterationResults));
+  }
+  if (result.conversationHistory && result.conversationHistory.length > 0) {
+    lines.push(...formatConversationHistory(result.conversationHistory, previewLength));
+  } else {
+    lines.push(
+      ...formatSingleTurnInputOutput(result.testCase.input, result.output, previewLength)
+    );
+  }
+  lines.push("**Verdicts:**");
+  for (const verdict of result.verdicts) {
+    lines.push(
+      `- ${passFailIcon(verdict.passed)} **${verdict.criterionId}**: ${verdict.score} - ${verdict.reasoning}`
+    );
+  }
+  lines.push("");
+  if (includeRaw) {
+    lines.push("<details>");
+    lines.push("<summary>Raw Output</summary>");
+    lines.push("");
+    lines.push(...jsonCodeBlock(result.output));
+    lines.push("</details>");
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatMultiTurnIterationStats(stats) {
+  const terminationSummary = Object.entries(stats.terminationCounts).map(([type, count]) => `${type}: ${count}`).join(", ") || "none";
+  return [
+    "**Multi-turn Iteration Statistics:**",
+    "",
+    "| Metric | Value |",
+    "|--------|-------|",
+    `| Avg Turns | ${stats.avgTurns.toFixed(1)} |`,
+    `| Min/Max Turns | ${stats.minTurns} / ${stats.maxTurns} |`,
+    `| Termination Distribution | ${terminationSummary} |`,
+    ""
+  ];
+}
+function formatIterationResults(stats, results) {
+  const lines = [
+    "**Iteration Results:**",
+    "",
+    "| # | Score | Passed | Latency |",
+    "|---|-------|--------|---------|"
+  ];
+  results.forEach((iter, idx) => {
+    lines.push(
+      `| ${idx + 1} | ${iter.overallScore.toFixed(1)} | ${passFailIcon(iter.passed)} | ${iter.metrics.latencyMs.toFixed(0)}ms |`
+    );
+  });
+  lines.push("");
+  lines.push(
+    `**Stats:** ${stats.mean.toFixed(1)} \xB1 ${stats.stdDev.toFixed(1)} (min: ${stats.min.toFixed(0)}, max: ${stats.max.toFixed(0)}, pass rate: ${(stats.passRate * 100).toFixed(0)}%)`
+  );
+  lines.push("");
+  return lines;
+}
+function formatConversationHistory(history, previewLength) {
+  const lines = ["**Conversation History:**", ""];
+  for (const turn of history) {
+    lines.push("<details>");
+    lines.push(`<summary>Turn ${turn.turn}</summary>`);
+    lines.push("");
+    lines.push("**Input:**");
+    lines.push(...jsonCodeBlock(turn.input, previewLength));
+    lines.push("");
+    lines.push("**Output:**");
+    lines.push(...jsonCodeBlock(turn.output, previewLength));
+    lines.push("</details>");
+    lines.push("");
+  }
+  return lines;
+}
+function formatSingleTurnInputOutput(input, output, previewLength) {
+  return [
+    "**Input:**",
+    ...jsonCodeBlock(input, previewLength),
+    "",
+    "**Output:**",
+    ...jsonCodeBlock(output, previewLength),
+    ""
+  ];
+}
+function formatSuggestion(suggestion) {
+  const lines = [];
+  const priorityIcon = { high: "\u{1F534}", medium: "\u{1F7E1}", low: "\u{1F7E2}" }[suggestion.priority] ?? "\u26AA";
+  lines.push(`### ${priorityIcon} [${suggestion.priority.toUpperCase()}] ${suggestion.type}`);
+  lines.push("");
+  lines.push(`**Reasoning:** ${suggestion.reasoning}`);
+  lines.push("");
+  lines.push(`**Expected Improvement:** ${suggestion.expectedImprovement}`);
+  lines.push("");
+  lines.push("**Diff:**");
+  lines.push("```diff");
+  lines.push(`- ${suggestion.currentValue.split("\n").join("\n- ")}`);
+  lines.push(`+ ${suggestion.suggestedValue.split("\n").join("\n+ ")}`);
+  lines.push("```");
+  lines.push("");
+  return lines.join("\n");
+}
+function compareReports(before, after) {
+  const scoreDelta = after.summary.avgScore - before.summary.avgScore;
+  const beforePassRate = before.summary.totalTests > 0 ? before.summary.passed / before.summary.totalTests : 0;
+  const afterPassRate = after.summary.totalTests > 0 ? after.summary.passed / after.summary.totalTests : 0;
+  const passRateDelta = afterPassRate - beforePassRate;
+  const metricsDelta = {
+    latencyMs: after.summary.metrics.avgLatencyMs - before.summary.metrics.avgLatencyMs,
+    tokenUsage: after.summary.metrics.totalTokens - before.summary.metrics.totalTokens
+  };
+  const beforeScores = buildScoreMap(before.results);
+  const afterScores = buildScoreMap(after.results);
+  const improved = [];
+  const regressed = [];
+  for (const [id, afterScore] of afterScores) {
+    const beforeScore = beforeScores.get(id);
+    if (beforeScore === void 0) continue;
+    if (afterScore > beforeScore) {
+      improved.push(id);
+    } else if (afterScore < beforeScore) {
+      regressed.push(id);
+    }
+  }
+  const removed = [...beforeScores.keys()].filter((id) => !afterScores.has(id));
+  return {
+    scoreDelta,
+    passRateDelta,
+    metricsDelta,
+    improved,
+    regressed,
+    removed
+  };
+}
+function buildScoreMap(results) {
+  const scoreMap = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    scoreMap.set(result.testCase.id ?? "unnamed", result.overallScore);
+  }
+  return scoreMap;
+}
+
+// src/reporter/json-reporter.ts
+var import_node_fs2 = require("fs");
+
+// src/reporter/cost-helpers.ts
+var import_core4 = require("@agtlantis/core");
+function toLanguageModelUsage(usage) {
+  return {
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    totalTokens: usage.totalTokens
+  };
+}
+var PROVIDER_MAPPING = {
+  gemini: "google",
+  openai: "openai",
+  anthropic: "anthropic",
+  google: "google"
+};
+function detectProvider(model) {
+  if (!model) return "google";
+  if (model.startsWith("gpt-") || model.startsWith("o1") || model.startsWith("o3")) {
+    return "openai";
+  }
+  if (model.startsWith("gemini-")) {
+    return "google";
+  }
+  if (model.startsWith("claude-")) {
+    return "anthropic";
+  }
+  return "google";
+}
+function normalizeProvider(provider) {
+  if (!provider) return "google";
+  return PROVIDER_MAPPING[provider] ?? provider;
+}
+function calculateComponentCost(tokenUsage, model, provider, config) {
+  if (!tokenUsage) return void 0;
+  const normalizedProvider = provider ? normalizeProvider(provider) : detectProvider(model);
+  const providerPricing = config?.providerPricing?.[normalizedProvider];
+  const result = (0, import_core4.calculateCostFromUsage)(
+    toLanguageModelUsage(tokenUsage),
+    model ?? "unknown",
+    normalizedProvider,
+    providerPricing
+  );
+  return result.total;
+}
+function buildCostBreakdown(costs) {
+  const total = (costs.agent ?? 0) + (costs.judge ?? 0) + (costs.improver ?? 0);
+  return {
+    ...costs,
+    total: total > 0 ? total : void 0
+  };
+}
+function calculateResultCost(result, config) {
+  const agentCost = calculateComponentCost(
+    result.metrics.tokenUsage,
+    result.agentMetadata?.model,
+    result.agentMetadata?.provider,
+    config
+  );
+  const judgeCost = result.judgeMetadata?.tokenUsage ? calculateComponentCost(
+    result.judgeMetadata.tokenUsage,
+    result.judgeMetadata.model,
+    result.judgeMetadata.provider,
+    config
+  ) : void 0;
+  return buildCostBreakdown({
+    agent: agentCost,
+    judge: judgeCost
+  });
+}
+function calculateReportCosts(report, config) {
+  let totalAgent = 0;
+  let totalJudge = 0;
+  for (const result of report.results) {
+    const breakdown = calculateResultCost(result, config);
+    totalAgent += breakdown.agent ?? 0;
+    totalJudge += breakdown.judge ?? 0;
+  }
+  return {
+    total: totalAgent + totalJudge,
+    byComponent: {
+      agent: totalAgent,
+      judge: totalJudge
+    }
+  };
+}
+function addCostsToResults(results, config) {
+  return results.map((result) => {
+    const costBreakdown = calculateResultCost(result, config);
+    const metricsWithCost = {
+      latencyMs: result.metrics.latencyMs,
+      tokenUsage: result.metrics.tokenUsage,
+      costBreakdown
+    };
+    return {
+      testCase: result.testCase,
+      output: result.output,
+      metrics: metricsWithCost,
+      error: result.error,
+      verdicts: result.verdicts,
+      overallScore: result.overallScore,
+      passed: result.passed
+    };
+  });
+}
+
+// src/reporter/format-utils.ts
+var import_node_fs = require("fs");
+var import_node_path = __toESM(require("path"), 1);
+function formatScoreDelta(delta) {
+  if (delta === null) {
+    return "-";
+  }
+  const sign = delta >= 0 ? "+" : "";
+  return `${sign}${delta.toFixed(1)}`;
+}
+function buildOutputPath(outputDir, name, extension, addTimestamp) {
+  (0, import_node_fs.mkdirSync)(outputDir, { recursive: true });
+  const filename = addTimestamp ? `${name}-${Date.now()}.${extension}` : `${name}.${extension}`;
+  return import_node_path.default.join(outputDir, filename);
+}
+function toISOStringIfDate(value) {
+  return value instanceof Date ? value.toISOString() : value;
+}
+
+// src/reporter/json-reporter.ts
+var JsonReporter = class {
+  outputDir;
+  pricing;
+  addTimestamp;
+  constructor(options) {
+    this.outputDir = options.outputDir;
+    this.pricing = options.pricing;
+    this.addTimestamp = options.addTimestamp ?? true;
+  }
+  save(report, name) {
+    const filepath = buildOutputPath(this.outputDir, name, "json", this.addTimestamp);
+    const costs = this.pricing ? calculateReportCosts(report, this.pricing) : void 0;
+    const output = {
+      summary: report.summary,
+      results: report.results,
+      suggestions: report.suggestions,
+      generatedAt: report.generatedAt.toISOString(),
+      promptVersion: report.promptVersion,
+      ...costs && { costs }
+    };
+    (0, import_node_fs2.writeFileSync)(filepath, JSON.stringify(output, null, 2));
+    return filepath;
+  }
+};
+
+// src/reporter/markdown-reporter.ts
+var import_node_fs3 = require("fs");
+var MarkdownReporter = class {
+  outputDir;
+  addTimestamp;
+  markdownOptions;
+  constructor(options) {
+    this.outputDir = options.outputDir;
+    this.addTimestamp = options.addTimestamp ?? true;
+    this.markdownOptions = options.markdown ?? {};
+  }
+  save(report, name) {
+    const filepath = buildOutputPath(this.outputDir, name, "md", this.addTimestamp);
+    const markdown = reportToMarkdown(report, this.markdownOptions);
+    (0, import_node_fs3.writeFileSync)(filepath, markdown);
+    return filepath;
+  }
+};
+
+// src/reporter/console-reporter.ts
+var ConsoleReporter = class {
+  verbosity;
+  pricing;
+  constructor(options = {}) {
+    this.verbosity = options.verbosity ?? "summary";
+    this.pricing = options.pricing;
+  }
+  log(report) {
+    const { summary } = report;
+    const passRate = summary.totalTests > 0 ? summary.passed / summary.totalTests : 0;
+    console.log(`
+\u{1F4CA} Eval Report: ${summary.totalTests} tests`);
+    console.log(`   Score: ${summary.avgScore.toFixed(1)} | Pass Rate: ${(passRate * 100).toFixed(0)}%`);
+    if (this.verbosity === "summary") {
+      this.logCostIfAvailable(report);
+      return;
+    }
+    console.log("");
+    for (const result of report.results) {
+      const testId = result.testCase.id || "unknown";
+      const status = result.passed ? "\u2713" : "\u2717";
+      console.log(`   ${status} [${testId}] Score: ${result.overallScore.toFixed(1)}`);
+      if (this.verbosity === "full") {
+        console.log(`      Input: ${truncate(JSON.stringify(result.testCase.input), 80)}`);
+        console.log(`      Output: ${truncate(String(result.output), 80)}`);
+      }
+    }
+    this.logCostIfAvailable(report);
+  }
+  logCostIfAvailable(report) {
+    if (this.pricing) {
+      const costs = calculateReportCosts(report, this.pricing);
+      console.log(`
+   \u{1F4B0} Cost: $${costs.total.toFixed(4)}`);
+    }
+  }
+};
+
+// src/reporter/composite-reporter.ts
+var CompositeReporter = class {
+  constructor(reporters) {
+    this.reporters = reporters;
+  }
+  /**
+   * Saves to all reporters that support saving.
+   * Returns the first successful file path (usually JsonReporter).
+   */
+  save(report, name) {
+    const errors = [];
+    let firstPath;
+    for (const reporter of this.reporters) {
+      if (!reporter.save) {
+        reporter.log?.(report);
+        continue;
+      }
+      try {
+        const savedPath = reporter.save(report, name);
+        if (!firstPath) firstPath = savedPath;
+      } catch (error) {
+        errors.push({
+          reporter: reporter.constructor.name,
+          error
+        });
+      }
+      reporter.log?.(report);
+    }
+    if (!firstPath) {
+      const details = errors.length > 0 ? errors.map((e) => `${e.reporter}: ${e.error.message}`).join(", ") : "No reporters support save()";
+      throw new Error(`No reporter saved successfully. ${details}`);
+    }
+    return firstPath;
+  }
+  log(report) {
+    for (const reporter of this.reporters) {
+      reporter.log?.(report);
+    }
+  }
+};
+
+// src/reporter/factory.ts
+function createJsonReporter(outputDir, options) {
+  return new JsonReporter({ outputDir, ...options });
+}
+function createMarkdownReporter(outputDir, options) {
+  return new MarkdownReporter({ outputDir, ...options });
+}
+function createConsoleReporter(options) {
+  return new ConsoleReporter(options);
+}
+function createCompositeReporter(reporters) {
+  return new CompositeReporter(reporters);
+}
+function createDefaultReporter(outputDir, options) {
+  return new CompositeReporter([
+    new JsonReporter({
+      outputDir,
+      pricing: options?.pricing,
+      addTimestamp: options?.addTimestamp
+    }),
+    new ConsoleReporter({
+      verbosity: options?.verbosity,
+      pricing: options?.pricing
+    })
+  ]);
+}
+
+// src/reporter/runner.ts
+function createReportRunner(options) {
+  const { outputDir, pricing, verbosity } = options;
+  const jsonReporter = new JsonReporter({ outputDir, pricing });
+  const consoleReporter = verbosity !== false ? new ConsoleReporter({ verbosity: verbosity || "summary", pricing }) : null;
+  return async (suite, testCases2, name) => {
+    const report = await suite.run(testCases2);
+    consoleReporter?.log(report);
+    const savedPath = jsonReporter.save(report, name);
+    return { report, savedPath };
+  };
+}
+
+// src/reporter/cycle-json.ts
+var import_node_fs4 = require("fs");
+var import_node_path2 = __toESM(require("path"), 1);
+function saveCycleJson(result, options) {
+  const { outputDir, name, directory, saveRounds = true } = options;
+  const cycleDir = resolveCycleDirectory(outputDir, name, directory);
+  (0, import_node_fs4.mkdirSync)(cycleDir, { recursive: true });
+  saveCycleSummary(cycleDir, result);
+  if (saveRounds) {
+    saveRoundReports(cycleDir, result.rounds);
+  }
+  return cycleDir;
+}
+function resolveCycleDirectory(outputDir, name, directory) {
+  if (directory) {
+    return directory;
+  }
+  if (outputDir && name) {
+    return import_node_path2.default.join(outputDir, `${name}-${Date.now()}`);
+  }
+  throw new Error('saveCycleJson requires either "directory" or both "outputDir" and "name"');
+}
+function saveCycleSummary(cycleDir, result) {
+  const summaryPath = import_node_path2.default.join(cycleDir, "cycle-summary.json");
+  const summary = {
+    rounds: result.rounds.map((round) => ({
+      round: round.round,
+      completedAt: toISOStringIfDate(round.completedAt),
+      score: round.report.summary.avgScore,
+      scoreDelta: round.scoreDelta,
+      cost: round.cost,
+      suggestionsGenerated: round.suggestionsGenerated.length,
+      suggestionsApproved: round.suggestionsApproved.length,
+      promptVersionAfter: round.promptVersionAfter
+    })),
+    terminationReason: result.terminationReason,
+    totalCost: result.totalCost,
+    roundCount: result.rounds.length,
+    initialScore: result.rounds[0]?.report.summary.avgScore ?? null,
+    finalScore: result.rounds[result.rounds.length - 1]?.report.summary.avgScore ?? null
+  };
+  (0, import_node_fs4.writeFileSync)(summaryPath, JSON.stringify(summary, null, 2));
+}
+function saveRoundReports(cycleDir, rounds) {
+  for (const round of rounds) {
+    const roundPath = import_node_path2.default.join(cycleDir, `round-${round.round}-report.json`);
+    const roundData = {
+      round: round.round,
+      completedAt: toISOStringIfDate(round.completedAt),
+      report: {
+        ...round.report,
+        generatedAt: toISOStringIfDate(round.report.generatedAt)
+      },
+      suggestionsGenerated: round.suggestionsGenerated,
+      suggestionsApproved: round.suggestionsApproved,
+      promptSnapshot: round.promptSnapshot,
+      cost: round.cost,
+      scoreDelta: round.scoreDelta
+    };
+    (0, import_node_fs4.writeFileSync)(roundPath, JSON.stringify(roundData, null, 2));
+  }
+}
+
+// src/reporter/cycle-console.ts
+function logCycle(result, options = {}) {
+  const { verbosity = "summary", showRounds = false } = options;
+  console.log("\n\u{1F504} Improvement Cycle Complete");
+  console.log(`   Rounds: ${result.rounds.length}`);
+  console.log(`   Termination: ${result.terminationReason}`);
+  console.log(`   Total Cost: $${result.totalCost.toFixed(4)}`);
+  if (result.rounds.length > 0) {
+    const firstScore = result.rounds[0].report.summary.avgScore;
+    const lastScore = result.rounds[result.rounds.length - 1].report.summary.avgScore;
+    const delta = lastScore - firstScore;
+    console.log(`   Score: ${firstScore.toFixed(1)} -> ${lastScore.toFixed(1)} (${formatScoreDelta(delta)})`);
+  }
+  if (showRounds) {
+    const consoleReporter = new ConsoleReporter({ verbosity });
+    for (const round of result.rounds) {
+      console.log(`
+   -- Round ${round.round} --`);
+      consoleReporter.log(round.report);
+    }
+  }
+}
+
+// src/reporter/cycle-markdown.ts
+var import_node_fs5 = require("fs");
+function cycleToMarkdown(result, options = {}) {
+  const { includeRoundDetails = true, showPromptEvolution = false } = options;
+  const lines = [];
+  lines.push("# Improvement Cycle Report");
+  lines.push("");
+  lines.push("## Summary");
+  lines.push("");
+  lines.push("| Metric | Value |");
+  lines.push("|--------|-------|");
+  lines.push(`| Rounds | ${result.rounds.length} |`);
+  lines.push(`| Termination | ${result.terminationReason} |`);
+  lines.push(`| Total Cost | $${result.totalCost.toFixed(4)} |`);
+  if (result.rounds.length > 0) {
+    const first = result.rounds[0].report.summary.avgScore;
+    const last = result.rounds[result.rounds.length - 1].report.summary.avgScore;
+    lines.push(`| Initial Score | ${first.toFixed(1)} |`);
+    lines.push(`| Final Score | ${last.toFixed(1)} |`);
+    lines.push(`| Improvement | ${formatScoreDelta(last - first)} |`);
+  }
+  lines.push("");
+  lines.push("## Score Progression");
+  lines.push("");
+  lines.push("| Round | Score | Delta | Cost |");
+  lines.push("|-------|-------|-------|------|");
+  for (const round of result.rounds) {
+    const delta = formatScoreDelta(round.scoreDelta);
+    lines.push(
+      `| ${round.round} | ${round.report.summary.avgScore.toFixed(1)} | ${delta} | $${round.cost.total.toFixed(4)} |`
+    );
+  }
+  lines.push("");
+  if (includeRoundDetails) {
+    lines.push("## Round Details");
+    lines.push("");
+    for (const round of result.rounds) {
+      lines.push(`### Round ${round.round}`);
+      lines.push("");
+      lines.push(reportToMarkdown(round.report));
+      lines.push("");
+    }
+  }
+  if (showPromptEvolution && result.rounds.length > 0) {
+    lines.push("## Prompt Evolution");
+    lines.push("");
+    lines.push("### Initial Prompt");
+    lines.push("");
+    lines.push("```");
+    lines.push(result.rounds[0].promptSnapshot.userTemplate);
+    lines.push("```");
+    lines.push("");
+    lines.push("### Final Prompt");
+    lines.push("");
+    lines.push("```");
+    const finalPrompt = result.finalPrompt;
+    if ("userTemplate" in finalPrompt && typeof finalPrompt.userTemplate === "string") {
+      lines.push(finalPrompt.userTemplate);
+    } else {
+      lines.push("[Compiled prompt - template not available]");
+    }
+    lines.push("```");
+  }
+  return lines.join("\n");
+}
+function saveCycleMarkdown(result, filePath, options) {
+  const markdown = cycleToMarkdown(result, options);
+  (0, import_node_fs5.writeFileSync)(filePath, markdown);
+}
+
+// src/improver/utils.ts
+var import_core5 = require("@agtlantis/core");
+function suggestionDiff(suggestion) {
+  const oldLines = suggestion.currentValue.split("\n");
+  const newLines = suggestion.suggestedValue.split("\n");
+  const lines = [];
+  lines.push(`--- ${suggestion.type} (current)`);
+  lines.push(`+++ ${suggestion.type} (suggested)`);
+  lines.push("");
+  for (const line of oldLines) {
+    lines.push(`- ${line}`);
+  }
+  for (const line of newLines) {
+    lines.push(`+ ${line}`);
+  }
+  return lines.join("\n");
+}
+function suggestionPreview(suggestion) {
+  const lines = [];
+  lines.push(`=== Suggestion Preview ===`);
+  lines.push(`Type: ${suggestion.type}`);
+  lines.push(`Priority: ${suggestion.priority}`);
+  lines.push(``);
+  lines.push(`Reasoning: ${suggestion.reasoning}`);
+  lines.push(``);
+  lines.push(`Expected Improvement: ${suggestion.expectedImprovement}`);
+  lines.push(``);
+  lines.push(`--- Current Value ---`);
+  lines.push(suggestion.currentValue);
+  lines.push(``);
+  lines.push(`--- Suggested Value ---`);
+  lines.push(suggestion.suggestedValue);
+  return lines.join("\n");
+}
+function suggestionSummary(suggestion) {
+  const priorityTag = `[${suggestion.priority.toUpperCase()}]`;
+  return `${priorityTag} ${suggestion.type}: ${truncate(suggestion.reasoning, 60)}`;
+}
+function safeReplace(str, search, replacement) {
+  return str.replace(search, () => replacement);
+}
+function bumpVersion(version, bump) {
+  const parts = version.split(".").map((n) => parseInt(n, 10));
+  if (parts.length !== 3 || parts.some(isNaN)) {
+    throw new EvalError(
+      `Invalid version format: "${version}". Expected semver format (x.y.z)`,
+      {
+        code: "SUGGESTION_APPLY_ERROR" /* SUGGESTION_APPLY_ERROR */,
+        context: { version, expectedFormat: "x.y.z" }
+      }
+    );
+  }
+  const [major, minor, patch] = parts;
+  switch (bump) {
+    case "major":
+      return `${major + 1}.0.0`;
+    case "minor":
+      return `${major}.${minor + 1}.0`;
+    case "patch":
+      return `${major}.${minor}.${patch + 1}`;
+  }
+}
+function applyPromptSuggestions(currentPrompt, suggestions, options) {
+  const approvedSuggestions = suggestions.filter((s) => s.approved);
+  if (approvedSuggestions.length === 0) {
+    return {
+      prompt: currentPrompt,
+      appliedCount: 0,
+      skipped: []
+    };
+  }
+  let newPrompt = { ...currentPrompt };
+  let appliedCount = 0;
+  const skipped = [];
+  for (const suggestion of approvedSuggestions) {
+    const applyResult = applySingleSuggestion(newPrompt, suggestion);
+    if (applyResult.success) {
+      newPrompt = applyResult.prompt;
+      appliedCount++;
+    } else {
+      skipped.push({ suggestion, reason: applyResult.reason });
+    }
+  }
+  if (options?.bumpVersion && appliedCount > 0) {
+    newPrompt = {
+      ...newPrompt,
+      version: bumpVersion(currentPrompt.version, options.bumpVersion)
+    };
+  }
+  return {
+    prompt: newPrompt,
+    appliedCount,
+    skipped
+  };
+}
+var AGENT_PROMPT_CORE_FIELDS = [
+  "id",
+  "version",
+  "system",
+  "renderUserPrompt",
+  "userTemplate"
+];
+function applySingleSuggestion(prompt, suggestion) {
+  switch (suggestion.type) {
+    case "system_prompt": {
+      if (!prompt.system.includes(suggestion.currentValue)) {
+        return {
+          success: false,
+          reason: `currentValue not found in system prompt: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      return {
+        success: true,
+        prompt: {
+          ...prompt,
+          system: safeReplace(
+            prompt.system,
+            suggestion.currentValue,
+            suggestion.suggestedValue
+          )
+        }
+      };
+    }
+    case "user_prompt": {
+      const userTemplate = prompt.userTemplate;
+      if (typeof userTemplate !== "string") {
+        throw new EvalError(
+          `Cannot apply user_prompt suggestion: prompt does not have a userTemplate field. The renderUserPrompt is a function and cannot be modified directly.`,
+          {
+            code: "SUGGESTION_APPLY_ERROR" /* SUGGESTION_APPLY_ERROR */,
+            context: {
+              suggestionType: suggestion.type,
+              hasUserTemplate: "userTemplate" in prompt
+            }
+          }
+        );
+      }
+      if (!userTemplate.includes(suggestion.currentValue)) {
+        return {
+          success: false,
+          reason: `currentValue not found in userTemplate: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      const newTemplate = safeReplace(
+        userTemplate,
+        suggestion.currentValue,
+        suggestion.suggestedValue
+      );
+      return {
+        success: true,
+        prompt: {
+          ...prompt,
+          userTemplate: newTemplate,
+          renderUserPrompt: (0, import_core5.compileTemplate)(newTemplate, prompt.id)
+        }
+      };
+    }
+    case "parameters": {
+      const updatedPrompt = { ...prompt };
+      let found = false;
+      for (const [key, value] of Object.entries(updatedPrompt)) {
+        if (AGENT_PROMPT_CORE_FIELDS.includes(
+          key
+        )) {
+          continue;
+        }
+        if (typeof value === "string" && value.includes(suggestion.currentValue)) {
+          updatedPrompt[key] = safeReplace(
+            value,
+            suggestion.currentValue,
+            suggestion.suggestedValue
+          );
+          found = true;
+          break;
+        }
+      }
+      if (!found) {
+        return {
+          success: false,
+          reason: `currentValue not found in any parameter field: "${truncate(suggestion.currentValue, 50)}"`
+        };
+      }
+      return {
+        success: true,
+        prompt: updatedPrompt
+      };
+    }
+    default: {
+      const _exhaustive = suggestion.type;
+      return {
+        success: false,
+        reason: `Unknown suggestion type: ${suggestion.type}`
+      };
+    }
+  }
+}
+
+// src/improver/llm-improver.ts
+var import_ai2 = require("ai");
+var import_zod2 = require("zod");
+
+// src/improver/prompts/default.ts
+var defaultImproverPrompt = {
+  id: "default-improver",
+  version: "2.0.0",
+  system: `You are an expert prompt engineer specializing in optimizing AI Agent prompts.
+
+Your role is to analyze test results and evaluation feedback to propose targeted improvements.
+
+## Improvement Principles
+
+1. **Focus on Impact**: Prioritize changes that address the lowest-scoring criteria
+   - Target specific failure patterns, not general improvements
+   - One well-crafted change is better than many superficial ones
+
+2. **Be Specific and Actionable**: Provide concrete changes, not vague suggestions
+   - Show exact text to add, modify, or remove
+   - Explain the mechanism by which the change will help
+
+3. **Consider Trade-offs**: Evaluate side effects of each change
+   - Will this fix break other test cases?
+   - Does it increase prompt length/cost significantly?
+   - Could it introduce new failure modes?
+
+4. **Maintain Prompt Quality**: Preserve clarity and structure
+   - Keep prompts readable and maintainable
+   - Avoid over-engineering or excessive constraints
+   - Ensure changes align with the agent's core purpose
+
+## Suggestion Priority Levels
+- **high**: Critical issues causing test failures, should be addressed immediately
+- **medium**: Issues affecting quality scores, recommended for next iteration
+- **low**: Minor optimizations, nice-to-have improvements
+
+## Response Format
+
+You MUST respond with valid JSON only. No additional text outside the JSON structure.
+
+{
+  "suggestions": [
+    {
+      "type": "system_prompt" | "user_prompt" | "parameters",
+      "priority": "high" | "medium" | "low",
+      "currentValue": "The specific text or value being changed",
+      "suggestedValue": "The proposed replacement text or value",
+      "reasoning": "Why this change addresses the identified issue",
+      "expectedImprovement": "Predicted impact on scores and behavior"
+    }
+  ]
+}`,
+  renderUserPrompt: (ctx) => {
+    const failedDetails = buildFailedCaseDetails(ctx.evaluatedResults);
+    return `
+## Current Agent Prompt
+
+### System Prompt
+\`\`\`
+${ctx.agentPrompt.system}
+\`\`\`
+
+## Test Results Summary
+- Total tests: ${ctx.evaluatedResults.length}
+- Passed: ${ctx.evaluatedResults.filter((r) => r.passed).length}
+- Failed: ${ctx.evaluatedResults.filter((r) => !r.passed).length}
+
+## Performance Metrics
+- Average latency: ${ctx.aggregatedMetrics.avgLatencyMs}ms
+- Total tokens used: ${ctx.aggregatedMetrics.totalTokens}
+
+## Failed/Low-Score Cases Details
+${failedDetails}
+
+Based on the above results, please propose specific prompt improvements.`.trim();
+  }
+};
+function buildFailedCaseDetails(results) {
+  const failedOrLowScore = results.filter((r) => !r.passed || r.overallScore < 70);
+  if (failedOrLowScore.length === 0) {
+    return "(None - all tests passed with acceptable scores)";
+  }
+  return failedOrLowScore.map(
+    (r) => `
+### ${r.testCase.id ?? "unnamed"} (Score: ${r.overallScore})
+**Input:** ${truncate(JSON.stringify(r.testCase.input), 200)}
+**Output:** ${truncate(JSON.stringify(r.output), 200)}
+**Evaluation:**
+${r.verdicts.map((v) => `- ${v.criterionId}: ${v.score}/100 - ${v.reasoning}`).join("\n")}`
+  ).join("\n");
+}
+
+// src/improver/llm-improver.ts
+function toEvalTokenUsage2(usage) {
+  return {
+    inputTokens: usage.inputTokens ?? 0,
+    outputTokens: usage.outputTokens ?? 0,
+    totalTokens: usage.totalTokens ?? 0
+  };
+}
+var ImproverResponseSchema = import_zod2.z.object({
+  suggestions: import_zod2.z.array(
+    import_zod2.z.object({
+      type: import_zod2.z.enum(["system_prompt", "user_prompt", "parameters"]),
+      priority: import_zod2.z.enum(["high", "medium", "low"]),
+      currentValue: import_zod2.z.string(),
+      suggestedValue: import_zod2.z.string(),
+      reasoning: import_zod2.z.string(),
+      expectedImprovement: import_zod2.z.string()
+    })
+  )
+});
+function aggregateMetrics(results) {
+  if (results.length === 0) {
+    return {
+      avgLatencyMs: 0,
+      totalTokens: 0
+    };
+  }
+  let totalLatency = 0;
+  let totalTokens = 0;
+  for (const result of results) {
+    totalLatency += result.metrics.latencyMs;
+    totalTokens += result.metrics.tokenUsage.totalTokens;
+  }
+  return {
+    avgLatencyMs: Math.round(totalLatency / results.length),
+    totalTokens
+  };
+}
+function createImprover(config) {
+  const { provider, prompt = defaultImproverPrompt, model } = config;
+  return {
+    async improve(agentPrompt, results) {
+      const context = {
+        agentPrompt,
+        evaluatedResults: results,
+        aggregatedMetrics: aggregateMetrics(results)
+      };
+      const messages = [
+        { role: "system", content: prompt.system },
+        { role: "user", content: prompt.renderUserPrompt(context) }
+      ];
+      let response;
+      let llmUsage;
+      try {
+        const execution = provider.simpleExecution(async (session) => {
+          const result = await session.generateText({
+            messages,
+            output: import_ai2.Output.object({ schema: ImproverResponseSchema })
+          });
+          return result.output;
+        });
+        const executionResult = await execution.result();
+        if (executionResult.status !== "succeeded") {
+          throw executionResult.status === "failed" ? executionResult.error : new Error("Execution was canceled");
+        }
+        response = executionResult.value;
+        llmUsage = executionResult.summary.totalLLMUsage;
+      } catch (cause) {
+        throw EvalError.from(cause, "LLM_API_ERROR" /* LLM_API_ERROR */, {
+          promptId: prompt.id,
+          promptVersion: prompt.version
+        });
+      }
+      const suggestions = response.suggestions.map((s) => ({
+        ...s,
+        approved: void 0,
+        modified: void 0
+      }));
+      const metadata = llmUsage ? { tokenUsage: toEvalTokenUsage2(llmUsage), model } : void 0;
+      return { suggestions, metadata };
+    }
+  };
+}
+
+// src/index.ts
+var import_testing = require("@agtlantis/core/testing");
+
+// src/testing/mock-agent.ts
+function createMockAgent(config = {}) {
+  const {
+    name = "MockAgent",
+    description = "A mock agent for testing",
+    response = {},
+    tokenUsage = { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
+    delay = 0,
+    shouldError = false,
+    errorMessage = "Mock agent execution failed",
+    executeFn
+  } = config;
+  return {
+    config: { name, description },
+    prompt: {
+      id: "mock-prompt",
+      version: "1.0.0",
+      system: "You are a mock agent",
+      renderUserPrompt: (input) => JSON.stringify(input)
+    },
+    execute: async (input) => {
+      if (executeFn) {
+        return executeFn(input);
+      }
+      if (delay > 0) {
+        await new Promise((resolve2) => setTimeout(resolve2, delay));
+      }
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      return {
+        result: response,
+        metadata: { tokenUsage }
+      };
+    }
+  };
+}
+function createMockJudge(config = {}) {
+  const {
+    score = 80,
+    passed = true,
+    verdicts = [
+      { criterionId: "default", score: 80, reasoning: "Default verdict", passed: true }
+    ],
+    metadata,
+    shouldError = false,
+    errorMessage = "Mock judge evaluation failed",
+    evaluateFn
+  } = config;
+  return {
+    evaluate: async (context) => {
+      if (evaluateFn) {
+        return evaluateFn(context);
+      }
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      return {
+        verdicts,
+        overallScore: score,
+        passed,
+        metadata
+      };
+    }
+  };
+}
+function createMockImprover(config = {}) {
+  const {
+    suggestions = [],
+    shouldError = false,
+    errorMessage = "Mock improver failed",
+    improveFn
+  } = config;
+  return {
+    improve: async (agentPrompt, results) => {
+      if (improveFn) {
+        return improveFn(agentPrompt, results);
+      }
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      return { suggestions };
+    }
+  };
+}
+
+// src/index.ts
+var import_core9 = require("@agtlantis/core");
+var import_core10 = require("@agtlantis/core");
+
+// src/cli/config/types.ts
+function defineConfig(config) {
+  return config;
+}
+
+// src/cli/config/loader.ts
+var import_node_fs6 = require("fs");
+var import_node_path3 = require("path");
+var import_node_url = require("url");
+var import_bundle_require = require("bundle-require");
+var import_fast_glob = __toESM(require("fast-glob"), 1);
+
+// src/cli/config/schema.ts
+var import_zod3 = require("zod");
+var llmConfigSchema = import_zod3.z.object({
+  provider: import_zod3.z.enum(["openai", "gemini"], {
+    errorMap: () => ({
+      message: "provider must be 'openai' or 'gemini'"
+    })
+  }),
+  apiKey: import_zod3.z.string().optional(),
+  defaultModel: import_zod3.z.string().optional(),
+  reasoningEffort: import_zod3.z.enum(["minimal", "low", "medium", "high"]).optional(),
+  defaultResponseFormat: import_zod3.z.object({
+    type: import_zod3.z.enum(["json_object", "text"])
+  }).optional()
+});
+var criterionSchema = import_zod3.z.object({
+  id: import_zod3.z.string().min(1, "Criterion id is required"),
+  name: import_zod3.z.string().min(1, "Criterion name is required"),
+  description: import_zod3.z.string().min(1, "Criterion description is required"),
+  weight: import_zod3.z.number().positive().optional(),
+  validator: import_zod3.z.function().optional()
+});
+var judgeConfigSchema = import_zod3.z.object({
+  llm: llmConfigSchema.optional(),
+  criteria: import_zod3.z.array(criterionSchema).min(1, "At least one criterion is required"),
+  passThreshold: import_zod3.z.number().min(0).max(100).optional(),
+  prompt: import_zod3.z.any().optional()
+});
+var improverConfigSchema = import_zod3.z.object({
+  llm: llmConfigSchema.optional(),
+  prompt: import_zod3.z.any().optional()
+}).optional();
+var outputConfigSchema = import_zod3.z.object({
+  dir: import_zod3.z.string().optional(),
+  filename: import_zod3.z.string().optional(),
+  verbose: import_zod3.z.boolean().optional()
+}).optional();
+var runConfigSchema = import_zod3.z.object({
+  concurrency: import_zod3.z.number().int().positive().optional(),
+  iterations: import_zod3.z.number().int().positive().optional(),
+  stopOnFirstFailure: import_zod3.z.boolean().optional()
+}).optional();
+var maxTurnsConditionSchema = import_zod3.z.object({
+  type: import_zod3.z.literal("maxTurns"),
+  count: import_zod3.z.number().int().positive()
+});
+var fieldSetConditionSchema = import_zod3.z.object({
+  type: import_zod3.z.literal("fieldSet"),
+  fieldPath: import_zod3.z.string().min(1)
+});
+var fieldValueConditionSchema = import_zod3.z.object({
+  type: import_zod3.z.literal("fieldValue"),
+  fieldPath: import_zod3.z.string().min(1),
+  expectedValue: import_zod3.z.unknown()
+});
+var customConditionSchema = import_zod3.z.object({
+  type: import_zod3.z.literal("custom"),
+  check: import_zod3.z.function(),
+  description: import_zod3.z.string().optional()
+});
+var terminationConditionSchema = import_zod3.z.union([
+  maxTurnsConditionSchema,
+  fieldSetConditionSchema,
+  fieldValueConditionSchema,
+  customConditionSchema
+]);
+var followUpInputSchema = import_zod3.z.object({
+  input: import_zod3.z.unknown(),
+  description: import_zod3.z.string().optional(),
+  turns: import_zod3.z.number().optional()
+});
+var multiTurnConfigSchema = import_zod3.z.object({
+  followUpInputs: import_zod3.z.array(followUpInputSchema).optional(),
+  terminateWhen: import_zod3.z.array(terminationConditionSchema).min(1, "At least one termination condition is required"),
+  maxTurns: import_zod3.z.number().int().positive().optional(),
+  onConditionMet: import_zod3.z.enum(["pass", "fail"]).optional(),
+  onMaxTurnsReached: import_zod3.z.enum(["pass", "fail"]).optional()
+});
+var testCaseSchema = import_zod3.z.object({
+  id: import_zod3.z.string().optional(),
+  input: import_zod3.z.unknown(),
+  tags: import_zod3.z.array(import_zod3.z.string()).optional(),
+  description: import_zod3.z.string().optional(),
+  expectedOutput: import_zod3.z.unknown().optional(),
+  files: import_zod3.z.array(import_zod3.z.any()).optional(),
+  multiTurn: multiTurnConfigSchema.optional()
+});
+var agentSchema = import_zod3.z.object({
+  config: import_zod3.z.object({
+    name: import_zod3.z.string(),
+    description: import_zod3.z.string().optional()
+  }),
+  prompt: import_zod3.z.object({
+    id: import_zod3.z.string(),
+    version: import_zod3.z.string(),
+    system: import_zod3.z.string(),
+    renderUserPrompt: import_zod3.z.function()
+  }),
+  execute: import_zod3.z.function()
+});
+var evalConfigSchema = import_zod3.z.object({
+  name: import_zod3.z.string().optional(),
+  agentDescription: import_zod3.z.string().optional(),
+  agent: agentSchema,
+  llm: llmConfigSchema,
+  judge: judgeConfigSchema,
+  improver: improverConfigSchema,
+  testCases: import_zod3.z.array(testCaseSchema).optional(),
+  output: outputConfigSchema,
+  run: runConfigSchema,
+  include: import_zod3.z.array(import_zod3.z.string().min(1, "Include pattern cannot be empty")).min(1, "Include array must have at least one pattern").optional(),
+  agents: import_zod3.z.record(import_zod3.z.string(), agentSchema).optional()
+}).refine(
+  (data) => {
+    const hasTestCases = (data.testCases?.length ?? 0) > 0;
+    const hasInclude = (data.include?.length ?? 0) > 0;
+    return hasTestCases || hasInclude;
+  },
+  {
+    message: "Either testCases or include must be provided. Use testCases for inline TypeScript tests, or include for YAML file discovery.",
+    path: ["testCases"]
+  }
+);
+
+// src/cli/config/loader.ts
+var ConfigError = class extends Error {
+  constructor(message, code, context) {
+    super(message);
+    this.code = code;
+    this.context = context;
+    this.name = "ConfigError";
+  }
+};
+async function discoverEvalFiles(config, options = {}) {
+  const patterns = options.include ?? config.include;
+  if (!patterns || patterns.length === 0) {
+    throw new ConfigError(
+      `No include patterns specified.
+
+Add an include field to your config:
+  include: ['evals/**/*.eval.yaml']
+
+Or use the --include CLI option:
+  npx agent-eval --include "evals/**/*.eval.yaml"`,
+      "CONFIG_NO_INCLUDE_PATTERN"
+    );
+  }
+  const cwd = options.cwd ?? process.cwd();
+  const ignore = options.ignore ?? ["**/node_modules/**"];
+  const files = await (0, import_fast_glob.default)(patterns, {
+    absolute: true,
+    cwd,
+    ignore,
+    onlyFiles: true,
+    dot: false,
+    followSymbolicLinks: false,
+    unique: true,
+    suppressErrors: false
+  });
+  return files.sort();
+}
+
+// src/improvement-cycle/types.ts
+function isTargetScoreCondition(condition) {
+  return condition.type === "targetScore";
+}
+function isMaxRoundsCondition(condition) {
+  return condition.type === "maxRounds";
+}
+function isNoImprovementCondition(condition) {
+  return condition.type === "noImprovement";
+}
+function isMaxCostCondition(condition) {
+  return condition.type === "maxCost";
+}
+function isCustomCycleCondition(condition) {
+  return condition.type === "custom";
+}
+function isCycleTerminated(result) {
+  return result.terminated === true;
+}
+
+// src/improvement-cycle/conditions.ts
+function targetScore(threshold) {
+  if (!Number.isFinite(threshold)) {
+    throw new EvalError("threshold must be a finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { threshold }
+    });
+  }
+  if (threshold < 0 || threshold > 100) {
+    throw new EvalError("threshold must be between 0 and 100", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { threshold }
+    });
+  }
+  return { type: "targetScore", threshold };
+}
+function maxRounds(count) {
+  if (!Number.isInteger(count) || count < 1) {
+    throw new EvalError("count must be a positive integer", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { count }
+    });
+  }
+  return { type: "maxRounds", count };
+}
+function noImprovement(consecutiveRounds, minDelta) {
+  if (!Number.isInteger(consecutiveRounds) || consecutiveRounds < 1) {
+    throw new EvalError("consecutiveRounds must be a positive integer", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { consecutiveRounds }
+    });
+  }
+  if (minDelta !== void 0 && (!Number.isFinite(minDelta) || minDelta < 0)) {
+    throw new EvalError("minDelta must be a non-negative finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { minDelta }
+    });
+  }
+  return {
+    type: "noImprovement",
+    consecutiveRounds,
+    ...minDelta !== void 0 && { minDelta }
+  };
+}
+function maxCost(maxUSD) {
+  if (!Number.isFinite(maxUSD) || maxUSD <= 0) {
+    throw new EvalError("maxUSD must be a positive finite number", {
+      code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+      context: { maxUSD }
+    });
+  }
+  return { type: "maxCost", maxUSD };
+}
+function customCondition(check, description) {
+  return {
+    type: "custom",
+    check,
+    ...description !== void 0 && { description }
+  };
+}
+function and2(...conditions) {
+  if (conditions.length === 0) {
+    return {
+      type: "custom",
+      check: () => false,
+      description: formatCompositeDescription("and", [])
+    };
+  }
+  return {
+    type: "custom",
+    check: createAndCheck(conditions, checkCycleCondition),
+    description: formatCompositeDescription("and", conditions)
+  };
+}
+function or2(...conditions) {
+  if (conditions.length === 0) {
+    return {
+      type: "custom",
+      check: () => false,
+      description: formatCompositeDescription("or", [])
+    };
+  }
+  return {
+    type: "custom",
+    check: createOrCheck(conditions, checkCycleCondition),
+    description: formatCompositeDescription("or", conditions)
+  };
+}
+function not2(condition) {
+  return {
+    type: "custom",
+    check: createNotCheck(condition, checkCycleCondition),
+    description: `not(${condition.type})`
+  };
+}
+function checkTargetScore(condition, ctx) {
+  if (ctx.latestScore >= condition.threshold) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Target score ${condition.threshold} reached (current: ${ctx.latestScore})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Score ${ctx.latestScore} below target ${condition.threshold}`
+  };
+}
+function checkMaxRounds(condition, ctx) {
+  if (ctx.currentRound >= condition.count) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Maximum rounds reached (${condition.count})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Round ${ctx.currentRound} of ${condition.count}`
+  };
+}
+function checkNoImprovement(condition, ctx) {
+  const { consecutiveRounds, minDelta = 0 } = condition;
+  const { history } = ctx;
+  let noImprovementCount = 0;
+  for (let i = history.length - 1; i >= 0; i--) {
+    const round = history[i];
+    if (round.scoreDelta === null) break;
+    if (round.scoreDelta <= minDelta) {
+      noImprovementCount++;
+    } else {
+      break;
+    }
+  }
+  if (noImprovementCount >= consecutiveRounds) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `No improvement for ${noImprovementCount} consecutive round${noImprovementCount === 1 ? "" : "s"}`
+    };
+  }
+  const roundWord = noImprovementCount === 1 ? "round" : "rounds";
+  return {
+    terminated: false,
+    reason: `${noImprovementCount} ${roundWord} without improvement (need ${consecutiveRounds})`
+  };
+}
+function checkMaxCost(condition, ctx) {
+  if (ctx.totalCost >= condition.maxUSD) {
+    return {
+      terminated: true,
+      matchedCondition: condition,
+      reason: `Cost limit exceeded ($${ctx.totalCost.toFixed(2)} >= $${condition.maxUSD.toFixed(2)})`
+    };
+  }
+  return {
+    terminated: false,
+    reason: `Cost $${ctx.totalCost.toFixed(2)} under limit $${condition.maxUSD.toFixed(2)}`
+  };
+}
+async function checkCustomCondition(condition, ctx) {
+  const description = condition.description ?? "Custom condition";
+  try {
+    const shouldTerminate = await condition.check(ctx);
+    if (shouldTerminate) {
+      return {
+        terminated: true,
+        matchedCondition: condition,
+        reason: `${description} met`
+      };
+    }
+    return {
+      terminated: false,
+      reason: `${description} not met`
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      terminated: false,
+      reason: `${description} check failed: ${message}`
+    };
+  }
+}
+async function checkCycleCondition(condition, context) {
+  if (isTargetScoreCondition(condition)) {
+    return checkTargetScore(condition, context);
+  }
+  if (isMaxRoundsCondition(condition)) {
+    return checkMaxRounds(condition, context);
+  }
+  if (isNoImprovementCondition(condition)) {
+    return checkNoImprovement(condition, context);
+  }
+  if (isMaxCostCondition(condition)) {
+    return checkMaxCost(condition, context);
+  }
+  if (isCustomCycleCondition(condition)) {
+    return checkCustomCondition(condition, context);
+  }
+  const _exhaustive = condition;
+  throw new EvalError(`Unknown condition type: ${JSON.stringify(_exhaustive)}`, {
+    code: "UNKNOWN_ERROR" /* UNKNOWN_ERROR */,
+    context: { condition: _exhaustive }
+  });
+}
+async function checkCycleTermination(conditions, context) {
+  if (conditions.length === 0) {
+    return {
+      terminated: false,
+      reason: "No termination conditions specified"
+    };
+  }
+  for (const condition of conditions) {
+    const result = await checkCycleCondition(condition, context);
+    if (result.terminated) {
+      return result;
+    }
+  }
+  return {
+    terminated: false,
+    reason: "No termination conditions met"
+  };
+}
+
+// src/improvement-cycle/runner.ts
+var import_core7 = require("@agtlantis/core");
+
+// src/improvement-cycle/history.ts
+var import_node_crypto = __toESM(require("crypto"), 1);
+var import_node_fs7 = require("fs");
+var import_promises2 = require("fs/promises");
+var import_node_path4 = require("path");
+var import_core6 = require("@agtlantis/core");
+var defaultHistoryStorage = {
+  readFile: (path3) => (0, import_promises2.readFile)(path3, "utf-8"),
+  writeFile: (path3, content) => (0, import_promises2.writeFile)(path3, content, "utf-8"),
+  exists: import_node_fs7.existsSync,
+  mkdir: (path3, options) => (0, import_promises2.mkdir)(path3, options)
+};
+function hasUserTemplate(prompt) {
+  return typeof prompt.userTemplate === "string";
+}
+function serializePrompt(prompt) {
+  const p = prompt;
+  if (!hasUserTemplate(p)) {
+    throw new EvalError("Cannot serialize prompt: userTemplate field is required", {
+      code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+      context: { promptId: p.id }
+    });
+  }
+  const { id, version, system, userTemplate, renderUserPrompt, ...rest } = p;
+  const customFields = Object.keys(rest).length > 0 ? rest : void 0;
+  return {
+    id,
+    version,
+    system,
+    userTemplate,
+    ...customFields && { customFields }
+  };
+}
+function validateDeserializedPrompt(obj, promptId) {
+  const requiredStrings = ["id", "version", "system", "userTemplate"];
+  for (const field of requiredStrings) {
+    if (typeof obj[field] !== "string") {
+      throw new EvalError(`Invalid deserialized prompt: ${field} must be a string`, {
+        code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+        context: { promptId, field, actual: typeof obj[field] }
+      });
+    }
+  }
+  if (typeof obj.renderUserPrompt !== "function") {
+    throw new EvalError("Invalid deserialized prompt: renderUserPrompt must be a function", {
+      code: "PROMPT_INVALID_FORMAT" /* PROMPT_INVALID_FORMAT */,
+      context: { promptId, actual: typeof obj.renderUserPrompt }
+    });
+  }
+}
+function deserializePrompt(serialized) {
+  const { id, version, system, userTemplate, customFields } = serialized;
+  let renderUserPrompt;
+  try {
+    renderUserPrompt = (0, import_core6.compileTemplate)(userTemplate, id);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new EvalError(`Failed to compile userTemplate: ${message}`, {
+      code: "TEMPLATE_COMPILE_ERROR" /* TEMPLATE_COMPILE_ERROR */,
+      context: { promptId: id, userTemplate }
+    });
+  }
+  const result = {
+    ...customFields,
+    id,
+    version,
+    system,
+    userTemplate,
+    renderUserPrompt
+  };
+  validateDeserializedPrompt(result, id);
+  return result;
+}
+function serializeRoundResult(result) {
+  const { summary } = result.report;
+  return {
+    round: result.round,
+    completedAt: result.completedAt.toISOString(),
+    avgScore: summary.avgScore,
+    passed: summary.passed,
+    failed: summary.failed,
+    totalTests: summary.totalTests,
+    suggestionsGenerated: result.suggestionsGenerated,
+    suggestionsApproved: result.suggestionsApproved,
+    promptSnapshot: result.promptSnapshot,
+    promptVersionAfter: result.promptVersionAfter,
+    cost: result.cost,
+    scoreDelta: result.scoreDelta
+  };
+}
+function validateHistorySchema(data) {
+  if (typeof data !== "object" || data === null) {
+    throw new EvalError("Invalid history: not an object", {
+      code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */
+    });
+  }
+  const h = data;
+  if (h.schemaVersion !== "1.1.0") {
+    throw new EvalError(`Unsupported schema version: ${String(h.schemaVersion)}`, {
+      code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */,
+      context: { schemaVersion: h.schemaVersion }
+    });
+  }
+  const requiredFields = [
+    "sessionId",
+    "startedAt",
+    "initialPrompt",
+    "currentPrompt",
+    "rounds",
+    "totalCost"
+  ];
+  for (const field of requiredFields) {
+    if (!(field in h)) {
+      throw new EvalError(`Invalid history: missing field "${field}"`, {
+        code: "SCHEMA_VALIDATION_ERROR" /* SCHEMA_VALIDATION_ERROR */,
+        context: { missingField: field }
+      });
+    }
+  }
+}
+var ImprovementSessionImpl = class {
+  _history;
+  _isUpdating = false;
+  _savePromise = Promise.resolve();
+  config;
+  constructor(history, config = {}) {
+    this._history = history;
+    this.config = {
+      autoSave: config.autoSave ?? false,
+      ...config
+    };
+  }
+  get sessionId() {
+    return this._history.sessionId;
+  }
+  get history() {
+    return this._history;
+  }
+  get canSave() {
+    return this.config.path !== void 0;
+  }
+  addRound(roundResult, updatedPrompt) {
+    if (this._isUpdating) {
+      throw new EvalError("Session is being updated", {
+        code: "CONCURRENT_MODIFICATION" /* CONCURRENT_MODIFICATION */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    if (this._history.completedAt) {
+      throw new EvalError("Cannot add round to completed session", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    this._isUpdating = true;
+    try {
+      const serializedRound = serializeRoundResult(roundResult);
+      this._history = {
+        ...this._history,
+        currentPrompt: updatedPrompt,
+        rounds: [...this._history.rounds, serializedRound],
+        totalCost: this._history.totalCost + roundResult.cost.total
+      };
+      if (this.config.autoSave && this.canSave) {
+        this.save().catch((err) => this.handleAutoSaveError(err));
+      }
+    } finally {
+      this._isUpdating = false;
+    }
+  }
+  complete(terminationReason) {
+    this._history = {
+      ...this._history,
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      terminationReason
+    };
+    if (this.config.autoSave && this.canSave) {
+      this.save().catch((err) => this.handleAutoSaveError(err));
+    }
+  }
+  handleAutoSaveError(error) {
+    const err = error instanceof Error ? error : new Error(String(error));
+    if (this.config.onAutoSaveError) {
+      this.config.onAutoSaveError(err);
+    } else {
+      console.error("Auto-save failed:", err);
+    }
+  }
+  async save() {
+    if (!this.config.path) {
+      throw new EvalError("Cannot save: no path configured", {
+        code: "INVALID_CONFIG" /* INVALID_CONFIG */,
+        context: { sessionId: this.sessionId }
+      });
+    }
+    this._savePromise = this._savePromise.then(async () => {
+      await saveHistory(this._history, this.config.path, this.config.storage);
+    });
+    return this._savePromise;
+  }
+  async flush() {
+    return this._savePromise;
+  }
+};
+function createSession(initialPrompt, config) {
+  const serializedPrompt = serializePrompt(initialPrompt);
+  const history = {
+    schemaVersion: "1.1.0",
+    sessionId: import_node_crypto.default.randomUUID(),
+    startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    initialPrompt: serializedPrompt,
+    currentPrompt: serializedPrompt,
+    rounds: [],
+    totalCost: 0
+  };
+  return new ImprovementSessionImpl(history, config);
+}
+async function resumeSession(path3, config) {
+  const history = await loadHistory(path3, config?.storage);
+  const reopenedHistory = {
+    ...history,
+    completedAt: void 0,
+    terminationReason: void 0
+  };
+  return new ImprovementSessionImpl(reopenedHistory, { ...config, path: path3 });
+}
+async function saveHistory(history, path3, storage = defaultHistoryStorage) {
+  try {
+    const dir = (0, import_node_path4.dirname)(path3);
+    if (dir && dir !== "." && dir !== "/" && !storage.exists(dir)) {
+      await storage.mkdir(dir, { recursive: true });
+    }
+    await storage.writeFile(path3, JSON.stringify(history, null, 2));
+  } catch (error) {
+    if (error instanceof EvalError) throw error;
+    throw EvalError.from(error, "FILE_WRITE_ERROR" /* FILE_WRITE_ERROR */, { path: path3 });
+  }
+}
+async function loadHistory(path3, storage = defaultHistoryStorage) {
+  try {
+    if (!storage.exists(path3)) {
+      throw new EvalError(`History file not found: ${path3}`, {
+        code: "FILE_READ_ERROR" /* FILE_READ_ERROR */,
+        context: { path: path3 }
+      });
+    }
+    const content = await storage.readFile(path3);
+    const history = JSON.parse(content);
+    validateHistorySchema(history);
+    return history;
+  } catch (error) {
+    if (error instanceof EvalError) throw error;
+    throw EvalError.from(error, "FILE_READ_ERROR" /* FILE_READ_ERROR */, { path: path3 });
+  }
+}
+
+// src/improvement-cycle/runner.ts
+function initializeCycleState(initialPrompt, existingSession) {
+  const resumeFromRound = existingSession ? existingSession.history.rounds.length : 0;
+  return {
+    currentPrompt: initialPrompt,
+    currentRound: resumeFromRound,
+    previousScores: existingSession ? existingSession.history.rounds.map((r) => r.avgScore) : [],
+    totalCost: existingSession ? existingSession.history.totalCost : 0,
+    completedRounds: []
+  };
+}
+function calculateScoreDelta(currentScore, previousScores) {
+  if (previousScores.length === 0) {
+    return null;
+  }
+  const previousScore = previousScores[previousScores.length - 1];
+  return currentScore - previousScore;
+}
+function buildCycleContext(state, currentScore) {
+  return {
+    currentRound: state.currentRound,
+    latestScore: currentScore,
+    previousScores: [...state.previousScores],
+    totalCost: state.totalCost,
+    history: state.completedRounds
+  };
+}
+function createRoundResult(state, report, improveResult, cost, scoreDelta, promptSnapshot) {
+  return {
+    round: state.currentRound,
+    report,
+    completedAt: /* @__PURE__ */ new Date(),
+    suggestionsGenerated: improveResult.suggestions,
+    suggestionsApproved: [],
+    // Will be updated after decision
+    promptSnapshot,
+    promptVersionAfter: state.currentPrompt.version,
+    cost,
+    scoreDelta
+  };
+}
+async function handleStopDecision(state, session, roundResult, promptSnapshot, terminatedByCondition, conditionReason) {
+  const reason = terminatedByCondition ? conditionReason : "User requested stop";
+  session.addRound(roundResult, promptSnapshot);
+  session.complete(reason);
+  await session.flush();
+  state.completedRounds.push(roundResult);
+  return {
+    rounds: state.completedRounds,
+    finalPrompt: deserializePrompt(session.history.currentPrompt),
+    terminationReason: reason,
+    totalCost: state.totalCost,
+    history: session.history
+  };
+}
+function handleRollbackDecision(state, rollbackToRound) {
+  const targetRoundIndex = rollbackToRound - 1;
+  if (targetRoundIndex < 0 || targetRoundIndex >= state.completedRounds.length) {
+    throw new Error(`Cannot rollback to round ${rollbackToRound}: round not found`);
+  }
+  const targetRound = state.completedRounds[targetRoundIndex];
+  state.currentPrompt = deserializePrompt(targetRound.promptSnapshot);
+  state.previousScores = state.previousScores.slice(0, rollbackToRound - 1);
+}
+function handleContinueDecision(state, session, roundResult, approvedSuggestions, versionBump) {
+  const updatedRoundResult = {
+    ...roundResult,
+    suggestionsApproved: approvedSuggestions
+  };
+  if (approvedSuggestions.length > 0) {
+    const applyResult = applyPromptSuggestions(state.currentPrompt, approvedSuggestions, {
+      bumpVersion: versionBump
+    });
+    state.currentPrompt = applyResult.prompt;
+    updatedRoundResult.promptVersionAfter = state.currentPrompt.version;
+  }
+  const updatedPromptSnapshot = serializePrompt(state.currentPrompt);
+  session.addRound(updatedRoundResult, updatedPromptSnapshot);
+  state.completedRounds.push(updatedRoundResult);
+  return updatedRoundResult;
+}
+async function executeRound(config, state, pricingConfig) {
+  const { createAgent, judge, improver, testCases: testCases2, options = {} } = config;
+  const agent = createAgent(state.currentPrompt);
+  const suite = createEvalSuite({
+    agent,
+    judge,
+    agentDescription: options.agentDescription
+  });
+  const report = await suite.run(testCases2, options.runOptions);
+  const improveResult = improver ? await improver.improve(state.currentPrompt, report.results) : { suggestions: [] };
+  const cost = calculateRoundCost(report, improveResult, pricingConfig);
+  return { report, improveResult, cost };
+}
+function detectProviderForImprover(model) {
+  if (!model) return "anthropic";
+  if (model.startsWith("claude-")) return "anthropic";
+  if (model.startsWith("gpt-") || model.startsWith("o1") || model.startsWith("o3")) return "openai";
+  if (model.startsWith("gemini-")) return "google";
+  return "anthropic";
+}
+function toLanguageModelUsage2(usage) {
+  return {
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    totalTokens: usage.totalTokens
+  };
+}
+function calculateImproverCost(improveResult, pricingConfig) {
+  const usage = improveResult.metadata?.tokenUsage;
+  if (!usage) return 0;
+  const model = improveResult.metadata?.model ?? "unknown";
+  const provider = detectProviderForImprover(model);
+  const providerPricing = pricingConfig?.providerPricing?.[provider];
+  const result = (0, import_core7.calculateCostFromUsage)(
+    toLanguageModelUsage2(usage),
+    model,
+    provider,
+    providerPricing
+  );
+  return result.total;
+}
+function calculateRoundCost(report, improveResult, pricingConfig) {
+  const reportCosts = pricingConfig ? calculateReportCosts(report, pricingConfig) : { total: 0, byComponent: { agent: 0, judge: 0 } };
+  const improverCost = calculateImproverCost(improveResult, pricingConfig);
+  return {
+    agent: reportCosts.byComponent.agent ?? 0,
+    judge: reportCosts.byComponent.judge ?? 0,
+    improver: improverCost,
+    total: reportCosts.total + improverCost
+  };
+}
+async function* runImprovementCycle(config) {
+  const { initialPrompt, terminateWhen = [], options = {} } = config;
+  const { pricingConfig, versionBump = "patch", history: historyConfig, session: existingSession } = options;
+  const session = existingSession ?? createSession(
+    initialPrompt,
+    historyConfig ? { path: historyConfig.path, autoSave: historyConfig.autoSave } : void 0
+  );
+  const state = initializeCycleState(initialPrompt, existingSession);
+  try {
+    while (true) {
+      state.currentRound++;
+      const { report, improveResult, cost } = await executeRound(config, state, pricingConfig);
+      state.totalCost += cost.total;
+      const currentScore = report.summary.avgScore;
+      const scoreDelta = calculateScoreDelta(currentScore, state.previousScores);
+      const promptSnapshot = serializePrompt(state.currentPrompt);
+      const roundResult = createRoundResult(state, report, improveResult, cost, scoreDelta, promptSnapshot);
+      const context = buildCycleContext(state, currentScore);
+      state.previousScores.push(currentScore);
+      const terminationCheck = await checkCycleTermination(terminateWhen, context);
+      const pendingSuggestions = improveResult.suggestions.map((s) => ({
+        ...s,
+        approved: false
+      }));
+      const roundYield = {
+        roundResult,
+        pendingSuggestions,
+        terminationCheck,
+        context
+      };
+      const decision = yield roundYield;
+      if (!decision || decision.action === "stop") {
+        return await handleStopDecision(
+          state,
+          session,
+          roundResult,
+          promptSnapshot,
+          terminationCheck.terminated,
+          terminationCheck.reason
+        );
+      }
+      if (decision.action === "rollback" && decision.rollbackToRound !== void 0) {
+        handleRollbackDecision(state, decision.rollbackToRound);
+        continue;
+      }
+      handleContinueDecision(
+        state,
+        session,
+        roundResult,
+        decision.approvedSuggestions ?? [],
+        versionBump
+      );
+    }
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    session.complete(`Error: ${errorMessage}`);
+    throw error;
+  }
+}
+async function runImprovementCycleAuto(config) {
+  const cycle = runImprovementCycle(config);
+  let iteratorResult = await cycle.next();
+  while (!iteratorResult.done) {
+    const roundYield = iteratorResult.value;
+    let decision;
+    if (roundYield.terminationCheck.terminated) {
+      decision = { action: "stop" };
+    } else {
+      const approvedSuggestions = roundYield.pendingSuggestions.map((s) => ({
+        ...s,
+        approved: true
+      }));
+      decision = { action: "continue", approvedSuggestions };
+    }
+    iteratorResult = await cycle.next(decision);
+  }
+  return iteratorResult.value;
+}
+
+// src/core/test-case-collection.ts
+var TestCaseCollection = class _TestCaseCollection {
+  cases;
+  constructor(cases) {
+    this.cases = Object.freeze([...cases]);
+  }
+  // ============================================================================
+  // Static Factories
+  // ============================================================================
+  /**
+   * Create a collection from an array of test cases.
+   */
+  static from(cases) {
+    return new _TestCaseCollection(cases);
+  }
+  /**
+   * Create an empty collection.
+   */
+  static empty() {
+    return new _TestCaseCollection([]);
+  }
+  // ============================================================================
+  // Properties
+  // ============================================================================
+  /**
+   * Number of test cases in the collection.
+   */
+  get length() {
+    return this.cases.length;
+  }
+  /**
+   * Whether the collection is empty.
+   */
+  get isEmpty() {
+    return this.cases.length === 0;
+  }
+  // ============================================================================
+  // Selection Methods (return new TestCaseCollection - chainable)
+  // ============================================================================
+  /**
+   * Returns all test cases.
+   * Returns `this` since the collection is immutable (frozen array).
+   * Useful as explicit starting point in chains.
+   */
+  all() {
+    return this;
+  }
+  /**
+   * Returns the first N test cases (default: 1).
+   * Useful for cost-controlled testing during development.
+   */
+  minimal(count = 1) {
+    return this.first(count);
+  }
+  /**
+   * Returns the first N test cases.
+   */
+  first(count) {
+    if (count <= 0) {
+      return _TestCaseCollection.empty();
+    }
+    return new _TestCaseCollection([...this.cases.slice(0, count)]);
+  }
+  /**
+   * Returns the last N test cases (default: 1).
+   * Preserves original order (earlier cases first).
+   */
+  last(count = 1) {
+    if (count <= 0) {
+      return _TestCaseCollection.empty();
+    }
+    const startIndex = Math.max(0, this.cases.length - count);
+    return new _TestCaseCollection([...this.cases.slice(startIndex)]);
+  }
+  /**
+   * Returns N random test cases.
+   *
+   * @param count - Number of cases to select
+   * @param options - Optional seed for reproducibility
+   *
+   * @example
+   * ```typescript
+   * // Different each time
+   * collection.random(5)
+   *
+   * // Same result with same seed
+   * collection.random(5, { seed: 42 })
+   * ```
+   */
+  random(count, options) {
+    if (count <= 0 || this.cases.length === 0) {
+      return _TestCaseCollection.empty();
+    }
+    const actualCount = Math.min(count, this.cases.length);
+    const indices = [...Array(this.cases.length).keys()];
+    const rng = options?.seed !== void 0 ? createSeededRng(options.seed) : Math.random;
+    for (let i = indices.length - 1; i > 0; i--) {
+      const j = Math.floor(rng() * (i + 1));
+      [indices[i], indices[j]] = [indices[j], indices[i]];
+    }
+    const selected = indices.slice(0, actualCount).map((i) => this.cases[i]);
+    return new _TestCaseCollection([...selected]);
+  }
+  /**
+   * Filter test cases by predicate.
+   */
+  filter(predicate) {
+    return new _TestCaseCollection([...this.cases.filter(predicate)]);
+  }
+  /**
+   * Find test case by ID.
+   * Returns collection with single case or empty collection.
+   */
+  byId(id) {
+    const found = this.cases.find((tc) => tc.id === id);
+    return found ? new _TestCaseCollection([found]) : _TestCaseCollection.empty();
+  }
+  /**
+   * Find test cases by multiple IDs.
+   * Preserves order of provided IDs (first occurrence).
+   * Skips non-existent IDs. Duplicate IDs in input are deduplicated.
+   *
+   * @example
+   * ```typescript
+   * collection.byIds(['a', 'b', 'a'])  // returns [case-a, case-b] (deduplicated)
+   * collection.byIds(['b', 'a'])       // returns [case-b, case-a] (order preserved)
+   * ```
+   */
+  byIds(ids) {
+    const uniqueIds = [...new Set(ids)];
+    const idSet = new Set(uniqueIds);
+    const idToCase = /* @__PURE__ */ new Map();
+    for (const tc of this.cases) {
+      if (tc.id && idSet.has(tc.id) && !idToCase.has(tc.id)) {
+        idToCase.set(tc.id, tc);
+      }
+    }
+    const result = uniqueIds.map((id) => idToCase.get(id)).filter((tc) => tc !== void 0);
+    return new _TestCaseCollection(result);
+  }
+  // ============================================================================
+  // Access Methods
+  // ============================================================================
+  /**
+   * Get test case by ID.
+   * Returns undefined if not found.
+   */
+  get(id) {
+    return this.cases.find((tc) => tc.id === id);
+  }
+  /**
+   * Get test case by index.
+   * Supports negative indices (e.g., -1 for last item).
+   * Returns undefined if index is out of bounds.
+   */
+  at(index) {
+    const normalizedIndex = index < 0 ? this.cases.length + index : index;
+    if (normalizedIndex < 0 || normalizedIndex >= this.cases.length) {
+      return void 0;
+    }
+    return this.cases[normalizedIndex];
+  }
+  // ============================================================================
+  // Conversion Methods
+  // ============================================================================
+  /**
+   * Convert to array.
+   * Returns a mutable copy of the internal array.
+   */
+  toArray() {
+    return [...this.cases];
+  }
+  // ============================================================================
+  // Iterator Support
+  // ============================================================================
+  /**
+   * Iterator support for for...of loops and spread operator.
+   */
+  [Symbol.iterator]() {
+    return this.cases[Symbol.iterator]();
+  }
+};
+var autoIdCounter = 0;
+function testCase(input, id) {
+  return {
+    id: id ?? `test-${++autoIdCounter}`,
+    input
+  };
+}
+function testCases(inputs, prefix = "case") {
+  return inputs.map((input, index) => ({
+    id: `${prefix}-${index}`,
+    input
+  }));
+}
+function createSeededRng(seed) {
+  let state = seed;
+  return () => {
+    state = state + 1831565813 | 0;
+    let t = Math.imul(state ^ state >>> 15, state | 1);
+    t ^= t + Math.imul(t ^ t >>> 7, t | 61);
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ANTHROPIC_PRICING,
+  CompositeReporter,
+  ConsoleReporter,
+  DEFAULT_PRICING_CONFIG,
+  EvalError,
+  EvalErrorCode,
+  GOOGLE_PRICING,
+  JsonReporter,
+  MarkdownReporter,
+  MockProvider,
+  OPENAI_PRICING,
+  TestCaseCollection,
+  accuracy,
+  addCostsToResults,
+  afterTurns,
+  aggregateIterationResults,
+  aiUser,
+  and,
+  applyPromptSuggestions,
+  bumpVersion,
+  calculateAvgPassRate,
+  calculateAvgStdDev,
+  calculateCostFromUsage,
+  calculateIterationStats,
+  calculateMultiTurnIterationStats,
+  calculateReportCosts,
+  calculateResultCost,
+  checkCondition,
+  checkCycleCondition,
+  checkCycleTermination,
+  checkTermination,
+  compareReports,
+  compileTemplate,
+  consistency,
+  createCompositeReporter,
+  createConsoleReporter,
+  createDefaultReporter,
+  createEvalSuite,
+  createFilePromptRepository,
+  createImprover,
+  createJsonReporter,
+  createJudge,
+  createMarkdownReporter,
+  createMockAgent,
+  createMockImprover,
+  createMockJudge,
+  createReportRunner,
+  createSession,
+  customCondition,
+  cycleAnd,
+  cycleNot,
+  cycleOr,
+  cycleToMarkdown,
+  defaultHistoryStorage,
+  defineConfig,
+  deserializePrompt,
+  discoverEvalFiles,
+  executeMultiTurnTestCase,
+  executeTestCase,
+  fieldEquals,
+  fieldIsSet,
+  getFieldValue,
+  getFileSourceDisplayInfo,
+  getFileSourcesDisplayInfo,
+  inferMediaType,
+  isCustomCondition,
+  isCustomCycleCondition,
+  isCycleTerminated,
+  isFieldSetCondition,
+  isFieldValueCondition,
+  isFileSource,
+  isFileSourceBase64,
+  isFileSourceData,
+  isFileSourcePath,
+  isFileSourceUrl,
+  isIteratedResult,
+  isMaxCostCondition,
+  isMaxRoundsCondition,
+  isMaxTurnsCondition,
+  isMultiTurnResult,
+  isMultiTurnTestCase,
+  isNoImprovementCondition,
+  isSingleTurnResult,
+  isTargetScoreCondition,
+  isTerminated,
+  loadHistory,
+  logCycle,
+  maxCost,
+  maxRounds,
+  mock,
+  naturalLanguage,
+  noImprovement,
+  not,
+  or,
+  relevance,
+  reportToMarkdown,
+  resolveFileSource,
+  resolveFileSourcesInInput,
+  resumeSession,
+  runImprovementCycle,
+  runImprovementCycleAuto,
+  runWithConcurrency,
+  saveCycleJson,
+  saveCycleMarkdown,
+  saveHistory,
+  saveReportMarkdown,
+  scanForFileSources,
+  schema,
+  selectRepresentativeResult,
+  serializePrompt,
+  suggestionDiff,
+  suggestionPreview,
+  suggestionSummary,
+  targetScore,
+  testCase,
+  testCases,
+  toEvalAgent
+});
+//# sourceMappingURL=index.cjs.map