@google/adk 2.4.0 → 2.5.0

package/dist/esm/cli/cli_eval.js

@@ -37,6 +37,9 @@ import {
   isLlmAgent
 } from "../agents/llm_agent.js";
 import { InMemoryArtifactService } from "../artifacts/in_memory_artifact_service.js";
+import { InputValidationError } from "../errors/input_validation_error.js";
+import { NotFoundError } from "../errors/not_found_error.js";
+import { publicSession } from "../events/internal_metadata.js";
 import { Runner } from "../runner/runner.js";
 import { InMemorySessionService } from "../sessions/in_memory_session_service.js";
 import { logger } from "../utils/logger.js";
@@ -44,6 +47,10 @@ import {
   convertSessionToEvalFormat,
   createEmptyState
 } from "./utils/evals.js";
+import {
+  isStrictlyInside,
+  validatePathSegment
+} from "./utils/path_validation.js";
 const MISSING_EVAL_DEPENDENCIES_MESSAGE = "Eval module is not installed, please install via `pip install google-adk[eval]`.";
 const TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score";
 const RESPONSE_MATCH_SCORE_KEY = "response_match_score";
@@ -124,8 +131,9 @@ function parseAndGetEvalsToRun(evalSetFilePaths) {
   for (const inputEvalSet of evalSetFilePaths) {
     let evalSetFile;
     let evals = [];
-    if (inputEvalSet.includes(":")) {
-      const colonIndex = inputEvalSet.indexOf(":");
+    const hasDrivePrefix = /^[A-Za-z]:[\\/]/.test(inputEvalSet);
+    const colonIndex = inputEvalSet.indexOf(":", hasDrivePrefix ? 3 : 0);
+    if (colonIndex !== -1) {
       evalSetFile = inputEvalSet.slice(0, colonIndex);
       evals = inputEvalSet.slice(colonIndex + 1).split(",").filter((s) => s.length > 0);
     } else {
@@ -272,6 +280,23 @@ function evaluateSingleMetric(metric, actualInvocations, expectedInvocations) {
   };
 }
 __name(evaluateSingleMetric, "evaluateSingleMetric");
+const MOCK_TOOL_OUTPUT_RESPONSE_KEY = "result";
+const declaredToolResponses = /* @__PURE__ */ new WeakMap();
+function getMockToolResponse(toolUse) {
+  const declared = declaredToolResponses.get(toolUse);
+  if (declared !== void 0) {
+    return declared;
+  }
+  if (toolUse.mock_tool_output !== void 0) {
+    return { [MOCK_TOOL_OUTPUT_RESPONSE_KEY]: toolUse.mock_tool_output };
+  }
+  return void 0;
+}
+__name(getMockToolResponse, "getMockToolResponse");
+function isEvalEntry(evalCase) {
+  return "data" in evalCase && Array.isArray(evalCase.data);
+}
+__name(isEvalEntry, "isEvalEntry");
 function installMockToolOutputCallbacks(agent, interceptor, savedCallbacks) {
   if (isLlmAgent(agent) && !savedCallbacks.has(agent)) {
     savedCallbacks.set(agent, agent.beforeToolCallback);
@@ -284,7 +309,7 @@ function installMockToolOutputCallbacks(agent, interceptor, savedCallbacks) {
 }
 __name(installMockToolOutputCallbacks, "installMockToolOutputCallbacks");
 async function generateActualInvocations(rootAgent, evalCase, resetFunc, sessionService, artifactService) {
-  var _a, _b, _c, _d, _e, _f, _g;
+  var _a, _b, _c, _d, _e, _f, _g, _h;
   if (resetFunc) {
     await resetFunc();
   }
@@ -306,8 +331,13 @@ async function generateActualInvocations(rootAgent, evalCase, resetFunc, session
   const pendingMockToolUses = [];
   for (const turn of evalCase.data) {
     for (const expectedTool of (_g = turn.expected_tool_use) != null ? _g : []) {
-      if (expectedTool.mock_tool_output !== void 0) {
-        pendingMockToolUses.push({ ...expectedTool });
+      const response = getMockToolResponse(expectedTool);
+      if (response !== void 0) {
+        pendingMockToolUses.push({
+          toolName: expectedTool.tool_name,
+          toolInput: (_h = expectedTool.tool_input) != null ? _h : {},
+          response
+        });
       }
     }
   }
@@ -315,14 +345,11 @@ async function generateActualInvocations(rootAgent, evalCase, resetFunc, session
   if (pendingMockToolUses.length > 0) {
     const interceptor = /* @__PURE__ */ __name(({ tool, args }) => {
       const idx = pendingMockToolUses.findIndex(
-        (expected) => {
-          var _a2;
-          return expected.mock_tool_output !== void 0 && tool.name === expected.tool_name && deepEqual(args, (_a2 = expected.tool_input) != null ? _a2 : {});
-        }
+        (expected) => tool.name === expected.toolName && deepEqual(args, expected.toolInput)
       );
       if (idx !== -1) {
         const [matched] = pendingMockToolUses.splice(idx, 1);
-        return { result: matched.mock_tool_output };
+        return matched.response;
       }
       return void 0;
     }, "interceptor");
@@ -366,27 +393,97 @@ async function generateActualInvocations(rootAgent, evalCase, resetFunc, session
       reference: ""
     });
   }
-  return { actualInvocations, sessionId, userId };
+  return { actualInvocations, sessionId, userId, recordedSession };
 }
 __name(generateActualInvocations, "generateActualInvocations");
+function evalCaseToEvalEntry(evalCase) {
+  if (isEvalEntry(evalCase)) {
+    return evalCase;
+  }
+  const structured = evalCase;
+  const data = structured.conversation.map((invocation) => {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
+    const query = (_c = (_b = (_a = invocation.userContent) == null ? void 0 : _a.parts) == null ? void 0 : _b.map((p) => {
+      var _a2;
+      return (_a2 = p.text) != null ? _a2 : "";
+    }).join("")) != null ? _c : "";
+    const reference = (_f = (_e = (_d = invocation.finalResponse) == null ? void 0 : _d.parts) == null ? void 0 : _e.map((p) => {
+      var _a2;
+      return (_a2 = p.text) != null ? _a2 : "";
+    }).join("")) != null ? _f : "";
+    const toolResponsesById = /* @__PURE__ */ new Map();
+    for (const toolResponse of (_h = (_g = invocation.intermediateData) == null ? void 0 : _g.toolResponses) != null ? _h : []) {
+      if (toolResponse.id) {
+        toolResponsesById.set(toolResponse.id, toolResponse);
+      }
+    }
+    const expected_tool_use = ((_j = (_i = invocation.intermediateData) == null ? void 0 : _i.toolUses) != null ? _j : []).map((fc) => {
+      var _a2, _b2;
+      const toolUse = {
+        tool_name: (_a2 = fc.name) != null ? _a2 : "",
+        tool_input: (_b2 = fc.args) != null ? _b2 : {}
+      };
+      const toolResponse = fc.id ? toolResponsesById.get(fc.id) : void 0;
+      if ((toolResponse == null ? void 0 : toolResponse.response) !== void 0) {
+        declaredToolResponses.set(toolUse, toolResponse.response);
+      }
+      return toolUse;
+    });
+    const expected_intermediate_agent_responses = ((_l = (_k = invocation.intermediateData) == null ? void 0 : _k.intermediateResponses) != null ? _l : []).map(
+      ([author, parts]) => [author, parts]
+    );
+    return {
+      query,
+      expected_tool_use,
+      expected_intermediate_agent_responses,
+      reference
+    };
+  });
+  const sessionInput = structured.sessionInput;
+  return {
+    name: structured.evalId,
+    data,
+    ...sessionInput ? {
+      initial_session: {
+        ...sessionInput.appName ? { app_name: sessionInput.appName } : {},
+        ...sessionInput.userId ? { user_id: sessionInput.userId } : {},
+        state: sessionInput.state
+      }
+    } : {}
+  };
+}
+__name(evalCaseToEvalEntry, "evalCaseToEvalEntry");
 function resolveEvalCasesForSet(evalSetId, entries) {
   if (entries.length > 0 && typeof entries[0] === "string") {
     const filterNames = entries;
-    const fileData = JSON.parse(
+    const rawFileData = JSON.parse(
       fs.readFileSync(evalSetId, "utf-8")
     );
-    return fileData.filter((e) => filterNames.includes(e.name));
+    if (Array.isArray(rawFileData)) {
+      return rawFileData.filter(
+        (e) => filterNames.includes(e.name)
+      );
+    }
+    const loadedSet = loadEvalSetFromFile(evalSetId, evalSetId);
+    return loadedSet.evalCases.filter((e) => filterNames.includes(e.evalId)).map(evalCaseToEvalEntry);
   }
   if (entries.length === 0 && fs.existsSync(evalSetId) && fs.statSync(evalSetId).isFile()) {
-    return JSON.parse(fs.readFileSync(evalSetId, "utf-8"));
+    const rawFileData = JSON.parse(
+      fs.readFileSync(evalSetId, "utf-8")
+    );
+    if (Array.isArray(rawFileData)) {
+      return rawFileData;
+    }
+    const loadedSet = loadEvalSetFromFile(evalSetId, evalSetId);
+    return loadedSet.evalCases.map(evalCaseToEvalEntry);
   }
-  return entries;
+  return entries.map(evalCaseToEvalEntry);
 }
 __name(resolveEvalCasesForSet, "resolveEvalCasesForSet");
 function printCaseDetailedResults(result) {
   var _a;
   logger.info(
-    `Eval result for ${result.eval_set_file}:${result.eval_id}: ${EvalStatus[result.final_eval_status]}`
+    `Eval result for ${result.eval_set_id}:${result.eval_id}: ${EvalStatus[result.final_eval_status]}`
   );
   for (const metricResult of result.overall_eval_metric_results) {
     logger.info(
@@ -435,7 +532,7 @@ async function* runEvals(optionsOrMap, rootAgentArg, resetFuncArg, evalMetricsAr
     const evalCases = resolveEvalCasesForSet(evalSetId, rawEntries);
     for (const evalCase of evalCases) {
       const evalName = evalCase.name;
-      const { actualInvocations, sessionId, userId } = await generateActualInvocations(
+      const { actualInvocations, sessionId, userId, recordedSession } = await generateActualInvocations(
         rootAgent,
         evalCase,
         resetFunc,
@@ -478,12 +575,14 @@ async function* runEvals(optionsOrMap, rootAgentArg, resetFuncArg, evalMetricsAr
       }
       const result = {
         eval_set_file: evalSetId,
+        eval_set_id: evalSetId,
         eval_id: evalName,
         final_eval_status: finalEvalStatus,
         eval_metric_results: evalMetricResultsPairs,
         overall_eval_metric_results: overallEvalMetricResults,
         eval_metric_result_per_invocation: perInvocationMap,
         session_id: sessionId,
+        session_details: recordedSession && publicSession(recordedSession),
         user_id: userId
       };
       if (printDetailedResults) {
@@ -494,17 +593,781 @@ async function* runEvals(optionsOrMap, rootAgentArg, resetFuncArg, evalMetricsAr
   }
 }
 __name(runEvals, "runEvals");
+const ADK_EVAL_HISTORY_DIR = ".adk/eval_history";
+const EVAL_SET_RESULT_FILE_EXTENSION = ".evalset_result.json";
+function sanitizeEvalSetResultName(evalSetResultName) {
+  return evalSetResultName.replace(/\//g, "_");
+}
+__name(sanitizeEvalSetResultName, "sanitizeEvalSetResultName");
+function toRunEvalResult(caseResult, fallbackEvalSetId) {
+  const record = caseResult;
+  const evalSetFile = typeof record.evalSetFile === "string" ? record.evalSetFile : typeof record.eval_set_file === "string" ? record.eval_set_file : void 0;
+  const evalSetId = typeof record.evalSetId === "string" ? record.evalSetId : typeof record.eval_set_id === "string" ? record.eval_set_id : fallbackEvalSetId != null ? fallbackEvalSetId : "";
+  const evalId = typeof record.evalId === "string" ? record.evalId : typeof record.eval_id === "string" ? record.eval_id : "";
+  const finalEvalStatus = typeof record.finalEvalStatus === "number" ? record.finalEvalStatus : typeof record.final_eval_status === "number" ? record.final_eval_status : 3 /* NOT_EVALUATED */;
+  const evalMetricResults = Array.isArray(record.evalMetricResults) ? record.evalMetricResults : Array.isArray(record.eval_metric_results) ? record.eval_metric_results : void 0;
+  const overallEvalMetricResults = Array.isArray(
+    record.overallEvalMetricResults
+  ) ? record.overallEvalMetricResults : Array.isArray(record.overall_eval_metric_results) ? record.overall_eval_metric_results : [];
+  const evalMetricResultPerInvocation = Array.isArray(
+    record.evalMetricResultPerInvocation
+  ) ? record.evalMetricResultPerInvocation : Array.isArray(record.eval_metric_result_per_invocation) ? record.eval_metric_result_per_invocation : [];
+  const sessionId = typeof record.sessionId === "string" ? record.sessionId : typeof record.session_id === "string" ? record.session_id : "";
+  const rawSessionDetails = record.sessionDetails !== void 0 && record.sessionDetails !== null ? record.sessionDetails : record.session_details !== void 0 && record.session_details !== null ? record.session_details : void 0;
+  const sessionDetails = rawSessionDetails && publicSession(rawSessionDetails);
+  const userId = typeof record.userId === "string" ? record.userId : typeof record.user_id === "string" ? record.user_id : void 0;
+  return {
+    ...evalSetFile !== void 0 ? { evalSetFile } : {},
+    evalSetId,
+    evalId,
+    finalEvalStatus,
+    ...evalMetricResults !== void 0 ? { evalMetricResults } : {},
+    overallEvalMetricResults,
+    evalMetricResultPerInvocation,
+    sessionId,
+    ...sessionDetails !== void 0 ? { sessionDetails } : {},
+    ...userId !== void 0 ? { userId } : {}
+  };
+}
+__name(toRunEvalResult, "toRunEvalResult");
+function createEvalSetResult({
+  appName,
+  evalSetId,
+  evalCaseResults,
+  creationTimestamp = (performance.timeOrigin + performance.now()) / 1e3
+}) {
+  const evalSetResultId = `${appName}_${evalSetId}_${creationTimestamp}`;
+  const evalSetResultName = sanitizeEvalSetResultName(evalSetResultId);
+  return {
+    evalSetResultId,
+    evalSetResultName,
+    evalSetId,
+    evalCaseResults: evalCaseResults.map(
+      (caseResult) => toRunEvalResult(caseResult, evalSetId)
+    ),
+    creationTimestamp
+  };
+}
+__name(createEvalSetResult, "createEvalSetResult");
+function parseEvalSetResultJson(evalSetResultJson) {
+  var _a, _b;
+  const decoded = JSON.parse(evalSetResultJson);
+  const target = typeof decoded === "string" ? JSON.parse(decoded) : decoded;
+  if (typeof target !== "object" || target === null || Array.isArray(target)) {
+    throw new Error("Invalid EvalSetResult JSON: expected an object.");
+  }
+  const record = target;
+  const evalSetResultId = typeof record.evalSetResultId === "string" ? record.evalSetResultId : typeof record.eval_set_result_id === "string" ? record.eval_set_result_id : void 0;
+  const evalSetId = typeof record.evalSetId === "string" ? record.evalSetId : typeof record.eval_set_id === "string" ? record.eval_set_id : void 0;
+  if (!evalSetResultId || !evalSetId) {
+    throw new Error(
+      "Invalid EvalSetResult JSON: missing evalSetResultId or evalSetId."
+    );
+  }
+  const evalSetResultName = typeof record.evalSetResultName === "string" ? record.evalSetResultName : typeof record.eval_set_result_name === "string" ? record.eval_set_result_name : sanitizeEvalSetResultName(evalSetResultId);
+  const rawCases = (_b = (_a = record.evalCaseResults) != null ? _a : record.eval_case_results) != null ? _b : [];
+  if (!Array.isArray(rawCases)) {
+    throw new Error(
+      "Invalid EvalSetResult JSON: evalCaseResults must be an array."
+    );
+  }
+  const evalCaseResults = rawCases.map((rawCase) => {
+    if (typeof rawCase !== "object" || rawCase === null || Array.isArray(rawCase)) {
+      throw new Error(
+        "Invalid EvalSetResult JSON: each evalCaseResult must be an object."
+      );
+    }
+    return toRunEvalResult(rawCase, evalSetId);
+  });
+  const creationTimestamp = typeof record.creationTimestamp === "number" ? record.creationTimestamp : typeof record.creation_timestamp === "number" ? record.creation_timestamp : 0;
+  return {
+    evalSetResultId,
+    evalSetResultName,
+    evalSetId,
+    evalCaseResults,
+    creationTimestamp
+  };
+}
+__name(parseEvalSetResultJson, "parseEvalSetResultJson");
+const _LocalEvalSetResultsManager = class _LocalEvalSetResultsManager {
+  constructor(agentsDir) {
+    this.agentsDir = agentsDir;
+  }
+  /**
+   * Creates and saves a new {@link EvalSetResult} for `evalCaseResults`.
+   */
+  saveEvalSetResult({
+    appName,
+    evalSetId,
+    evalCaseResults
+  }) {
+    validatePathSegment(appName, "appName");
+    validatePathSegment(evalSetId, "evalSetId");
+    const evalSetResult = createEvalSetResult({
+      appName,
+      evalSetId,
+      evalCaseResults
+    });
+    const appEvalHistoryDir = this.getEvalHistoryDir(appName);
+    if (!fs.existsSync(appEvalHistoryDir)) {
+      fs.mkdirSync(appEvalHistoryDir, { recursive: true });
+    }
+    const evalSetResultFilePath = path.join(
+      appEvalHistoryDir,
+      `${evalSetResult.evalSetResultName}${EVAL_SET_RESULT_FILE_EXTENSION}`
+    );
+    logger.info(`Writing eval result to file: ${evalSetResultFilePath}`);
+    fs.writeFileSync(
+      evalSetResultFilePath,
+      JSON.stringify(evalSetResult, null, 2),
+      "utf-8"
+    );
+    return evalSetResult;
+  }
+  /**
+   * Loads an {@link EvalSetResult} by `appName` and `evalSetResultId`.
+   *
+   * @throws {NotFoundError} If the result file does not exist.
+   */
+  getEvalSetResult(appName, evalSetResultId) {
+    const appEvalHistoryDir = this.getEvalHistoryDir(appName);
+    validatePathSegment(evalSetResultId, "evalSetResultId");
+    const filePath = path.resolve(
+      appEvalHistoryDir,
+      `${evalSetResultId}${EVAL_SET_RESULT_FILE_EXTENSION}`
+    );
+    if (!isStrictlyInside(appEvalHistoryDir, filePath)) {
+      throw new InputValidationError(
+        `evalSetResultId '${evalSetResultId}' resolves outside the eval history directory.`
+      );
+    }
+    if (!fs.existsSync(filePath)) {
+      throw new NotFoundError(
+        `Eval set result \`${evalSetResultId}\` not found.`
+      );
+    }
+    const content = fs.readFileSync(filePath, "utf-8");
+    return parseEvalSetResultJson(content);
+  }
+  /**
+   * Lists all saved eval set result IDs for `appName`.
+   */
+  listEvalSetResults(appName) {
+    const appEvalHistoryDir = this.getEvalHistoryDir(appName);
+    if (!fs.existsSync(appEvalHistoryDir)) {
+      return [];
+    }
+    return fs.readdirSync(appEvalHistoryDir).filter((file) => file.endsWith(EVAL_SET_RESULT_FILE_EXTENSION)).map((file) => file.slice(0, -EVAL_SET_RESULT_FILE_EXTENSION.length));
+  }
+  getEvalHistoryDir(appName) {
+    validatePathSegment(appName, "appName");
+    const baseDir = path.resolve(this.agentsDir);
+    const appDir = path.resolve(baseDir, appName);
+    if (!isStrictlyInside(baseDir, appDir)) {
+      throw new InputValidationError(
+        `appName '${appName}' resolves outside agentsDir.`
+      );
+    }
+    return path.join(appDir, ADK_EVAL_HISTORY_DIR);
+  }
+};
+__name(_LocalEvalSetResultsManager, "LocalEvalSetResultsManager");
+let LocalEvalSetResultsManager = _LocalEvalSetResultsManager;
+const EVAL_SET_FILE_EXTENSION = ".evalset.json";
+const VALID_EVAL_ID_PATTERN = /^[a-zA-Z0-9_]+$/;
+function validateEvalId(idName, idValue) {
+  if (!VALID_EVAL_ID_PATTERN.test(idValue)) {
+    const formattedName = idName.length > 0 ? idName[0].toUpperCase() + idName.slice(1) : idName;
+    throw new InputValidationError(
+      `Invalid ${idName}. ${formattedName} should have the \`${VALID_EVAL_ID_PATTERN.source}\` format`
+    );
+  }
+}
+__name(validateEvalId, "validateEvalId");
+function readCreationTimestamp(record) {
+  if (typeof record.creationTimestamp === "number") {
+    return record.creationTimestamp;
+  }
+  if (typeof record.creation_timestamp === "number") {
+    return record.creation_timestamp;
+  }
+  return void 0;
+}
+__name(readCreationTimestamp, "readCreationTimestamp");
+function normalizeContent(raw, defaultRole) {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new InputValidationError("Invalid Content: expected an object.");
+  }
+  const record = raw;
+  if (record.parts !== void 0 && !Array.isArray(record.parts)) {
+    throw new InputValidationError(
+      "Invalid Content: parts must be an array when provided."
+    );
+  }
+  const parts = Array.isArray(record.parts) ? record.parts : [];
+  const role = typeof record.role === "string" ? record.role : defaultRole !== void 0 ? defaultRole : void 0;
+  return {
+    parts,
+    ...role !== void 0 ? { role } : {}
+  };
+}
+__name(normalizeContent, "normalizeContent");
+function normalizeIntermediateData(raw) {
+  var _a, _b, _c, _d, _e, _f;
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new InputValidationError(
+      "Invalid EvalIntermediateData: expected an object."
+    );
+  }
+  const record = raw;
+  const rawToolUses = (_b = (_a = record.toolUses) != null ? _a : record.tool_uses) != null ? _b : [];
+  if (!Array.isArray(rawToolUses)) {
+    throw new InputValidationError(
+      "Invalid EvalIntermediateData: toolUses must be an array."
+    );
+  }
+  const toolUses = rawToolUses.map((tu) => {
+    if (typeof tu !== "object" || tu === null || Array.isArray(tu)) {
+      throw new InputValidationError(
+        "Invalid FunctionCall in toolUses: expected an object."
+      );
+    }
+    const fc = tu;
+    if (fc.name !== void 0 && typeof fc.name !== "string") {
+      throw new InputValidationError(
+        "Invalid FunctionCall: name must be a string."
+      );
+    }
+    if (fc.args !== void 0 && (typeof fc.args !== "object" || fc.args === null || Array.isArray(fc.args))) {
+      throw new InputValidationError(
+        "Invalid FunctionCall: args must be an object."
+      );
+    }
+    return {
+      ...typeof fc.id === "string" ? { id: fc.id } : {},
+      ...typeof fc.name === "string" ? { name: fc.name } : {},
+      ...fc.args !== void 0 ? { args: fc.args } : {}
+    };
+  });
+  const rawToolResponses = (_d = (_c = record.toolResponses) != null ? _c : record.tool_responses) != null ? _d : [];
+  if (!Array.isArray(rawToolResponses)) {
+    throw new InputValidationError(
+      "Invalid EvalIntermediateData: toolResponses must be an array."
+    );
+  }
+  const toolResponses = rawToolResponses.map((tr) => {
+    if (typeof tr !== "object" || tr === null || Array.isArray(tr)) {
+      throw new InputValidationError(
+        "Invalid FunctionResponse in toolResponses: expected an object."
+      );
+    }
+    const fr = tr;
+    if (fr.id !== void 0 && typeof fr.id !== "string") {
+      throw new InputValidationError(
+        "Invalid FunctionResponse: id must be a string."
+      );
+    }
+    if (fr.name !== void 0 && typeof fr.name !== "string") {
+      throw new InputValidationError(
+        "Invalid FunctionResponse: name must be a string."
+      );
+    }
+    if (fr.response !== void 0 && (typeof fr.response !== "object" || fr.response === null || Array.isArray(fr.response))) {
+      throw new InputValidationError(
+        "Invalid FunctionResponse: response must be an object."
+      );
+    }
+    return {
+      ...typeof fr.id === "string" ? { id: fr.id } : {},
+      ...typeof fr.name === "string" ? { name: fr.name } : {},
+      ...fr.response !== void 0 ? { response: fr.response } : {}
+    };
+  });
+  const rawIntermediate = (_f = (_e = record.intermediateResponses) != null ? _e : record.intermediate_responses) != null ? _f : [];
+  if (!Array.isArray(rawIntermediate)) {
+    throw new InputValidationError(
+      "Invalid EvalIntermediateData: intermediateResponses must be an array."
+    );
+  }
+  const intermediateResponses = rawIntermediate.map((entry) => {
+    if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string" || !Array.isArray(entry[1])) {
+      throw new InputValidationError(
+        "Invalid intermediateResponses entry: expected [author, Part[]]."
+      );
+    }
+    return [entry[0], entry[1]];
+  });
+  return {
+    toolUses,
+    ...toolResponses.length > 0 ? { toolResponses } : {},
+    intermediateResponses
+  };
+}
+__name(normalizeIntermediateData, "normalizeIntermediateData");
+function normalizeInvocation(raw) {
+  var _a, _b, _c;
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new InputValidationError(
+      "Invalid EvalInvocation: expected an object."
+    );
+  }
+  const record = raw;
+  const rawUserContent = (_a = record.userContent) != null ? _a : record.user_content;
+  if (rawUserContent === void 0) {
+    throw new InputValidationError(
+      "Invalid EvalInvocation: userContent is required."
+    );
+  }
+  const userContent = normalizeContent(rawUserContent);
+  const rawFinalResponse = (_b = record.finalResponse) != null ? _b : record.final_response;
+  const finalResponse = rawFinalResponse !== void 0 && rawFinalResponse !== null ? normalizeContent(rawFinalResponse) : void 0;
+  const rawIntermediateData = (_c = record.intermediateData) != null ? _c : record.intermediate_data;
+  const intermediateData = rawIntermediateData !== void 0 && rawIntermediateData !== null ? normalizeIntermediateData(rawIntermediateData) : void 0;
+  const invocationId = typeof record.invocationId === "string" ? record.invocationId : typeof record.invocation_id === "string" ? record.invocation_id : void 0;
+  const creationTimestamp = readCreationTimestamp(record);
+  return {
+    ...invocationId !== void 0 ? { invocationId } : {},
+    userContent,
+    ...finalResponse !== void 0 ? { finalResponse } : {},
+    ...intermediateData !== void 0 ? { intermediateData } : {},
+    ...creationTimestamp !== void 0 ? { creationTimestamp } : {}
+  };
+}
+__name(normalizeInvocation, "normalizeInvocation");
+function normalizeSessionInput(raw) {
+  var _a, _b, _c, _d, _e;
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new InputValidationError(
+      "Invalid EvalSessionInput: expected an object."
+    );
+  }
+  const record = raw;
+  const rawAppName = (_b = (_a = record.appName) != null ? _a : record.app_name) != null ? _b : "";
+  const rawUserId = (_d = (_c = record.userId) != null ? _c : record.user_id) != null ? _d : "";
+  const rawState = (_e = record.state) != null ? _e : {};
+  if (typeof rawAppName !== "string") {
+    throw new InputValidationError(
+      "Invalid EvalSessionInput: appName must be a string."
+    );
+  }
+  if (typeof rawUserId !== "string") {
+    throw new InputValidationError(
+      "Invalid EvalSessionInput: userId must be a string."
+    );
+  }
+  if (typeof rawState !== "object" || rawState === null || Array.isArray(rawState)) {
+    throw new InputValidationError(
+      "Invalid EvalSessionInput: state must be an object."
+    );
+  }
+  return {
+    appName: rawAppName,
+    userId: rawUserId,
+    state: rawState
+  };
+}
+__name(normalizeSessionInput, "normalizeSessionInput");
+function normalizeEvalCase(raw) {
+  var _a, _b;
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new InputValidationError("Invalid EvalCase: expected an object.");
+  }
+  const record = raw;
+  const rawEvalId = (_a = record.evalId) != null ? _a : record.eval_id;
+  if (typeof rawEvalId !== "string") {
+    throw new InputValidationError(
+      "Invalid EvalCase: evalId must be a string."
+    );
+  }
+  if (!Array.isArray(record.conversation)) {
+    throw new InputValidationError(
+      "Invalid EvalCase: conversation must be an array."
+    );
+  }
+  const conversation = record.conversation.map(normalizeInvocation);
+  const rawSessionInput = (_b = record.sessionInput) != null ? _b : record.session_input;
+  const sessionInput = rawSessionInput !== void 0 && rawSessionInput !== null ? normalizeSessionInput(rawSessionInput) : void 0;
+  const creationTimestamp = readCreationTimestamp(record);
+  return {
+    evalId: rawEvalId,
+    conversation,
+    ...sessionInput !== void 0 ? { sessionInput } : {},
+    ...creationTimestamp !== void 0 ? { creationTimestamp } : {}
+  };
+}
+__name(normalizeEvalCase, "normalizeEvalCase");
+function convertLegacyInvocationToSchema(rawInvocation) {
+  var _a, _b, _c;
+  if (typeof rawInvocation !== "object" || rawInvocation === null || Array.isArray(rawInvocation)) {
+    throw new InputValidationError(
+      "Invalid legacy invocation: expected an object."
+    );
+  }
+  const record = rawInvocation;
+  if (typeof record.query !== "string") {
+    throw new InputValidationError(
+      "Invalid legacy invocation: query must be a string."
+    );
+  }
+  const query = record.query;
+  const rawReference = (_a = record.reference) != null ? _a : "";
+  if (typeof rawReference !== "string") {
+    throw new InputValidationError(
+      "Invalid legacy invocation: reference must be a string."
+    );
+  }
+  const rawExpectedToolUse = (_b = record.expected_tool_use) != null ? _b : [];
+  if (!Array.isArray(rawExpectedToolUse)) {
+    throw new InputValidationError(
+      "Invalid legacy invocation: expected_tool_use must be an array."
+    );
+  }
+  const toolResponses = [];
+  const toolUses = rawExpectedToolUse.map((oldToolUse) => {
+    var _a2;
+    if (typeof oldToolUse !== "object" || oldToolUse === null || Array.isArray(oldToolUse)) {
+      throw new InputValidationError(
+        "Invalid legacy tool_use entry: expected an object."
+      );
+    }
+    const tu = oldToolUse;
+    if (typeof tu.tool_name !== "string") {
+      throw new InputValidationError(
+        "Invalid legacy tool_use: tool_name must be a string."
+      );
+    }
+    if (tu.tool_input !== void 0 && (typeof tu.tool_input !== "object" || tu.tool_input === null || Array.isArray(tu.tool_input))) {
+      throw new InputValidationError(
+        "Invalid legacy tool_use: tool_input must be an object."
+      );
+    }
+    const toolUse = {
+      name: tu.tool_name,
+      args: (_a2 = tu.tool_input) != null ? _a2 : {}
+    };
+    if (tu.mock_tool_output !== void 0) {
+      toolUse.id = randomUUID();
+      toolResponses.push({
+        id: toolUse.id,
+        name: tu.tool_name,
+        response: { [MOCK_TOOL_OUTPUT_RESPONSE_KEY]: tu.mock_tool_output }
+      });
+    }
+    return toolUse;
+  });
+  const rawExpectedIntermediate = (_c = record.expected_intermediate_agent_responses) != null ? _c : [];
+  if (!Array.isArray(rawExpectedIntermediate)) {
+    throw new InputValidationError(
+      "Invalid legacy invocation: expected_intermediate_agent_responses must be an array."
+    );
+  }
+  const intermediateResponses = rawExpectedIntermediate.map((oldIntermediate) => {
+    if (Array.isArray(oldIntermediate)) {
+      if (oldIntermediate.length !== 2 || typeof oldIntermediate[0] !== "string" || !Array.isArray(oldIntermediate[1])) {
+        throw new InputValidationError(
+          "Invalid legacy intermediate response tuple: expected [author, Part[]]."
+        );
+      }
+      return [oldIntermediate[0], oldIntermediate[1]];
+    }
+    if (typeof oldIntermediate === "object" && oldIntermediate !== null) {
+      const ir = oldIntermediate;
+      if (typeof ir.author !== "string" || typeof ir.text !== "string") {
+        throw new InputValidationError(
+          "Invalid legacy intermediate response: author and text must be strings."
+        );
+      }
+      return [ir.author, [{ text: ir.text }]];
+    }
+    throw new InputValidationError(
+      "Invalid legacy intermediate response: expected an object or tuple."
+    );
+  });
+  return {
+    invocationId: randomUUID(),
+    userContent: {
+      parts: [{ text: query }],
+      role: "user"
+    },
+    finalResponse: {
+      parts: [{ text: rawReference }],
+      role: "model"
+    },
+    intermediateData: {
+      toolUses,
+      ...toolResponses.length > 0 ? { toolResponses } : {},
+      intermediateResponses
+    },
+    creationTimestamp: Date.now() / 1e3
+  };
+}
+__name(convertLegacyInvocationToSchema, "convertLegacyInvocationToSchema");
+function convertEvalSetToSchema(evalSetId, evalSetInJsonFormat) {
+  if (!Array.isArray(evalSetInJsonFormat)) {
+    throw new InputValidationError(
+      "Invalid legacy eval set JSON: expected an array of eval entries."
+    );
+  }
+  const evalCases = [];
+  for (const rawEvalCase of evalSetInJsonFormat) {
+    if (typeof rawEvalCase !== "object" || rawEvalCase === null || Array.isArray(rawEvalCase)) {
+      throw new InputValidationError(
+        "Invalid legacy eval case: expected an object."
+      );
+    }
+    const oldEvalCase = rawEvalCase;
+    if (typeof oldEvalCase.name !== "string") {
+      throw new InputValidationError(
+        "Invalid legacy eval case: name must be a string."
+      );
+    }
+    if (!Array.isArray(oldEvalCase.data)) {
+      throw new InputValidationError(
+        "Invalid legacy eval case: data must be an array."
+      );
+    }
+    const newInvocations = oldEvalCase.data.map(
+      convertLegacyInvocationToSchema
+    );
+    let sessionInput;
+    if (oldEvalCase.initial_session !== void 0 && oldEvalCase.initial_session !== null) {
+      if (typeof oldEvalCase.initial_session !== "object" || Array.isArray(oldEvalCase.initial_session)) {
+        throw new InputValidationError(
+          "Invalid legacy eval case: initial_session must be an object."
+        );
+      }
+      const initSess = oldEvalCase.initial_session;
+      if (Object.keys(initSess).length > 0) {
+        sessionInput = normalizeSessionInput(initSess);
+      }
+    }
+    evalCases.push({
+      evalId: oldEvalCase.name,
+      conversation: newInvocations,
+      ...sessionInput !== void 0 ? { sessionInput } : {},
+      creationTimestamp: Date.now() / 1e3
+    });
+  }
+  return {
+    evalSetId,
+    name: evalSetId,
+    creationTimestamp: Date.now() / 1e3,
+    evalCases
+  };
+}
+__name(convertEvalSetToSchema, "convertEvalSetToSchema");
+function parseEvalSetObject(raw) {
+  var _a, _b;
+  const rawEvalSetId = (_a = raw.evalSetId) != null ? _a : raw.eval_set_id;
+  const rawEvalCases = (_b = raw.evalCases) != null ? _b : raw.eval_cases;
+  if (typeof rawEvalSetId !== "string" || !Array.isArray(rawEvalCases)) {
+    throw new InputValidationError(
+      "Invalid EvalSet JSON: missing evalSetId or evalCases."
+    );
+  }
+  const name = typeof raw.name === "string" ? raw.name : void 0;
+  const description = typeof raw.description === "string" ? raw.description : void 0;
+  const creationTimestamp = readCreationTimestamp(raw);
+  return {
+    evalSetId: rawEvalSetId,
+    ...name !== void 0 ? { name } : {},
+    ...description !== void 0 ? { description } : {},
+    evalCases: rawEvalCases.map(normalizeEvalCase),
+    ...creationTimestamp !== void 0 ? { creationTimestamp } : {}
+  };
+}
+__name(parseEvalSetObject, "parseEvalSetObject");
+function loadEvalSetFromFile(evalSetFilePath, evalSetId) {
+  const content = fs.readFileSync(evalSetFilePath, "utf-8");
+  const parsed = JSON.parse(content);
+  if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) && ("evalSetId" in parsed || "eval_set_id" in parsed || "evalCases" in parsed || "eval_cases" in parsed)) {
+    return parseEvalSetObject(parsed);
+  }
+  return convertEvalSetToSchema(evalSetId, parsed);
+}
+__name(loadEvalSetFromFile, "loadEvalSetFromFile");
+const _LocalEvalSetsManager = class _LocalEvalSetsManager {
+  constructor(agentsDir) {
+    this.agentsDir = agentsDir;
+  }
+  /**
+   * Loads an {@link EvalSet} identified by `appName` and `evalSetId`, or
+   * returns `undefined` if the file does not exist.
+   */
+  getEvalSet(appName, evalSetId) {
+    const evalSetFilePath = this.getEvalSetFilePath(appName, evalSetId);
+    if (!fs.existsSync(evalSetFilePath)) {
+      return void 0;
+    }
+    return loadEvalSetFromFile(evalSetFilePath, evalSetId);
+  }
+  /**
+   * Creates an empty {@link EvalSet} file for `appName` and `evalSetId`.
+   *
+   * @throws {InputValidationError} If `evalSetId` is invalid or the eval set
+   *     already exists.
+   */
+  createEvalSet(appName, evalSetId) {
+    validateEvalId("eval set ID", evalSetId);
+    const newEvalSetPath = this.getEvalSetFilePath(appName, evalSetId);
+    if (fs.existsSync(newEvalSetPath)) {
+      throw new InputValidationError(
+        `EvalSet ${evalSetId} already exists for app ${appName}.`
+      );
+    }
+    logger.info(`Creating eval set file \`${newEvalSetPath}\``);
+    const newEvalSet = {
+      evalSetId,
+      name: evalSetId,
+      evalCases: [],
+      creationTimestamp: Date.now() / 1e3
+    };
+    this.writeEvalSet(newEvalSetPath, newEvalSet);
+    return newEvalSet;
+  }
+  /**
+   * Lists all eval set IDs stored under `appName`, sorted alphabetically.
+   */
+  listEvalSets(appName) {
+    validatePathSegment(appName, "appName");
+    const baseDir = path.resolve(this.agentsDir);
+    const appDir = path.resolve(baseDir, appName);
+    if (!isStrictlyInside(baseDir, appDir)) {
+      throw new InputValidationError(`Invalid appName '${appName}'.`);
+    }
+    if (!fs.existsSync(appDir)) {
+      return [];
+    }
+    const evalSets = [];
+    for (const file of fs.readdirSync(appDir)) {
+      if (file.endsWith(EVAL_SET_FILE_EXTENSION)) {
+        evalSets.push(file.slice(0, -EVAL_SET_FILE_EXTENSION.length));
+      }
+    }
+    return evalSets.sort();
+  }
+  /**
+   * Adds `evalCase` to the {@link EvalSet} identified by `appName` and
+   * `evalSetId`.
+   *
+   * @throws {NotFoundError} If the eval set does not exist.
+   * @throws {InputValidationError} If `evalCase.evalId` is invalid or already exists.
+   */
+  addEvalCase(appName, evalSetId, evalCase) {
+    const normalizedCase = normalizeEvalCase(evalCase);
+    const evalCaseId = normalizedCase.evalId;
+    validateEvalId("Eval Case ID", evalCaseId);
+    const evalSet = this.getEvalSet(appName, evalSetId);
+    if (!evalSet) {
+      throw new NotFoundError(`Eval set \`${evalSetId}\` not found.`);
+    }
+    if (evalSet.evalCases.some((x) => x.evalId === evalCaseId)) {
+      throw new InputValidationError(
+        `Eval id \`${evalCaseId}\` already exists in \`${evalSetId}\` eval set.`
+      );
+    }
+    evalSet.evalCases.push(normalizedCase);
+    const evalSetFilePath = this.getEvalSetFilePath(appName, evalSetId);
+    this.writeEvalSet(evalSetFilePath, evalSet);
+  }
+  /**
+   * Returns a single {@link EvalCase} by `appName`, `evalSetId`, and
+   * `evalCaseId`, or `undefined` if not found.
+   */
+  getEvalCase(appName, evalSetId, evalCaseId) {
+    const evalSet = this.getEvalSet(appName, evalSetId);
+    if (!evalSet) {
+      return void 0;
+    }
+    return evalSet.evalCases.find((c) => c.evalId === evalCaseId);
+  }
+  /**
+   * Updates an existing {@link EvalCase} in the {@link EvalSet} identified by
+   * `appName` and `evalSetId`.
+   *
+   * @throws {NotFoundError} If the eval set or eval case is not found.
+   */
+  updateEvalCase(appName, evalSetId, updatedEvalCase) {
+    const normalizedCase = normalizeEvalCase(updatedEvalCase);
+    const evalCaseId = normalizedCase.evalId;
+    const evalSet = this.getEvalSet(appName, evalSetId);
+    if (!evalSet || !evalSet.evalCases.some((c) => c.evalId === evalCaseId)) {
+      throw new NotFoundError(
+        `Eval Set \`${evalSetId}\` or Eval id \`${evalCaseId}\` not found.`
+      );
+    }
+    evalSet.evalCases = evalSet.evalCases.filter(
+      (c) => c.evalId !== evalCaseId
+    );
+    evalSet.evalCases.push(normalizedCase);
+    const evalSetFilePath = this.getEvalSetFilePath(appName, evalSetId);
+    this.writeEvalSet(evalSetFilePath, evalSet);
+  }
+  /**
+   * Deletes an {@link EvalCase} identified by `appName`, `evalSetId`, and
+   * `evalCaseId`.
+   *
+   * @throws {NotFoundError} If the eval set or eval case to delete is not found.
+   */
+  deleteEvalCase(appName, evalSetId, evalCaseId) {
+    const evalSet = this.getEvalSet(appName, evalSetId);
+    if (!evalSet || !evalSet.evalCases.some((c) => c.evalId === evalCaseId)) {
+      throw new NotFoundError(
+        `Eval Set \`${evalSetId}\` or Eval id \`${evalCaseId}\` not found.`
+      );
+    }
+    logger.info(
+      `EvalCase \`${evalCaseId}\` was found in the eval set. It will be removed permanently.`
+    );
+    evalSet.evalCases = evalSet.evalCases.filter(
+      (c) => c.evalId !== evalCaseId
+    );
+    const evalSetFilePath = this.getEvalSetFilePath(appName, evalSetId);
+    this.writeEvalSet(evalSetFilePath, evalSet);
+  }
+  getEvalSetFilePath(appName, evalSetId) {
+    validatePathSegment(appName, "appName");
+    validatePathSegment(evalSetId, "evalSetId");
+    validateEvalId("eval set ID", evalSetId);
+    const baseDir = path.resolve(this.agentsDir);
+    const appDir = path.resolve(baseDir, appName);
+    if (!isStrictlyInside(baseDir, appDir)) {
+      throw new InputValidationError(`Invalid appName '${appName}'.`);
+    }
+    const filePath = path.resolve(
+      appDir,
+      `${evalSetId}${EVAL_SET_FILE_EXTENSION}`
+    );
+    if (!isStrictlyInside(appDir, filePath)) {
+      throw new InputValidationError(`Invalid evalSetId '${evalSetId}'.`);
+    }
+    return filePath;
+  }
+  writeEvalSet(evalSetPath, evalSet) {
+    fs.mkdirSync(path.dirname(evalSetPath), { recursive: true });
+    fs.writeFileSync(evalSetPath, JSON.stringify(evalSet, null, 2), "utf-8");
+  }
+};
+__name(_LocalEvalSetsManager, "LocalEvalSetsManager");
+let LocalEvalSetsManager = _LocalEvalSetsManager;
 export {
   DEFAULT_CRITERIA,
   EVAL_SESSION_ID_PREFIX,
   EvalStatus,
+  LocalEvalSetResultsManager,
+  LocalEvalSetsManager,
   MISSING_EVAL_DEPENDENCIES_MESSAGE,
   RESPONSE_EVALUATION_SCORE_KEY,
   RESPONSE_MATCH_SCORE_KEY,
   TOOL_TRAJECTORY_SCORE_KEY,
+  convertEvalSetToSchema,
+  createEvalSetResult,
   getEvaluationCriteriaOrDefault,
   getRootAgent,
+  loadEvalSetFromFile,
+  normalizeEvalCase,
   parseAndGetEvalsToRun,
+  parseEvalSetResultJson,
   runEvals,
   tryGetResetFunc
 };