@google/adk 2.2.1 → 2.4.0

package/dist/esm/cli/cli_eval.js

@@ -0,0 +1,510 @@
+import {createRequire as topLevelCreateRequire} from 'module';
+const require = topLevelCreateRequire(import.meta.url);
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { randomUUID } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { pathToFileURL } from "node:url";
+import { isBaseAgent } from "../agents/base_agent.js";
+import {
+  isLlmAgent
+} from "../agents/llm_agent.js";
+import { InMemoryArtifactService } from "../artifacts/in_memory_artifact_service.js";
+import { Runner } from "../runner/runner.js";
+import { InMemorySessionService } from "../sessions/in_memory_session_service.js";
+import { logger } from "../utils/logger.js";
+import {
+  convertSessionToEvalFormat,
+  createEmptyState
+} from "./utils/evals.js";
+const MISSING_EVAL_DEPENDENCIES_MESSAGE = "Eval module is not installed, please install via `pip install google-adk[eval]`.";
+const TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score";
+const RESPONSE_MATCH_SCORE_KEY = "response_match_score";
+const RESPONSE_EVALUATION_SCORE_KEY = "response_evaluation_score";
+const EVAL_SESSION_ID_PREFIX = "___eval___session___";
+const DEFAULT_CRITERIA = {
+  [TOOL_TRAJECTORY_SCORE_KEY]: 1,
+  [RESPONSE_MATCH_SCORE_KEY]: 0.8
+};
+var EvalStatus = /* @__PURE__ */ ((EvalStatus2) => {
+  EvalStatus2[EvalStatus2["PASSED"] = 1] = "PASSED";
+  EvalStatus2[EvalStatus2["FAILED"] = 2] = "FAILED";
+  EvalStatus2[EvalStatus2["NOT_EVALUATED"] = 3] = "NOT_EVALUATED";
+  return EvalStatus2;
+})(EvalStatus || {});
+function getEvaluationCriteriaOrDefault(evalConfigFilePath) {
+  if (evalConfigFilePath) {
+    const content = fs.readFileSync(evalConfigFilePath, "utf-8");
+    const configData = JSON.parse(content);
+    if (typeof configData === "object" && configData !== null && "criteria" in configData && typeof configData.criteria === "object" && configData.criteria !== null && !Array.isArray(configData.criteria)) {
+      return configData.criteria;
+    }
+    throw new Error(
+      `Invalid format for test_config.json at ${evalConfigFilePath}. Expected a 'criteria' dictionary.`
+    );
+  }
+  logger.info("No config file supplied. Using default criteria.");
+  return { ...DEFAULT_CRITERIA };
+}
+__name(getEvaluationCriteriaOrDefault, "getEvaluationCriteriaOrDefault");
+function resolveAgentModulePath(agentModuleFilePath) {
+  const resolved = path.resolve(agentModuleFilePath);
+  if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) {
+    const candidates = ["agent.js", "agent.ts", "index.js", "index.ts"];
+    for (const candidate of candidates) {
+      const candidatePath = path.join(resolved, candidate);
+      if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isFile()) {
+        return candidatePath;
+      }
+    }
+  }
+  return resolved;
+}
+__name(resolveAgentModulePath, "resolveAgentModulePath");
+async function getRootAgent(agentModuleFilePath) {
+  var _a, _b, _c, _d;
+  const resolvedPath = resolveAgentModulePath(agentModuleFilePath);
+  const moduleUrl = pathToFileURL(resolvedPath).href;
+  const mod = await Promise.resolve().then(() => __toESM(require(moduleUrl)));
+  const nestedAgent = typeof mod.agent === "object" && mod.agent !== null ? mod.agent : void 0;
+  const candidate = (_d = (_c = (_b = (_a = mod.rootAgent) != null ? _a : mod.root_agent) != null ? _b : mod.default) != null ? _c : nestedAgent == null ? void 0 : nestedAgent.rootAgent) != null ? _d : nestedAgent == null ? void 0 : nestedAgent.root_agent;
+  if (isBaseAgent(candidate)) {
+    return candidate;
+  }
+  throw new Error(
+    `No rootAgent found in module ${agentModuleFilePath}. Export a BaseAgent as 'rootAgent'.`
+  );
+}
+__name(getRootAgent, "getRootAgent");
+async function tryGetResetFunc(agentModuleFilePath) {
+  var _a, _b, _c;
+  try {
+    const resolvedPath = resolveAgentModulePath(agentModuleFilePath);
+    const moduleUrl = pathToFileURL(resolvedPath).href;
+    const mod = await Promise.resolve().then(() => __toESM(require(moduleUrl)));
+    const nestedAgent = typeof mod.agent === "object" && mod.agent !== null ? mod.agent : void 0;
+    const resetCandidate = (_c = (_b = (_a = mod.reset_data) != null ? _a : mod.resetData) != null ? _b : nestedAgent == null ? void 0 : nestedAgent.reset_data) != null ? _c : nestedAgent == null ? void 0 : nestedAgent.resetData;
+    if (typeof resetCandidate === "function") {
+      return resetCandidate;
+    }
+  } catch {
+  }
+  return void 0;
+}
+__name(tryGetResetFunc, "tryGetResetFunc");
+function parseAndGetEvalsToRun(evalSetFilePaths) {
+  const evalSetToEvals = {};
+  for (const inputEvalSet of evalSetFilePaths) {
+    let evalSetFile;
+    let evals = [];
+    if (inputEvalSet.includes(":")) {
+      const colonIndex = inputEvalSet.indexOf(":");
+      evalSetFile = inputEvalSet.slice(0, colonIndex);
+      evals = inputEvalSet.slice(colonIndex + 1).split(",").filter((s) => s.length > 0);
+    } else {
+      evalSetFile = inputEvalSet;
+    }
+    if (!Object.prototype.hasOwnProperty.call(evalSetToEvals, evalSetFile)) {
+      evalSetToEvals[evalSetFile] = [];
+    }
+    evalSetToEvals[evalSetFile].push(...evals);
+  }
+  return evalSetToEvals;
+}
+__name(parseAndGetEvalsToRun, "parseAndGetEvalsToRun");
+function deepEqual(a, b) {
+  if (Object.is(a, b)) {
+    return true;
+  }
+  if (typeof a !== "object" || a === null || typeof b !== "object" || b === null) {
+    return false;
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) {
+    return false;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      return false;
+    }
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) {
+        return false;
+      }
+    }
+    return true;
+  }
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) {
+    return false;
+  }
+  for (const key of keysA) {
+    if (!Object.prototype.hasOwnProperty.call(b, key) || !deepEqual(
+      a[key],
+      b[key]
+    )) {
+      return false;
+    }
+  }
+  return true;
+}
+__name(deepEqual, "deepEqual");
+function toolUsesEqual(actual, expected) {
+  if (actual.length !== expected.length) {
+    return false;
+  }
+  for (let i = 0; i < actual.length; i++) {
+    if (actual[i].tool_name !== expected[i].tool_name || !deepEqual(actual[i].tool_input, expected[i].tool_input)) {
+      return false;
+    }
+  }
+  return true;
+}
+__name(toolUsesEqual, "toolUsesEqual");
+function computeRouge1FMeasure(target, prediction) {
+  var _a, _b, _c, _d;
+  const targetTokens = (_a = target.toLowerCase().match(/[a-z0-9]+/g)) != null ? _a : [];
+  const predictionTokens = (_b = prediction.toLowerCase().match(/[a-z0-9]+/g)) != null ? _b : [];
+  if (targetTokens.length === 0 && predictionTokens.length === 0) {
+    return 1;
+  }
+  if (targetTokens.length === 0 || predictionTokens.length === 0) {
+    return 0;
+  }
+  const targetCounts = /* @__PURE__ */ new Map();
+  for (const token of targetTokens) {
+    targetCounts.set(token, ((_c = targetCounts.get(token)) != null ? _c : 0) + 1);
+  }
+  let overlap = 0;
+  for (const token of predictionTokens) {
+    const remaining = (_d = targetCounts.get(token)) != null ? _d : 0;
+    if (remaining > 0) {
+      overlap += 1;
+      targetCounts.set(token, remaining - 1);
+    }
+  }
+  const precision = overlap / predictionTokens.length;
+  const recall = overlap / targetTokens.length;
+  if (precision + recall === 0) {
+    return 0;
+  }
+  return 2 * precision * recall / (precision + recall);
+}
+__name(computeRouge1FMeasure, "computeRouge1FMeasure");
+function evaluateSingleMetric(metric, actualInvocations, expectedInvocations) {
+  var _a, _b, _c, _d;
+  const numInvocations = Math.min(
+    actualInvocations.length,
+    expectedInvocations.length
+  );
+  if (numInvocations === 0) {
+    return {
+      overall: {
+        metric_name: metric.metric_name,
+        threshold: metric.threshold,
+        score: void 0,
+        eval_status: 3 /* NOT_EVALUATED */
+      },
+      perInvocation: []
+    };
+  }
+  const perInvocation = [];
+  let totalScore = 0;
+  for (let i = 0; i < numInvocations; i++) {
+    const actual = actualInvocations[i];
+    const expected = expectedInvocations[i];
+    let score = 0;
+    if (metric.metric_name === TOOL_TRAJECTORY_SCORE_KEY) {
+      score = toolUsesEqual(
+        (_a = actual.expected_tool_use) != null ? _a : [],
+        (_b = expected.expected_tool_use) != null ? _b : []
+      ) ? 1 : 0;
+    } else if (metric.metric_name === RESPONSE_MATCH_SCORE_KEY || metric.metric_name === RESPONSE_EVALUATION_SCORE_KEY) {
+      score = computeRouge1FMeasure(
+        (_c = expected.reference) != null ? _c : "",
+        (_d = actual.reference) != null ? _d : ""
+      );
+    }
+    totalScore += score;
+    perInvocation.push({
+      metric_name: metric.metric_name,
+      threshold: metric.threshold,
+      score,
+      eval_status: score >= metric.threshold ? 1 /* PASSED */ : 2 /* FAILED */
+    });
+  }
+  const avgScore = totalScore / numInvocations;
+  return {
+    overall: {
+      metric_name: metric.metric_name,
+      threshold: metric.threshold,
+      score: avgScore,
+      eval_status: avgScore >= metric.threshold ? 1 /* PASSED */ : 2 /* FAILED */
+    },
+    perInvocation
+  };
+}
+__name(evaluateSingleMetric, "evaluateSingleMetric");
+function installMockToolOutputCallbacks(agent, interceptor, savedCallbacks) {
+  if (isLlmAgent(agent) && !savedCallbacks.has(agent)) {
+    savedCallbacks.set(agent, agent.beforeToolCallback);
+    const existing = agent.beforeToolCallback;
+    agent.beforeToolCallback = existing ? [interceptor, ...Array.isArray(existing) ? existing : [existing]] : interceptor;
+  }
+  for (const subAgent of agent.subAgents) {
+    installMockToolOutputCallbacks(subAgent, interceptor, savedCallbacks);
+  }
+}
+__name(installMockToolOutputCallbacks, "installMockToolOutputCallbacks");
+async function generateActualInvocations(rootAgent, evalCase, resetFunc, sessionService, artifactService) {
+  var _a, _b, _c, _d, _e, _f, _g;
+  if (resetFunc) {
+    await resetFunc();
+  }
+  const appName = (_b = (_a = evalCase.initial_session) == null ? void 0 : _a.app_name) != null ? _b : "EvaluationGenerator";
+  const userId = (_d = (_c = evalCase.initial_session) == null ? void 0 : _c.user_id) != null ? _d : "test_user_id";
+  const sessionId = `${EVAL_SESSION_ID_PREFIX}${randomUUID()}`;
+  const activeSessionService = sessionService != null ? sessionService : new InMemorySessionService();
+  const activeArtifactService = artifactService != null ? artifactService : new InMemoryArtifactService();
+  const initialState = createEmptyState(
+    rootAgent,
+    (_f = (_e = evalCase.initial_session) == null ? void 0 : _e.state) != null ? _f : {}
+  );
+  await activeSessionService.createSession({
+    appName,
+    userId,
+    sessionId,
+    state: initialState
+  });
+  const pendingMockToolUses = [];
+  for (const turn of evalCase.data) {
+    for (const expectedTool of (_g = turn.expected_tool_use) != null ? _g : []) {
+      if (expectedTool.mock_tool_output !== void 0) {
+        pendingMockToolUses.push({ ...expectedTool });
+      }
+    }
+  }
+  const savedCallbacks = /* @__PURE__ */ new Map();
+  if (pendingMockToolUses.length > 0) {
+    const interceptor = /* @__PURE__ */ __name(({ tool, args }) => {
+      const idx = pendingMockToolUses.findIndex(
+        (expected) => {
+          var _a2;
+          return expected.mock_tool_output !== void 0 && tool.name === expected.tool_name && deepEqual(args, (_a2 = expected.tool_input) != null ? _a2 : {});
+        }
+      );
+      if (idx !== -1) {
+        const [matched] = pendingMockToolUses.splice(idx, 1);
+        return { result: matched.mock_tool_output };
+      }
+      return void 0;
+    }, "interceptor");
+    installMockToolOutputCallbacks(rootAgent, interceptor, savedCallbacks);
+  }
+  const runner = new Runner({
+    appName,
+    agent: rootAgent,
+    sessionService: activeSessionService,
+    artifactService: activeArtifactService
+  });
+  try {
+    for (const turn of evalCase.data) {
+      for await (const _event of runner.runAsync({
+        userId,
+        sessionId,
+        newMessage: {
+          role: "user",
+          parts: [{ text: turn.query }]
+        }
+      })) {
+      }
+    }
+  } finally {
+    for (const [llmAgent, originalCallback] of savedCallbacks.entries()) {
+      llmAgent.beforeToolCallback = originalCallback;
+    }
+  }
+  const recordedSession = await activeSessionService.getSession({
+    appName,
+    userId,
+    sessionId
+  });
+  const actualInvocations = convertSessionToEvalFormat(recordedSession);
+  while (actualInvocations.length < evalCase.data.length) {
+    const idx = actualInvocations.length;
+    actualInvocations.push({
+      query: evalCase.data[idx].query,
+      expected_tool_use: [],
+      expected_intermediate_agent_responses: [],
+      reference: ""
+    });
+  }
+  return { actualInvocations, sessionId, userId };
+}
+__name(generateActualInvocations, "generateActualInvocations");
+function resolveEvalCasesForSet(evalSetId, entries) {
+  if (entries.length > 0 && typeof entries[0] === "string") {
+    const filterNames = entries;
+    const fileData = JSON.parse(
+      fs.readFileSync(evalSetId, "utf-8")
+    );
+    return fileData.filter((e) => filterNames.includes(e.name));
+  }
+  if (entries.length === 0 && fs.existsSync(evalSetId) && fs.statSync(evalSetId).isFile()) {
+    return JSON.parse(fs.readFileSync(evalSetId, "utf-8"));
+  }
+  return entries;
+}
+__name(resolveEvalCasesForSet, "resolveEvalCasesForSet");
+function printCaseDetailedResults(result) {
+  var _a;
+  logger.info(
+    `Eval result for ${result.eval_set_file}:${result.eval_id}: ${EvalStatus[result.final_eval_status]}`
+  );
+  for (const metricResult of result.overall_eval_metric_results) {
+    logger.info(
+      `  Metric ${metricResult.metric_name}: score=${(_a = metricResult.score) != null ? _a : "N/A"} threshold=${metricResult.threshold} status=${EvalStatus[metricResult.eval_status]}`
+    );
+  }
+}
+__name(printCaseDetailedResults, "printCaseDetailedResults");
+async function* runEvals(optionsOrMap, rootAgentArg, resetFuncArg, evalMetricsArg, printDetailedResultsArg = false, sessionServiceArg, artifactServiceArg) {
+  var _a, _b;
+  let evalCasesByEvalSetId;
+  let rootAgent;
+  let resetFunc;
+  let evalMetrics;
+  let printDetailedResults;
+  let sessionService;
+  let artifactService;
+  if (rootAgentArg === void 0 && "evalCasesByEvalSetId" in optionsOrMap && "rootAgent" in optionsOrMap && isBaseAgent(optionsOrMap.rootAgent)) {
+    const opts = optionsOrMap;
+    evalCasesByEvalSetId = opts.evalCasesByEvalSetId;
+    rootAgent = opts.rootAgent;
+    resetFunc = opts.resetFunc;
+    evalMetrics = (_a = opts.evalMetrics) != null ? _a : Object.entries(DEFAULT_CRITERIA).map(([metric_name, threshold]) => ({
+      metric_name,
+      threshold
+    }));
+    printDetailedResults = (_b = opts.printDetailedResults) != null ? _b : false;
+    sessionService = opts.sessionService;
+    artifactService = opts.artifactService;
+  } else {
+    evalCasesByEvalSetId = optionsOrMap;
+    if (!rootAgentArg) {
+      throw new Error("rootAgent is required to run evaluations.");
+    }
+    rootAgent = rootAgentArg;
+    resetFunc = resetFuncArg;
+    evalMetrics = evalMetricsArg != null ? evalMetricsArg : Object.entries(DEFAULT_CRITERIA).map(([metric_name, threshold]) => ({
+      metric_name,
+      threshold
+    }));
+    printDetailedResults = printDetailedResultsArg;
+    sessionService = sessionServiceArg;
+    artifactService = artifactServiceArg;
+  }
+  for (const [evalSetId, rawEntries] of Object.entries(evalCasesByEvalSetId)) {
+    const evalCases = resolveEvalCasesForSet(evalSetId, rawEntries);
+    for (const evalCase of evalCases) {
+      const evalName = evalCase.name;
+      const { actualInvocations, sessionId, userId } = await generateActualInvocations(
+        rootAgent,
+        evalCase,
+        resetFunc,
+        sessionService,
+        artifactService
+      );
+      const overallEvalMetricResults = [];
+      const evalMetricResultsPairs = [];
+      const perInvocationMap = evalCase.data.map(
+        (expectedInvocation, idx) => ({
+          actual_invocation: actualInvocations[idx],
+          expected_invocation: expectedInvocation,
+          eval_metric_results: []
+        })
+      );
+      for (const metric of evalMetrics) {
+        const { overall, perInvocation } = evaluateSingleMetric(
+          metric,
+          actualInvocations,
+          evalCase.data
+        );
+        overallEvalMetricResults.push(overall);
+        evalMetricResultsPairs.push([metric, overall]);
+        for (let i = 0; i < perInvocation.length; i++) {
+          if (perInvocationMap[i]) {
+            perInvocationMap[i].eval_metric_results.push(perInvocation[i]);
+          }
+        }
+      }
+      let finalEvalStatus = 3 /* NOT_EVALUATED */;
+      for (const overallMetricResult of overallEvalMetricResults) {
+        if (overallMetricResult.eval_status === 1 /* PASSED */) {
+          if (finalEvalStatus === 3 /* NOT_EVALUATED */) {
+            finalEvalStatus = 1 /* PASSED */;
+          }
+        } else if (overallMetricResult.eval_status === 2 /* FAILED */) {
+          finalEvalStatus = 2 /* FAILED */;
+          break;
+        }
+      }
+      const result = {
+        eval_set_file: evalSetId,
+        eval_id: evalName,
+        final_eval_status: finalEvalStatus,
+        eval_metric_results: evalMetricResultsPairs,
+        overall_eval_metric_results: overallEvalMetricResults,
+        eval_metric_result_per_invocation: perInvocationMap,
+        session_id: sessionId,
+        user_id: userId
+      };
+      if (printDetailedResults) {
+        printCaseDetailedResults(result);
+      }
+      yield result;
+    }
+  }
+}
+__name(runEvals, "runEvals");
+export {
+  DEFAULT_CRITERIA,
+  EVAL_SESSION_ID_PREFIX,
+  EvalStatus,
+  MISSING_EVAL_DEPENDENCIES_MESSAGE,
+  RESPONSE_EVALUATION_SCORE_KEY,
+  RESPONSE_MATCH_SCORE_KEY,
+  TOOL_TRAJECTORY_SCORE_KEY,
+  getEvaluationCriteriaOrDefault,
+  getRootAgent,
+  parseAndGetEvalsToRun,
+  runEvals,
+  tryGetResetFunc
+};

package/dist/esm/cli/utils/envs.js

@@ -0,0 +1,78 @@
+import {createRequire as topLevelCreateRequire} from 'module';
+const require = topLevelCreateRequire(import.meta.url);
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { logger } from "../../utils/logger.js";
+function walkToRootUntilFound(folder, filename = ".env") {
+  const checkpath = path.join(folder, filename);
+  if (fs.existsSync(checkpath) && fs.statSync(checkpath).isFile()) {
+    return path.resolve(checkpath);
+  }
+  const parentFolder = path.dirname(folder);
+  if (parentFolder === folder) {
+    return "";
+  }
+  return walkToRootUntilFound(parentFolder, filename);
+}
+__name(walkToRootUntilFound, "walkToRootUntilFound");
+function parseAndLoadDotenv(content) {
+  const lines = content.split(/\r?\n/);
+  for (const rawLine of lines) {
+    const trimmed = rawLine.trim();
+    if (!trimmed || trimmed.startsWith("#")) {
+      continue;
+    }
+    const stripped = trimmed.replace(/^export\s+/, "");
+    const eqIndex = stripped.indexOf("=");
+    if (eqIndex <= 0) {
+      continue;
+    }
+    const key = stripped.slice(0, eqIndex).trim();
+    if (!key) {
+      continue;
+    }
+    let value = stripped.slice(eqIndex + 1).trim();
+    if (value.length >= 2 && (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'"))) {
+      const quote = value[0];
+      value = value.slice(1, -1);
+      if (quote === '"') {
+        value = value.replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\"/g, '"').replace(/\\\\/g, "\\");
+      }
+    } else {
+      const commentIndex = value.search(/\s+#/);
+      if (commentIndex !== -1) {
+        value = value.slice(0, commentIndex).trimEnd();
+      }
+    }
+    process.env[key] = value;
+  }
+}
+__name(parseAndLoadDotenv, "parseAndLoadDotenv");
+function loadDotenvForAgent(agentName, agentParentFolder = ".", filename = ".env") {
+  let startingFolder = path.resolve(agentParentFolder, agentName);
+  if (fs.existsSync(startingFolder) && fs.statSync(startingFolder).isFile()) {
+    startingFolder = path.dirname(startingFolder);
+  }
+  const dotenvFilePath = walkToRootUntilFound(startingFolder, filename);
+  if (dotenvFilePath) {
+    const content = fs.readFileSync(dotenvFilePath, "utf-8");
+    parseAndLoadDotenv(content);
+    logger.info(
+      `Loaded ${filename} file for ${agentName} at ${dotenvFilePath}`
+    );
+  } else {
+    logger.info(`No ${filename} file found for ${agentName}`);
+  }
+}
+__name(loadDotenvForAgent, "loadDotenvForAgent");
+export {
+  loadDotenvForAgent,
+  walkToRootUntilFound
+};

package/dist/esm/cli/utils/evals.js

@@ -0,0 +1,109 @@
+import {createRequire as topLevelCreateRequire} from 'module';
+const require = topLevelCreateRequire(import.meta.url);
+var __defProp = Object.defineProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { isLlmAgent } from "../../agents/llm_agent.js";
+function convertSessionToEvalFormat(session) {
+  var _a, _b, _c;
+  const events = session && session.events ? session.events : [];
+  const evalCases = [];
+  let i = 0;
+  while (i < events.length) {
+    const event = events[i];
+    if (event.author === "user") {
+      let userQuery = "";
+      if (event.content && event.content.parts) {
+        for (const part of event.content.parts) {
+          if (part.text) {
+            userQuery = part.text;
+            break;
+          }
+        }
+      }
+      const expectedToolUse = [];
+      const expectedIntermediateAgentResponses = [];
+      let finalResponse = "";
+      let j = i + 1;
+      while (j < events.length) {
+        const subsequentEvent = events[j];
+        if (subsequentEvent.author === "user") {
+          break;
+        }
+        if (subsequentEvent.content && subsequentEvent.content.parts) {
+          let hasToolAction = false;
+          for (const part of subsequentEvent.content.parts) {
+            if (part.functionCall) {
+              expectedToolUse.push({
+                tool_name: (_a = part.functionCall.name) != null ? _a : "",
+                tool_input: (_b = part.functionCall.args) != null ? _b : {}
+              });
+              hasToolAction = true;
+            } else if (part.functionResponse) {
+              hasToolAction = true;
+            }
+          }
+          if (!hasToolAction) {
+            expectedIntermediateAgentResponses.push([
+              (_c = subsequentEvent.author) != null ? _c : "",
+              subsequentEvent.content.parts
+            ]);
+          }
+        }
+        j += 1;
+      }
+      if (expectedIntermediateAgentResponses.length > 0) {
+        const lastResponse = expectedIntermediateAgentResponses.pop();
+        if (lastResponse) {
+          const [, lastParts] = lastResponse;
+          finalResponse = lastParts.filter((part) => Boolean(part.text)).map((part) => {
+            var _a2;
+            return (_a2 = part.text) != null ? _a2 : "";
+          }).join("");
+        }
+      }
+      evalCases.push({
+        query: userQuery,
+        expected_tool_use: expectedToolUse,
+        expected_intermediate_agent_responses: expectedIntermediateAgentResponses,
+        reference: finalResponse
+      });
+      i = j;
+    } else {
+      i += 1;
+    }
+  }
+  return evalCases;
+}
+__name(convertSessionToEvalFormat, "convertSessionToEvalFormat");
+function populateEmptyState(agent, allState) {
+  for (const subAgent of agent.subAgents) {
+    populateEmptyState(subAgent, allState);
+  }
+  if (isLlmAgent(agent) && agent.instruction && typeof agent.instruction === "string") {
+    const matches = agent.instruction.matchAll(/\{([\w]+)\}/g);
+    for (const match of matches) {
+      allState[match[1]] = "";
+    }
+  }
+}
+__name(populateEmptyState, "populateEmptyState");
+function createEmptyState(agent, initializedStates) {
+  const subAgentsState = {};
+  populateEmptyState(agent, subAgentsState);
+  if (initializedStates) {
+    for (const [k, v] of Object.entries(initializedStates)) {
+      subAgentsState[k] = v;
+    }
+  }
+  return subAgentsState;
+}
+__name(createEmptyState, "createEmptyState");
+export {
+  convertSessionToEvalFormat,
+  createEmptyState
+};