@kortyx/cli 0.11.4 → 0.13.0

package/dist/index.js

@@ -23,15 +23,17 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_promises4 = require("fs/promises");
+var import_promises5 = require("fs/promises");
 var import_node_module = __toESM(require("module"));
 var import_node_os3 = require("os");
-var import_node_path4 = require("path");
-var import_agent = require("@kortyx/agent");
+var import_node_path6 = require("path");
+var import_agent2 = require("@kortyx/agent");
 var import_errors = require("@kortyx/core/errors");
 var import_telemetry_contracts6 = require("@kortyx/telemetry-contracts");
-var import_commander4 = require("commander");
+var import_commander6 = require("commander");
 var import_api = require("tsx/cjs/api");
+var import_api2 = require("tsx/esm/api");
+var import_typescript2 = __toESM(require("typescript"));
 
 // src/connections-command.ts
 var import_telemetry_contracts2 = require("@kortyx/telemetry-contracts");
@@ -87,7 +89,7 @@ var normalizeConnectionUrl = (input) => {
   }
   return url.toString().replace(/\/+$/, "");
 };
-var StudioReadClient = class {
+var StudioApiTransport = class {
   constructor(apiUrl, apiKey, request = fetch) {
     this.apiKey = apiKey;
     this.request = request;
@@ -106,27 +108,21 @@ var StudioReadClient = class {
     protocolVersion: null,
     release: null
   };
-  async get(path, schema, query = {}) {
-    if (!/^\/v1\/studio\/(?:context|catalogs|workflows|(?:runs|sessions|interrupts)(?:\/[^/?#]+)?)$/.test(
-      path
-    ) || /(?:^|\/)(?:\.|\.\.|%2e(?:%2e)?)(?:\/|$)/i.test(path)) {
-      throw new StudioReadError(
-        "invalid_endpoint",
-        "Only supported Studio read endpoints are allowed."
-      );
-    }
+  async requestJson(method, path, schema, query = {}, body) {
     const url = new URL(`${this.apiUrl}${path}`);
     for (const [key, value2] of Object.entries(query))
       url.searchParams.set(key, value2);
     let response;
     try {
       response = await this.request(url, {
-        method: "GET",
+        method,
         headers: {
           authorization: `Bearer ${this.apiKey}`,
           accept: "application/json",
-          "x-kortyx-studio-api-version": import_telemetry_contracts.STUDIO_API_PROTOCOL_VERSION
+          "x-kortyx-studio-api-version": import_telemetry_contracts.STUDIO_API_PROTOCOL_VERSION,
+          ...body === void 0 ? {} : { "content-type": "application/json" }
         },
+        ...body === void 0 ? {} : { body: JSON.stringify(body) },
         redirect: "error",
         signal: AbortSignal.timeout(15e3)
       });
@@ -152,7 +148,7 @@ var StudioReadClient = class {
     }
     if (!response.ok) {
       await response.body?.cancel();
-      const message = response.status === 401 ? "Invalid, expired, or revoked Studio read key." : response.status === 403 ? "The API key lacks studio:read permission." : response.status === 404 ? "Entity not found in this connection's project. Check the URL and selected connection." : `Studio API returned HTTP ${response.status}.`;
+      const message = response.status === 401 ? "Invalid, expired, or revoked Studio read key." : response.status === 403 ? "The API key lacks studio:read or the operation's required permission (eval:run for execution)." : response.status === 404 ? "Entity not found in this connection's project. Check the URL and selected connection." : `Studio API returned HTTP ${response.status}.`;
       throw new StudioReadError(
         "api_error",
         message,
@@ -169,7 +165,7 @@ var StudioReadClient = class {
       );
     try {
       const decoder = new TextDecoder();
-      let body = "";
+      let body2 = "";
       let bytes = 0;
       for (; ; ) {
         const chunk = await reader.read();
@@ -182,10 +178,10 @@ var StudioReadClient = class {
             "Studio response exceeded the 20 MiB safety limit. Use a smaller query or inspect an individual run."
           );
         }
-        body += decoder.decode(chunk.value, { stream: true });
+        body2 += decoder.decode(chunk.value, { stream: true });
       }
-      body += decoder.decode();
-      value = JSON.parse(body);
+      body2 += decoder.decode();
+      value = JSON.parse(body2);
     } catch (error) {
       if (error instanceof StudioReadError) throw error;
       throw new StudioReadError(
@@ -210,6 +206,19 @@ var StudioReadClient = class {
     return result.data;
   }
 };
+var StudioReadClient = class extends StudioApiTransport {
+  async get(path, schema, query = {}) {
+    if (!/^\/v1\/studio\/(?:context|catalogs|workflows|(?:runs|sessions|interrupts)(?:\/[^/?#]+)?)$/.test(
+      path
+    ) || /(?:^|\/)(?:\.|\.\.|%2e(?:%2e)?)(?:\/|$)/i.test(path)) {
+      throw new StudioReadError(
+        "invalid_endpoint",
+        "Only supported Studio read endpoints are allowed."
+      );
+    }
+    return this.requestJson("GET", path, schema, query);
+  }
+};
 
 // src/studio/state.ts
 var import_promises = require("fs/promises");
@@ -227,6 +236,7 @@ x-api-env: &api-env
 x-bootstrap-keys: &bootstrap-keys
   KORTYX_TELEMETRY_API_KEY: \${KORTYX_TELEMETRY_API_KEY}
   KORTYX_STUDIO_API_KEY: \${KORTYX_STUDIO_API_KEY}
+  KORTYX_STUDIO_ENABLE_EVALS: \${KORTYX_STUDIO_ENABLE_EVALS:-0}
   KORTYX_STUDIO_ENABLE_REVIEWS: \${KORTYX_STUDIO_ENABLE_REVIEWS:-0}
 
 services:
@@ -247,6 +257,7 @@ services:
   db-init:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
+    user: "1000:1000"
     environment:
       <<: [*api-env, *bootstrap-keys]
     command: >
@@ -260,11 +271,18 @@ services:
   api:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
+    user: "1000:1000"
     environment:
       <<: *api-env
       NODE_ENV: production
       API_HOST: 0.0.0.0
       API_PORT: 6400
+      KORTYX_EVAL_JUDGE_MODEL: \${KORTYX_EVAL_JUDGE_MODEL:-}
+      KORTYX_EVAL_JUDGE_API_KEY: \${KORTYX_EVAL_JUDGE_API_KEY:-}
+      KORTYX_EVAL_JUDGE_API: \${KORTYX_EVAL_JUDGE_API:-responses}
+      KORTYX_EVAL_JUDGE_BASE_URL: \${KORTYX_EVAL_JUDGE_BASE_URL:-}
+      KORTYX_EVAL_JUDGE_ID: \${KORTYX_EVAL_JUDGE_ID:-}
+      KORTYX_EVAL_JUDGE_VERSION: \${KORTYX_EVAL_JUDGE_VERSION:-}
       KORTYX_STUDIO_RELEASE: \${KORTYX_STUDIO_IMAGE_TAG:-latest}
     ports:
       - "127.0.0.1:\${API_PORT:-6400}:6400"
@@ -313,6 +331,9 @@ services:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
     restart: unless-stopped
+    # The updater alone needs root for the host Docker socket and ownership-safe
+    # writes to the bind-mounted Studio state directory.
+    user: "0:0"
     command: ["node", "apps/api/dist/updater.js", "serve", "\${KORTYX_STUDIO_STATE_DIR}"]
     volumes:
       - type: bind
@@ -816,7 +837,7 @@ var createConnectionsCommand = (log = console.log, request = fetch) => {
         "/v1/studio/context",
         import_telemetry_contracts2.StudioContextResponseSchema
       );
-      if (!context.apiKey.scopes.includes("studio:read"))
+      if (!context.apiKey?.scopes.includes("studio:read"))
         throw new StudioReadError(
           "missing_scope",
           "The API key lacks studio:read permission."
@@ -837,49 +858,1215 @@ var createConnectionsCommand = (log = console.log, request = fetch) => {
       await saveConnections(config, options.configHome);
       print({ connection: profile, verified: true }, options);
     }
-  );
-  common(
-    command.command("use <name>").description(
-      "Select the default connection; agents should prefer per-command --connection."
-    )
-  ).action(async (name, options) => {
-    const config = await readConnections(options.configHome);
-    if (name === "local" ? !await readStudioConfig(options.home) : !config.profiles.some((profile) => profile.name === name)) {
+  );
+  common(
+    command.command("use <name>").description(
+      "Select the default connection; agents should prefer per-command --connection."
+    )
+  ).action(async (name, options) => {
+    const config = await readConnections(options.configHome);
+    if (name === "local" ? !await readStudioConfig(options.home) : !config.profiles.some((profile) => profile.name === name)) {
+      throw new StudioReadError(
+        "unknown_connection",
+        "Connection not found. Run kortyx connections list."
+      );
+    }
+    config.current = name;
+    await saveConnections(config, options.configHome);
+    print({ current: name }, options);
+  });
+  common(
+    command.command("remove <name>").description(
+      "Remove a saved locator, not its credential or remote project."
+    )
+  ).action(async (name, options) => {
+    const config = await readConnections(options.configHome);
+    if (!config.profiles.some((profile) => profile.name === name))
+      throw new StudioReadError(
+        "unknown_connection",
+        "Saved connection not found; managed local cannot be removed."
+      );
+    config.profiles = config.profiles.filter(
+      (profile) => profile.name !== name
+    );
+    if (config.current === name) delete config.current;
+    await saveConnections(config, options.configHome);
+    print({ removed: name }, options);
+  });
+  return command;
+};
+
+// src/evals/command.ts
+var import_node_path3 = require("path");
+var import_agent = require("@kortyx/agent");
+var import_commander2 = require("commander");
+
+// src/evals/reporter.ts
+var text = (value) => value.replace(/[\p{Cc}\p{Cf}]/gu, " ").trim();
+var duration = (ms) => ms < 1e3 ? `${Math.round(ms)} ms` : `${(ms / 1e3).toFixed(2)} s`;
+var wide = /[\u1100-\u115f\u2e80-\ua4cf\uac00-\ud7a3\uf900-\ufaff\ufe10-\ufe6f\uff01-\uff60\uffe0-\uffe6]|\p{Extended_Pictographic}/u;
+var fit = (value, columns) => {
+  const clean = value.replace(/[\p{Cc}\p{Cf}]/gu, " ").trimEnd();
+  let used = 0;
+  let result = "";
+  for (const char of clean) {
+    used += /\p{Mark}/u.test(char) ? 0 : wide.test(char) ? 2 : 1;
+    if (used > columns - 2) return `${result}\u2026`;
+    result += char;
+  }
+  return result;
+};
+var frames = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
+var phaseNames = {
+  params: "Preparing inputs",
+  setup: "Setting up test",
+  execute: "Running workflow",
+  responder: "Resolving human input",
+  reference: "Loading reference",
+  grading: "Checking expectations",
+  cleanup: "Cleaning up",
+  reporting: "Reporting results"
+};
+function createEvalTerminalReporter(write, color, options = {}) {
+  const now = options.now ?? Date.now;
+  const columns = () => Math.max(12, options.columns?.() ?? process.stdout.columns ?? 100);
+  const paint = (code, value) => color ? `\x1B[${code}m${value}\x1B[0m` : value;
+  const active = /* @__PURE__ */ new Map();
+  let names = /* @__PURE__ */ new Map();
+  let steps = /* @__PURE__ */ new Map();
+  let completed = 0;
+  let total = 0;
+  let started = 0;
+  let frame = 0;
+  let liveLines = 0;
+  let paused = false;
+  let running = false;
+  let timer;
+  const key = (id, repetition) => JSON.stringify([id, repetition]);
+  const clear = () => {
+    if (liveLines) write(`\x1B[${liveLines}A\r\x1B[0J`);
+    liveLines = 0;
+  };
+  const line = (value = "") => {
+    clear();
+    write(`${value}
+`);
+  };
+  const progressBar = (done, count, width, pulse = false) => {
+    const filled = count ? Math.min(width, Math.floor(done / count * width)) : 0;
+    const remaining = width - filled;
+    const pending = pulse && remaining > 0 ? Array.from(
+      { length: remaining },
+      (_, index) => index === frame % remaining ? "\u2578" : "\u2591"
+    ).join("") : "\u2591".repeat(remaining);
+    return `${"\u2501".repeat(filled)}${pending}`;
+  };
+  const render = () => {
+    if (!color || !running || paused) return;
+    clear();
+    const width = columns();
+    const barWidth = Math.max(6, Math.min(22, width - 60));
+    const percent = total ? Math.floor(completed / total * 100) : 0;
+    const lines = [
+      paint(
+        36,
+        fit(
+          `  RUN ${progressBar(completed, total, barWidth)} ${completed}/${total || "?"} \xB7 ${percent}%`,
+          width
+        )
+      ),
+      paint(
+        90,
+        fit(
+          `  ${active.size} running \xB7 ${Math.max(0, total - completed - active.size)} queued \xB7 ${((now() - started) / 1e3).toFixed(1)}s elapsed`,
+          width
+        )
+      )
+    ];
+    for (const item of [...active.values()].slice(0, 6)) {
+      const name = names.get(item.caseId) ?? item.caseId;
+      lines.push(
+        paint(
+          33,
+          fit(
+            `  ${frames[frame % frames.length]} ${name} #${item.repetition}`,
+            width
+          )
+        )
+      );
+      const stepCount = steps.get(item.caseId) ?? 1;
+      const step = item.stepIndex === void 0 ? "" : `Step ${item.stepIndex + 1}/${stepCount} \xB7 `;
+      lines.push(
+        paint(
+          item.phase === "grading" ? 35 : 90,
+          fit(
+            `    ${progressBar(item.completedSteps, stepCount, 6, true)} ${step}${phaseNames[item.phase]} \xB7 ${((now() - item.started) / 1e3).toFixed(1)}s`,
+            width
+          )
+        )
+      );
+    }
+    if (active.size > 6)
+      lines.push(paint(90, `  +${active.size - 6} more running`));
+    write(`${lines.join("\n")}
+`);
+    liveLines = lines.length;
+    frame++;
+  };
+  const stop = () => {
+    if (timer) clearInterval(timer);
+    timer = void 0;
+    running = false;
+    clear();
+  };
+  return {
+    start(suite) {
+      stop();
+      names = new Map(
+        suite.cases.map((item) => [item.id, item.name ?? item.id])
+      );
+      steps = new Map(suite.cases.map((item) => [item.id, item.steps.length]));
+      active.clear();
+      completed = total = frame = 0;
+      paused = false;
+      started = now();
+      line();
+      if (color)
+        line(paint(35, "  \u2501".padEnd(Math.min(columns() - 1, 64), "\u2501")));
+      line(paint("1;35", `\u25C6 Kortyx Evals \xB7 ${text(suite.name ?? suite.id)}`));
+      line(paint(90, `  ${text(suite.id)} \xB7 local execution`));
+      line();
+      running = true;
+      if (color) {
+        timer = setInterval(render, 100);
+        timer.unref();
+      }
+    },
+    progress(event) {
+      if (event.type === "run-started") {
+        total = event.caseIds.length * event.repetitions;
+        line(
+          paint(
+            90,
+            `  ${event.caseIds.length} ${event.caseIds.length === 1 ? "test" : "tests"} \xB7 ${total} ${total === 1 ? "attempt" : "attempts"} \xB7 concurrency ${event.concurrency}`
+          )
+        );
+        line();
+      } else if (event.type === "case-started") {
+        active.set(key(event.caseId, event.repetition), {
+          caseId: event.caseId,
+          repetition: event.repetition,
+          started: now(),
+          phase: "params",
+          completedSteps: 0
+        });
+        if (!color)
+          line(
+            `  \u2192 ${text(names.get(event.caseId) ?? event.caseId)} [${text(event.caseId)}] #${event.repetition} RUNNING`
+          );
+      } else if (event.type === "case-progress") {
+        const item = active.get(key(event.caseId, event.repetition));
+        if (item) {
+          item.phase = event.phase;
+          item.stepIndex = event.stepIndex;
+        }
+      } else if (event.type === "step-completed") {
+        const item = active.get(key(event.caseId, event.repetition));
+        if (item) item.completedSteps = event.step.index + 1;
+        const passed = event.step.status === "passed";
+        const criterionCount = event.step.criteria.length;
+        const detail = criterionCount ? ` \xB7 ${event.step.criteria.filter((criterion) => criterion.passed).length}/${criterionCount} criteria passed` : "";
+        line(
+          `    ${paint(passed ? 32 : 33, passed ? "\u2713" : "\u25C7")} ${text(event.caseId)} #${event.repetition} \xB7 Step ${event.step.index + 1}/${steps.get(event.caseId) ?? "?"} \xB7 ${event.step.observation.type === "interrupt" ? "Human input requested" : text(event.step.observation.type)}${detail}`
+        );
+      } else {
+        const item = event.result;
+        active.delete(key(item.caseId, item.repetition));
+        completed++;
+        const passed = item.status === "passed";
+        const cancelled = item.status === "cancelled";
+        const code = passed ? 32 : cancelled || item.status === "ungraded" ? 33 : 31;
+        const symbol = passed ? "\u2713" : cancelled ? "\u2212" : item.status === "ungraded" ? "\u25C7" : "\u2717";
+        line(
+          `  ${paint(code, symbol)} ${text(names.get(item.caseId) ?? item.caseId)} ${paint(90, `[${text(item.caseId)}] #${item.repetition} \xB7 ${duration(item.durationMs)}`)} ${paint(`${code};1`, item.status.toUpperCase())}`
+        );
+        line();
+      }
+      render();
+    },
+    finish(result) {
+      stop();
+      for (const item of result.cases) {
+        for (const step of item.steps) {
+          if (step.status === "passed") continue;
+          line();
+          line(
+            paint(
+              31,
+              `  ${text(names.get(item.caseId) ?? item.caseId)} \u203A attempt ${item.repetition} \u203A step ${step.index + 1}`
+            )
+          );
+          line(
+            `    Expected ${step.expectation.type} \xB7 observed ${step.observation.type}`
+          );
+          if (step.reason) line(`    ${text(step.reason)}`);
+          for (const criterion of step.criteria.filter(
+            (value) => !value.passed
+          )) {
+            line(`    \u2717 ${text(criterion.text)}`);
+            line(`      ${text(criterion.reason)}`);
+            for (const evidence of criterion.evidence)
+              line(paint(90, `      Evidence: ${text(evidence)}`));
+          }
+        }
+        for (const issue of item.errors)
+          line(
+            `    ${text(item.caseId)} \xB7 ${issue.phase} [${issue.code}] ${text(issue.message)}`
+          );
+      }
+      for (const issue of result.errors)
+        line(`    ${issue.phase} [${issue.code}] ${text(issue.message)}`);
+      const successful = result.status === "passed";
+      const code = successful ? 32 : result.status === "cancelled" || result.status === "ungraded" ? 33 : 31;
+      line();
+      if (color) {
+        line(
+          paint(
+            code,
+            `  ${progressBar(result.cases.length, result.cases.length, Math.min(22, columns() - 3))} ${result.cases.length}/${result.cases.length}`
+          )
+        );
+        line(
+          paint(
+            `${code};1`,
+            `  ${successful ? "\u2713 SUITE PASSED" : `\u25C7 SUITE ${result.status.toUpperCase()}`}`
+          )
+        );
+      }
+      line(
+        `  ${paint(result.counts.passed ? 32 : 90, `${result.counts.passed} passed`)} \xB7 ${paint(result.counts.failed ? 31 : 90, `${result.counts.failed} failed`)} \xB7 ${paint(result.counts.error ? 31 : 90, `${result.counts.error} errors`)} \xB7 ${paint(result.counts.cancelled ? 33 : 90, `${result.counts.cancelled} cancelled`)}${result.counts.ungraded ? ` \xB7 ${paint(33, `${result.counts.ungraded} ungraded`)}` : ""}`
+      );
+      line(
+        paint(
+          90,
+          `  Duration ${duration(result.durationMs)} \xB7 ${result.cases.length} ${result.cases.length === 1 ? "attempt" : "attempts"}`
+        )
+      );
+      line();
+    },
+    pause() {
+      paused = true;
+      clear();
+    },
+    resume() {
+      paused = false;
+      render();
+    },
+    close: stop
+  };
+}
+
+// src/evals/command.ts
+var integer = (value) => {
+  if (!/^\d+$/.test(value) || !Number.isSafeInteger(Number(value)) || Number(value) < 1)
+    throw new import_commander2.InvalidArgumentError("Use a positive integer.");
+  return Number(value);
+};
+var isRunner = (value) => value !== null && typeof value === "object" && "run" in value && typeof value.run === "function" && "describe" in value && typeof value.describe === "function";
+function createLocalEvalsCommand(runtime) {
+  const write = runtime.write ?? process.stdout.write.bind(process.stdout);
+  const exitCode = runtime.setExitCode ?? ((value) => {
+    process.exitCode = value;
+  });
+  const color = runtime.color ?? Boolean(
+    process.stdout.isTTY && process.env.NO_COLOR === void 0 && process.env.FORCE_COLOR !== "0"
+  );
+  const load = async (options) => {
+    const module2 = await runtime.load(
+      (0, import_node_path3.resolve)(runtime.cwd ?? process.cwd(), options.entry),
+      options.env
+    );
+    const selected = options.export ? module2[options.export] : module2.evals ?? module2.default;
+    if (!isRunner(selected))
+      throw new Error(
+        `Export ${options.export ?? '"evals" or default'} must be a createEvals instance.`
+      );
+    const suites = selected.describe().suites.map((suite) => (0, import_agent.parseEvalSuite)(suite));
+    if (!suites.length) throw new Error("No eval suites are configured.");
+    return { runner: selected, suites };
+  };
+  const withRunner = async (options, action) => {
+    try {
+      await action(await load(options));
+    } finally {
+      await runtime.release?.();
+    }
+  };
+  const entryOptions = (command) => command.requiredOption(
+    "--entry <path>",
+    "Module exporting your createEvals instance (TypeScript or JavaScript)."
+  ).option("--export <name>", "Named export; defaults to evals or default.").option(
+    "--env <path>",
+    "Load an env file before the entry; repeat for overlays. Replaces automatic env loading.",
+    (value, previous) => [...previous, value],
+    []
+  ).option(
+    "--json",
+    "Print machine-readable JSON instead of the terminal report."
+  );
+  const root = new import_commander2.Command("evals").description(
+    "Run application evals locally, without Studio."
+  );
+  entryOptions(
+    root.command("list").description("List suites exported by an application module.")
+  ).action(
+    (options) => withRunner(options, async ({ suites }) => {
+      write(
+        options.json ? `${JSON.stringify(suites)}
+` : suites.map(
+          (suite) => `${suite.id}	${suite.name ?? suite.id}	${suite.cases.length} cases
+`
+        ).join("")
+      );
+    })
+  );
+  entryOptions(
+    root.command("run").description(
+      "Execute suites and report progress, failures and results locally."
+    )
+  ).option(
+    "--suite <id>",
+    "Run one suite; omitted runs all suites in definition order."
+  ).option(
+    "--case <id...>",
+    "Select cases within --suite (space-separated IDs)."
+  ).option("--repetitions <count>", "Override attempts per case.", integer).option(
+    "--concurrency <count>",
+    "Override concurrent cases within each suite.",
+    integer
+  ).option("--no-color", "Disable terminal colors and live progress.").option("--color", "Force terminal colors and live progress.").action(
+    (options, command) => withRunner(options, async ({ runner, suites }) => {
+      const selected = options.suite ? suites.filter((suite) => suite.id === options.suite) : suites;
+      if (!selected.length)
+        throw new Error(`Unknown eval suite: ${options.suite}`);
+      if (options.case && (!options.suite || new Set(options.case).size !== options.case.length || options.case.some(
+        (id) => !selected[0]?.cases.some((item) => item.id === id)
+      )))
+        throw new Error(
+          "--case requires --suite and unique existing case IDs."
+        );
+      const controller = new AbortController();
+      const removeInterrupt = (runtime.onInterrupt ?? ((abort) => {
+        process.once("SIGINT", abort);
+        return () => {
+          process.removeListener("SIGINT", abort);
+        };
+      }))(() => controller.abort());
+      const interactive = options.color && !options.json && (color || command.getOptionValueSource("color") === "cli");
+      const reporter = createEvalTerminalReporter(write, interactive);
+      const restoreOutput = [];
+      if (!runtime.write && interactive) {
+        for (const stream of [process.stdout, process.stderr]) {
+          const original = stream.write;
+          const wrapped = new Proxy(original, {
+            apply(target, receiver, args) {
+              reporter.pause();
+              const written = Reflect.apply(target, receiver, args);
+              if (String(args[0]).endsWith("\n")) reporter.resume();
+              return written;
+            }
+          });
+          stream.write = wrapped;
+          restoreOutput.push(() => {
+            if (stream.write === wrapped) stream.write = original;
+          });
+        }
+      }
+      const runs = [];
+      try {
+        for (const suite of selected) {
+          if (controller.signal.aborted) break;
+          if (!options.json) reporter.start(suite);
+          const result = await runner.run({
+            suiteId: suite.id,
+            signal: controller.signal,
+            includeActivity: !options.json,
+            ...options.case ? { caseIds: options.case } : {},
+            ...options.repetitions !== void 0 ? { repetitions: options.repetitions } : {},
+            ...options.concurrency !== void 0 ? { concurrency: options.concurrency } : {},
+            ...!options.json ? { onProgress: reporter.progress } : {}
+          });
+          runs.push(result);
+          if (!options.json) reporter.finish(result);
+        }
+        const counts = { passed: 0, failed: 0, error: 0, cancelled: 0 };
+        for (const run2 of runs)
+          for (const key of Object.keys(counts))
+            counts[key] += run2.counts[key];
+        const cancelled = controller.signal.aborted || counts.cancelled > 0;
+        const passed = runs.length === selected.length && runs.every((run2) => run2.status === "passed");
+        const status = cancelled ? "cancelled" : counts.error || runs.some((run2) => run2.status === "error") ? "error" : passed ? "passed" : "failed";
+        if (options.json)
+          write(
+            `${JSON.stringify({ schemaVersion: 1, status, counts, runs })}
+`
+          );
+        else if (selected.length > 1)
+          write(
+            `Overall ${status.toUpperCase()} \xB7 ${runs.length}/${selected.length} suites \xB7 ${counts.passed} passed \xB7 ${counts.failed} failed \xB7 ${counts.error} errors
+`
+          );
+        exitCode(cancelled ? 130 : passed ? 0 : 1);
+      } finally {
+        reporter.close();
+        for (const restore of restoreOutput) restore();
+        removeInterrupt();
+      }
+    })
+  );
+  return root;
+}
+
+// src/evals/environment.ts
+var import_promises3 = require("fs/promises");
+var import_node_path4 = require("path");
+var import_node_util = require("util");
+var exists = async (file) => {
+  try {
+    await (0, import_promises3.lstat)(file);
+    return true;
+  } catch (error) {
+    if (error.code === "ENOENT") return false;
+    throw new Error(`Cannot read eval configuration: ${file}`);
+  }
+};
+var findConfiguration = async (cwd) => {
+  let directory = (0, import_node_path4.resolve)(cwd);
+  const directories = [];
+  for (; ; ) {
+    directories.unshift(directory);
+    const profile = (0, import_node_path4.join)(directory, ".env.evals.json");
+    if (await exists(profile)) return { profile, directories };
+    if (await exists((0, import_node_path4.join)(directory, ".git")) || await exists((0, import_node_path4.join)(directory, "pnpm-workspace.yaml")))
+      return { directories };
+    if ((0, import_node_path4.dirname)(directory) === directory)
+      return { directories: [(0, import_node_path4.resolve)(cwd)] };
+    directory = (0, import_node_path4.dirname)(directory);
+  }
+};
+var read = async (file, privateFile) => {
+  let stat;
+  try {
+    stat = await (0, import_promises3.lstat)(file);
+  } catch {
+    throw new Error(`Configured eval environment file is unavailable: ${file}`);
+  }
+  if (!stat.isFile() || privateFile && process.platform !== "win32" && ((stat.mode & 63) !== 0 || typeof process.getuid === "function" && stat.uid !== process.getuid()))
+    throw new Error(
+      `Eval configuration must be ${privateFile ? "an owner-only" : "a"} regular file: ${file}`
+    );
+  try {
+    return await (0, import_promises3.readFile)(file, "utf8");
+  } catch {
+    throw new Error(`Cannot read eval configuration: ${file}`);
+  }
+};
+async function loadEvalEnvironment(cwd, envFiles, shell = process.env) {
+  let files;
+  let privateFiles = false;
+  if (envFiles?.length) {
+    files = envFiles.map((file) => (0, import_node_path4.resolve)(cwd, file));
+  } else {
+    const { profile, directories } = await findConfiguration(cwd);
+    if (profile) {
+      let value;
+      try {
+        value = JSON.parse(await read(profile, true));
+      } catch (error) {
+        if (error instanceof SyntaxError)
+          throw new Error(`Invalid eval environment profile: ${profile}`);
+        throw error;
+      }
+      if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some((key) => key !== "envFiles") || !("envFiles" in value) || !Array.isArray(value.envFiles) || !value.envFiles.length || value.envFiles.some(
+        (file) => typeof file !== "string" || !file.trim()
+      ))
+        throw new Error(`Invalid eval environment profile: ${profile}`);
+      files = value.envFiles.map(
+        (file) => (0, import_node_path4.resolve)((0, import_node_path4.dirname)(profile), file)
+      );
+      privateFiles = true;
+    } else {
+      files = [];
+      for (const directory of directories)
+        for (const name of [".env", ".env.local"]) {
+          const file = (0, import_node_path4.resolve)(directory, name);
+          if (await exists(file)) files.push(file);
+        }
+    }
+  }
+  const environment = {};
+  for (const file of files)
+    Object.assign(environment, (0, import_node_util.parseEnv)(await read(file, privateFiles)));
+  return { ...environment, ...shell };
+}
+
+// src/studio/command.ts
+var import_commander5 = require("commander");
+
+// src/studio/eval-command.ts
+var import_commander3 = require("commander");
+
+// src/studio/eval-client.ts
+var import_evals = require("@kortyx/agent/evals");
+var import_zod3 = require("zod");
+var StartRequestSchema = import_evals.StudioEvalStartRequestSchema;
+var parseEvalRunTarget = (input) => {
+  let id = input;
+  let url;
+  if (/^[a-z][a-z0-9+.-]*:/i.test(input)) {
+    let locator;
+    try {
+      locator = new URL(input);
+    } catch {
+      throw new StudioReadError("invalid_target", "Invalid eval run URL.");
+    }
+    const match = locator.pathname.match(/\/evals\/runs\/([^/]+)\/?$/);
+    if (!match || !["http:", "https:"].includes(locator.protocol) || locator.username || locator.password)
+      throw new StudioReadError(
+        "invalid_target",
+        "Expected an HTTP(S) Studio /evals/runs/<uuid> URL without credentials."
+      );
+    id = match[1] ?? "";
+    locator.search = "";
+    locator.hash = "";
+    url = locator.toString();
+  }
+  if (!import_zod3.z.uuid().safeParse(id).success)
+    throw new StudioReadError("invalid_target", "Expected an eval run UUID.");
+  return { id, url };
+};
+var StudioEvalClient = class extends StudioApiTransport {
+  targets() {
+    return this.requestJson(
+      "GET",
+      "/v1/studio/evals/targets",
+      import_evals.StudioEvalTargetsResponseSchema
+    );
+  }
+  runs() {
+    return this.requestJson(
+      "GET",
+      "/v1/studio/evals/runs",
+      import_evals.StudioEvalHistorySchema
+    );
+  }
+  run(id) {
+    if (!import_zod3.z.uuid().safeParse(id).success)
+      throw new StudioReadError("invalid_target", "Expected an eval run UUID.");
+    return this.requestJson(
+      "GET",
+      `/v1/studio/evals/runs/${id}`,
+      import_evals.StudioEvalDetailSchema
+    );
+  }
+  start(input) {
+    const parsed = StartRequestSchema.safeParse(input);
+    if (!parsed.success)
+      throw new StudioReadError(
+        "invalid_eval_request",
+        "Invalid suite revision, case selection, or execution limits."
+      );
+    return this.requestJson(
+      "POST",
+      "/v1/studio/evals/runs",
+      import_zod3.z.object({ id: import_zod3.z.uuid() }),
+      {},
+      parsed.data
+    );
+  }
+  cancel(id) {
+    if (!import_zod3.z.uuid().safeParse(id).success)
+      throw new StudioReadError("invalid_target", "Expected an eval run UUID.");
+    return this.requestJson(
+      "POST",
+      `/v1/studio/evals/runs/${id}/cancel`,
+      import_zod3.z.object({ ok: import_zod3.z.literal(true) })
+    );
+  }
+};
+
+// src/studio/eval-doctor.ts
+var advice = {
+  environment_forbidden: {
+    message: "Target environment is not allowed in this Studio project.",
+    remedy: "Allow the target environment in the project, or correct the target's environment label."
+  },
+  environment_unavailable: {
+    message: "Studio could not check the target environment.",
+    remedy: "Inspect Studio API database connectivity and private server logs."
+  },
+  endpoint_not_found: {
+    message: "Application endpoint returned HTTP 404.",
+    remedy: "Ensure the eval route is mounted and enabled in the deployed application; check the target URL and reverse proxy route."
+  },
+  endpoint_unauthorized: {
+    message: "Application endpoint rejected discovery credentials.",
+    remedy: "Match the consumer handler service key to the Studio target. If the app authenticates a test actor during discovery, also check that actor's credentials and permissions."
+  },
+  endpoint_http_error: {
+    message: "Application endpoint returned an unsuccessful HTTP status.",
+    remedy: "Inspect the consumer and proxy logs. Check deployment configuration and any app-owned test-identity initialization."
+  },
+  endpoint_unreachable: {
+    message: "Studio API could not fetch the application manifest.",
+    remedy: "Check API-to-consumer networking, DNS, TLS and timeouts. From Docker Desktop use host.docker.internal for a host app; redirects are not followed."
+  },
+  manifest_invalid: {
+    message: "Application returned an empty, oversized or incompatible manifest.",
+    remedy: "Mount createEvalRouteHandler on the exact target URL and use compatible SDK/Studio releases. Check whether a proxy returned HTML instead of JSON."
+  }
+};
+function buildEvalDoctorReport(data, options) {
+  const checks = [
+    {
+      id: "studio_access",
+      status: "passed",
+      message: "Authenticated Studio discovery (studio:read)."
+    },
+    {
+      id: "execution_permission",
+      status: data.canRun ? "passed" : "failed",
+      message: data.canRun ? "Studio key has eval:run." : "Studio key lacks eval:run.",
+      ...!data.canRun ? {
+        remedy: "Grant eval:run to this project key. For local bootstrap rerun with KORTYX_STUDIO_ENABLE_EVALS=1 and the existing stored key."
+      } : {}
+    }
+  ];
+  const targets = data.targets.filter(
+    (target) => (!options.target || target.id === options.target) && (!options.environment || target.environment === options.environment)
+  );
+  checks.push({
+    id: "target_selection",
+    status: targets.length ? "passed" : "failed",
+    message: targets.length ? `${targets.length} matching application target(s).` : "No application target matches this connection and selection.",
+    ...!targets.length ? {
+      remedy: "Register the target with this key's organization/project and environment. Mount KORTYX_EVAL_TARGETS_FILE on the Studio API, restart it, and check --target/--environment and the connection's default environment."
+    } : {}
+  });
+  for (const target of targets) {
+    const prefix = `${target.id} (${target.environment})`;
+    const diagnostic = target.diagnostic;
+    const environmentFailed = diagnostic?.code === "environment_forbidden" || diagnostic?.code === "environment_unavailable";
+    checks.push({
+      id: `${target.id}:environment`,
+      status: environmentFailed ? "failed" : target.manifest || diagnostic ? "passed" : "skipped",
+      message: `${prefix}: ${environmentFailed ? advice[diagnostic.code].message : target.manifest || diagnostic ? "target environment allowed." : "environment check unavailable on this API."}`,
+      ...environmentFailed ? { remedy: advice[diagnostic.code].remedy } : {}
+    });
+    if (!target.manifest) {
+      const guidance = diagnostic && !environmentFailed ? advice[diagnostic.code] : void 0;
+      checks.push({
+        id: `${target.id}:manifest`,
+        status: environmentFailed ? "skipped" : "failed",
+        ...diagnostic ? { code: diagnostic.code } : {},
+        message: `${prefix}: ${environmentFailed ? "discovery skipped because the environment check failed." : guidance?.message ?? "consumer unavailable; this API did not supply a failure category."}${diagnostic?.httpStatus ? ` (HTTP ${diagnostic.httpStatus})` : ""}`,
+        ...!environmentFailed ? {
+          remedy: guidance?.remedy ?? "Check the deployed consumer route, matching service key and API-to-consumer reachability. Upgrade the Studio API for detailed discovery diagnostics."
+        } : {}
+      });
+      checks.push({
+        id: `${target.id}:judge`,
+        status: "skipped",
+        message: `${prefix}: judge compatibility cannot be checked without a manifest.`
+      });
+      continue;
+    }
+    const suites = target.manifest.suites;
+    const selectedSuite = options.suite ? suites.find((suite) => suite.id === options.suite) : void 0;
+    const suitesReady = options.suite ? Boolean(selectedSuite) : suites.length > 0;
+    checks.push({
+      id: `${target.id}:manifest`,
+      status: suitesReady ? "passed" : "failed",
+      message: `${prefix}: authenticated manifest valid; ${suites.length} registered suite(s).${options.suite ? ` Requested suite ${options.suite} ${selectedSuite ? "found" : "missing"}.` : ""}`,
+      ...!suitesReady ? {
+        remedy: "Pass the intended suite to createEvals({ suites: [...] }), deploy the application and refresh discovery. Suites are fetched from the consumer; topology publication does not register them."
+      } : {}
+    });
+    const judgeReady = options.judge === "studio" ? Boolean(data.studioJudge && target.manifest.studioJudging) : Boolean(target.manifest.judge);
+    checks.push({
+      id: `${target.id}:judge`,
+      status: judgeReady ? "passed" : "failed",
+      message: `${prefix}: ${options.judge} judge ${judgeReady ? "configured and advertised" : "unavailable"}.`,
+      ...!judgeReady ? {
+        remedy: options.judge === "app" ? "Provide a code judge to createEvals in the consumer, or explicitly select --judge studio." : !data.studioJudge ? "Configure KORTYX_EVAL_JUDGE_MODEL and KORTYX_EVAL_JUDGE_API_KEY on the Studio API and restart it, or explicitly select an available --judge app." : "Update the consumer SDK to advertise Studio judging support, or select an available --judge app."
+      } : {}
+    });
+  }
+  return {
+    schemaVersion: 1,
+    status: checks.some((check) => check.status === "failed") ? "failed" : "passed",
+    checks
+  };
+}
+function evalDoctorFailure(error) {
+  const safe = error instanceof StudioReadError ? error : null;
+  const connectionError = safe && [
+    "invalid_config",
+    "invalid_connection",
+    "not_configured",
+    "unknown_connection",
+    "invalid_key_env",
+    "missing_key",
+    "invalid_key",
+    "invalid_url",
+    "incompatible_studio_api",
+    "schema_mismatch",
+    "connection_failed"
+  ].includes(safe.code);
+  return {
+    schemaVersion: 1,
+    status: "failed",
+    checks: [
+      {
+        id: "studio_access",
+        status: "failed",
+        ...safe ? { code: safe.code } : {},
+        message: connectionError ? safe.message : safe?.status === 401 ? "Studio rejected the project key (HTTP 401)." : safe?.status === 403 ? "Studio discovery requires studio:read (HTTP 403)." : safe?.status === 404 ? "Studio eval discovery endpoint was not found (HTTP 404)." : "Could not resolve the Studio connection or read a compatible discovery response.",
+        remedy: "Check the selected connection, its API URL and project key, network access, and compatible CLI/Studio releases. No consumer or provider credentials belong in this CLI connection."
+      }
+    ]
+  };
+}
+function formatEvalDoctorReport(report) {
+  const symbols = { passed: "\u2713", failed: "\u2717", skipped: "\u2013" };
+  return [
+    "Kortyx Evals \xB7 Setup check",
+    ...report.checks.flatMap((check) => [
+      `  ${symbols[check.status]} ${check.message}`,
+      ...check.remedy ? [`    \u2192 ${check.remedy}`] : []
+    ]),
+    "",
+    report.status === "passed" ? "Configuration checks passed. Run one representative suite to verify the test identity, tools, model access and saved results." : "Setup checks failed. Resolve the failures and run doctor again.",
+    "No workflow or judge calls were started. Consumer GET discovery may run app-owned authentication logic."
+  ].join("\n").replace(/\p{Cc}/gu, (character) => character === "\n" ? character : "");
+}
+
+// src/studio/read-output.ts
+var parseStudioTarget = (input, entity) => {
+  if (/^[a-z][a-z0-9+.-]*:/i.test(input)) {
+    let url;
+    try {
+      url = new URL(input);
+    } catch {
+      throw new StudioReadError("invalid_target", "Invalid Studio URL.");
+    }
+    if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) {
+      throw new StudioReadError(
+        "invalid_target",
+        "Use an HTTP(S) Studio entity URL without embedded credentials."
+      );
+    }
+    const match = url.pathname.match(
+      /\/(runs|sessions|interrupts)\/([^/]+)\/?$/
+    );
+    if (!match)
+      throw new StudioReadError(
+        "invalid_target",
+        "Expected a Studio /runs/<id>, /sessions/<id>, or /interrupts/<id> URL."
+      );
+    if (entity && match[1] !== entity)
+      throw new StudioReadError(
+        "invalid_target",
+        "URL entity type does not match the command."
+      );
+    let id;
+    try {
+      id = decodeURIComponent(match[2] ?? "");
+    } catch {
+      throw new StudioReadError(
+        "invalid_target",
+        "Invalid entity ID encoding."
+      );
+    }
+    validateId(id);
+    const selection = {};
+    for (const key of [
+      "tab",
+      "sessionTab",
+      "call",
+      "branch",
+      "node",
+      "event",
+      "trace",
+      "detailView"
+    ]) {
+      const value = url.searchParams.get(key);
+      if (value) selection[key] = value;
+    }
+    url.search = "";
+    url.hash = "";
+    return {
+      entity: match[1],
+      id,
+      url: url.toString(),
+      selection
+    };
+  }
+  if (!entity)
+    throw new StudioReadError(
+      "invalid_target",
+      "inspect requires a Studio entity URL. Use runs/sessions/interrupts get for a bare ID."
+    );
+  validateId(input);
+  return { entity, id: input };
+};
+var validateId = (id) => {
+  if (!id || id.length > 512 || id === "." || id === ".." || /[\s/?#\\]/.test(id) || [...id].some(
+    (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127
+  )) {
+    throw new StudioReadError("invalid_target", "Invalid Studio entity ID.");
+  }
+};
+var secretKey = /(?:password|secret|authorization|cookie|apikey|credential|resumetoken|accesstoken|refreshtoken|bearertoken|sessiontoken|authtoken)/i;
+var contentKey = /^(?:input|output|inputs|outputs|inputmessages|outputmessages|messages|prompt|systemprompt|question|response|result|latestresult|arguments|args|content|text|options)$/i;
+var sanitizeStudioData = (value, includeContent, key = "") => {
+  const normalized = key.replace(/[^a-z0-9]/gi, "");
+  const keyPolicy = normalized.toLowerCase() === "apikey" && value && typeof value === "object" && "scopes" in value && "mode" in value;
+  if (secretKey.test(normalized) && !keyPolicy) return "[REDACTED]";
+  if (!includeContent && contentKey.test(normalized))
+    return "[CONTENT OMITTED: use --include-content]";
+  if (typeof value === "string")
+    return value.replace(/ktyx_(?:test|live)_[^\s"'<>]+/g, "[REDACTED]").replace(/Bearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [REDACTED]");
+  if (Array.isArray(value))
+    return value.map((item) => sanitizeStudioData(item, includeContent));
+  if (value && typeof value === "object")
+    return Object.fromEntries(
+      Object.entries(value).map(([name, item]) => [
+        name,
+        sanitizeStudioData(item, includeContent, name)
+      ])
+    );
+  return value;
+};
+var summarizeEvidence = (events) => {
+  const findings = events.flatMap((event) => {
+    const payload = event.payload;
+    const error = payload.error && typeof payload.error === "object" && !Array.isArray(payload.error) ? payload.error : {};
+    const suspension = event.type === "span.failed" && (error.controlFlow === true || ["GraphInterrupt", "NodeInterrupt", "ParallelChildWaiting"].includes(
+      String(error.name)
+    ));
+    const failed = !suspension && (/(?:failed|error|fault)/i.test(event.type) || payload.outcome === "fault" || payload.status === "failed");
+    const warning = suspension || /(?:retry|cancel|interrupt|limit_reached|denied|suspended|waiting)/i.test(
+      event.type
+    ) || ["denied", "cancelled"].includes(String(payload.outcome));
+    if (!failed && !warning) return [];
+    return [
+      {
+        eventId: event.id,
+        runId: event.runId,
+        nodeId: event.nodeId,
+        type: event.type,
+        occurredAt: event.occurredAt,
+        severity: failed ? "error" : "context",
+        evidence: payload
+      }
+    ];
+  });
+  return {
+    findings,
+    note: "Evidence, not an automated root-cause verdict. Interrupts, cancellations, and retries can be expected; correlate with the final run outcome. Missing telemetry does not prove an action did not occur."
+  };
+};
+var formatReadOutput = (value, json) => {
+  if (json) return JSON.stringify(value);
+  return JSON.stringify(value, null, 2);
+};
+
+// src/studio/eval-command.ts
+var integer2 = (max) => (value) => {
+  if (!/^\d+$/.test(value) || Number(value) < 1 || Number(value) > max)
+    throw new import_commander3.InvalidArgumentError(`Expected an integer between 1 and ${max}.`);
+  return Number(value);
+};
+var summary = ({ error: _error, ...run2 }) => run2;
+var compactDetail = (run2) => ({
+  ...summary(run2),
+  suite: void 0,
+  events: void 0,
+  result: void 0,
+  caseResults: run2.result?.cases.map(
+    ({ caseId, repetition, status, durationMs, steps }) => ({
+      caseId,
+      repetition,
+      status,
+      durationMs,
+      steps: steps.map(({ index, status: status2, criteria }) => ({
+        index,
+        status: status2,
+        criteria: criteria.map(({ id, passed }) => ({ id, passed }))
+      }))
+    })
+  )
+});
+function registerStudioEvalCommands(studio, log, request = fetch) {
+  const common = (command) => command.option(
+    "--connection <name>",
+    "Named project/API connection (or KORTYX_CONNECTION)."
+  ).option(
+    "--config-home <path>",
+    "Connection configuration directory.",
+    defaultConnectionsHome()
+  ).option(
+    "--home <path>",
+    "Managed local Studio directory.",
+    defaultStudioHome()
+  ).option("--api-url <url>", "Direct API URL; requires --api-key-env.").option(
+    "--api-key-env <name>",
+    "Environment variable holding a Studio key."
+  ).option("--json", "Print stable machine-readable JSON.").option(
+    "--include-content",
+    "Include suite definitions and captured execution content."
+  );
+  const print = (value, options) => log(
+    formatReadOutput(
+      sanitizeStudioData(
+        { schemaVersion: 1, ...value },
+        options.includeContent ?? false
+      ),
+      options.json ?? false
+    )
+  );
+  const clientFor = async (options, url) => {
+    const connection = await resolveConnection(options, url);
+    return {
+      connection,
+      client: new StudioEvalClient(
+        connection.apiUrl,
+        connection.apiKey,
+        request
+      )
+    };
+  };
+  const evals = studio.command("evals").description(
+    "Discover suites, enqueue evals, inspect saved results and cancel runs."
+  );
+  const suites = evals.command("suites").description("Discover application-owned eval suites.");
+  const discover = async (options) => {
+    const { connection, client } = await clientFor(options);
+    const data = await client.targets();
+    const environment = options.environment ?? connection.environment;
+    const targets = data.targets.filter(
+      (target) => (!options.target || target.id === options.target) && (!environment || target.environment === environment)
+    );
+    return {
+      connection,
+      client,
+      targets,
+      canRun: data.canRun,
+      studioJudge: data.studioJudge ?? null
+    };
+  };
+  const selectionOptions = (command) => common(command).option("--target <id>", "Application eval target ID.").option(
+    "--environment <name>",
+    "Filter targets; defaults to the connection environment."
+  );
+  selectionOptions(
+    evals.command("doctor").description(
+      "Check deployment wiring without starting workflows or model calls."
+    )
+  ).option("--suite <id>", "Check that a specific suite is registered.").option(
+    "--judge <location>",
+    "Judge to check: studio (default) or app.",
+    (value) => {
+      if (value !== "studio" && value !== "app")
+        throw new import_commander3.InvalidArgumentError("Expected studio or app.");
+      return value;
+    },
+    "studio"
+  ).action(async (options) => {
+    let report;
+    try {
+      const { connection, client } = await clientFor(options);
+      report = buildEvalDoctorReport(await client.targets(), {
+        ...options,
+        environment: options.environment ?? connection.environment,
+        judge: options.judge ?? "studio"
+      });
+    } catch (error) {
+      report = evalDoctorFailure(error);
+    }
+    log(
+      options.json ? JSON.stringify(report) : formatEvalDoctorReport(report)
+    );
+    if (report.status === "failed") process.exitCode = 1;
+  });
+  selectionOptions(
+    suites.command("list").description("List targets, suites, revisions and case IDs.")
+  ).action(async (options) => {
+    const { connection, targets, canRun, studioJudge } = await discover(options);
+    print(
+      {
+        connection: connection.name,
+        canRun,
+        studioJudge,
+        targets: targets.map((target) => ({
+          id: target.id,
+          name: target.name,
+          environment: target.environment,
+          available: target.manifest !== null,
+          appJudge: target.manifest?.judge ?? null,
+          studioJudging: Boolean(target.manifest?.studioJudging),
+          suites: target.manifest?.suites.map((suite) => ({
+            id: suite.id,
+            name: suite.name,
+            revision: target.revisions[suite.id],
+            caseIds: suite.cases.map((item) => item.id)
+          })) ?? []
+        }))
+      },
+      options
+    );
+  });
+  const select = async (suiteId, options) => {
+    const data = await discover(options);
+    const matches = data.targets.flatMap((target) => {
+      const suite = target.manifest?.suites.find((item) => item.id === suiteId);
+      return suite ? [{ target, suite }] : [];
+    });
+    const match = matches[0];
+    if (matches.length !== 1 || !match)
       throw new StudioReadError(
-        "unknown_connection",
-        "Connection not found. Run kortyx connections list."
+        "invalid_suite_selection",
+        matches.length ? "Suite exists on multiple targets. Select --target and/or --environment." : "Suite unavailable. List suites and check the target connection."
       );
-    }
-    config.current = name;
-    await saveConnections(config, options.configHome);
-    print({ current: name }, options);
+    return { ...data, ...match };
+  };
+  selectionOptions(
+    suites.command("get <suite-id>").description(
+      "Read a suite; --include-content returns its full definition."
+    )
+  ).action(async (suiteId, options) => {
+    const { connection, target, suite } = await select(suiteId, options);
+    print(
+      {
+        connection: connection.name,
+        targetId: target.id,
+        revision: target.revisions[suite.id],
+        suite: options.includeContent ? suite : {
+          id: suite.id,
+          name: suite.name,
+          caseIds: suite.cases.map((item) => item.id)
+        }
+      },
+      options
+    );
   });
-  common(
-    command.command("remove <name>").description(
-      "Remove a saved locator, not its credential or remote project."
+  const runs = evals.command("runs").description("Start and inspect saved eval runs.");
+  selectionOptions(
+    runs.command("start <suite-id>").description(
+      "Enqueue once and return immediately; does not wait for grades."
     )
-  ).action(async (name, options) => {
-    const config = await readConnections(options.configHome);
-    if (!config.profiles.some((profile) => profile.name === name))
+  ).option(
+    "--case <id>",
+    "Select a case; repeat for multiple cases.",
+    (value, previous) => [...previous, value],
+    []
+  ).option(
+    "--repetitions <count>",
+    "Attempts per selected case (1\u201320).",
+    integer2(20),
+    1
+  ).option("--concurrency <count>", "Parallel attempts (1\u20134).", integer2(4), 1).option(
+    "--judge <location>",
+    "Judge selection: studio (default) or app.",
+    (value) => {
+      if (value !== "studio" && value !== "app")
+        throw new import_commander3.InvalidArgumentError("Expected studio or app.");
+      return value;
+    },
+    "studio"
+  ).action(async (suiteId, options) => {
+    const { connection, client, target, suite, canRun } = await select(
+      suiteId,
+      options
+    );
+    if (!canRun)
       throw new StudioReadError(
-        "unknown_connection",
-        "Saved connection not found; managed local cannot be removed."
+        "eval_forbidden",
+        "This Studio key requires eval:run to execute suites."
       );
-    config.profiles = config.profiles.filter(
-      (profile) => profile.name !== name
+    const caseIds = options.case ?? [];
+    if (new Set(caseIds).size !== caseIds.length || caseIds.some((id) => !suite.cases.some((item) => item.id === id)) || (caseIds.length || suite.cases.length) * options.repetitions > 100)
+      throw new StudioReadError(
+        "invalid_case_selection",
+        "Select unique existing case IDs and at most 100 total attempts."
+      );
+    const run2 = await client.start({
+      targetId: target.id,
+      judge: options.judge ?? "studio",
+      suiteId,
+      suiteRevision: target.revisions[suiteId] ?? "",
+      repetitions: options.repetitions,
+      concurrency: options.concurrency,
+      ...caseIds.length ? { caseIds } : {}
+    });
+    print(
+      {
+        connection: connection.name,
+        id: run2.id,
+        status: "queued",
+        studioUrl: connection.studioUrl ? `${connection.studioUrl}/evals/runs/${run2.id}` : null
+      },
+      options
     );
-    if (config.current === name) delete config.current;
-    await saveConnections(config, options.configHome);
-    print({ removed: name }, options);
   });
-  return command;
-};
-
-// src/studio/command.ts
-var import_commander3 = require("commander");
+  common(
+    runs.command("list").description("List the latest 100 saved runs in this project.")
+  ).option(
+    "--environment <name>",
+    "Filter results; defaults to the connection environment."
+  ).action(async (options) => {
+    const { connection, client } = await clientFor(options);
+    const data = await client.runs();
+    const environment = options.environment ?? connection.environment;
+    print(
+      {
+        connection: connection.name,
+        runs: data.runs.filter((run2) => !environment || run2.environment === environment).map(summary)
+      },
+      options
+    );
+  });
+  common(
+    runs.command("get <id-or-url>").description(
+      "Read status, counts and criterion verdicts; content is opt-in."
+    )
+  ).action(async (input, options) => {
+    const target = parseEvalRunTarget(input);
+    const { connection, client } = await clientFor(options, target.url);
+    const { run: run2 } = await client.run(target.id);
+    print(
+      {
+        connection: connection.name,
+        run: options.includeContent ? run2 : compactDetail(run2)
+      },
+      options
+    );
+  });
+  common(
+    runs.command("cancel <id-or-url>").description(
+      "Request cooperative cancellation of a queued or running eval."
+    )
+  ).action(async (input, options) => {
+    const target = parseEvalRunTarget(input);
+    const { connection, client } = await clientFor(options, target.url);
+    const data = await client.cancel(target.id);
+    print({ connection: connection.name, id: target.id, ...data }, options);
+  });
+}
 
 // src/studio/local-stack.ts
-var import_promises3 = require("fs/promises");
+var import_promises4 = require("fs/promises");
 var import_telemetry_contracts3 = require("@kortyx/telemetry-contracts");
 var composeArgs = (home, args) => [
   "compose",
@@ -1033,9 +2220,20 @@ var startStudio = async (options, runtime) => {
     const studioImage = String(
       environment.KORTYX_STUDIO_IMAGE_REF ?? `${environment.KORTYX_STUDIO_IMAGE ?? "ghcr.io/kortyx-io/kortyx-studio"}:${config.imageTag}`
     );
-    await runtime.run("docker", ["pull", apiImage], { inherit: true });
-    if (studioImage !== apiImage)
-      await runtime.run("docker", ["pull", studioImage], { inherit: true });
+    const pullPolicy = process.env.KORTYX_STUDIO_PULL_POLICY ?? "always";
+    for (const image of /* @__PURE__ */ new Set([apiImage, studioImage])) {
+      if (pullPolicy === "never") continue;
+      if (pullPolicy === "missing") {
+        try {
+          await runtime.run("docker", ["image", "inspect", image], {
+            inherit: false
+          });
+          continue;
+        } catch {
+        }
+      }
+      await runtime.run("docker", ["pull", image], { inherit: true });
+    }
     const capability = await runtime.run(
       "docker",
       [
@@ -1049,7 +2247,7 @@ var startStudio = async (options, runtime) => {
       ],
       { inherit: false }
     );
-    await (0, import_promises3.writeFile)(
+    await (0, import_promises4.writeFile)(
       studioComposePath(options.home),
       studioComposeFile(
         process.platform !== "win32" && capability.stdout.trim() === "supported"
@@ -1152,7 +2350,7 @@ var resetStudio = async (home, confirm, runtime) => {
 
 // src/studio/read-command.ts
 var import_telemetry_contracts5 = require("@kortyx/telemetry-contracts");
-var import_commander2 = require("commander");
+var import_commander4 = require("commander");
 
 // src/studio/read-analysis.ts
 var import_telemetry_contracts4 = require("@kortyx/telemetry-contracts");
@@ -1459,141 +2657,10 @@ var compareRunAnalysis = (left, right) => {
   };
 };
 
-// src/studio/read-output.ts
-var parseStudioTarget = (input, entity) => {
-  if (/^[a-z][a-z0-9+.-]*:/i.test(input)) {
-    let url;
-    try {
-      url = new URL(input);
-    } catch {
-      throw new StudioReadError("invalid_target", "Invalid Studio URL.");
-    }
-    if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) {
-      throw new StudioReadError(
-        "invalid_target",
-        "Use an HTTP(S) Studio entity URL without embedded credentials."
-      );
-    }
-    const match = url.pathname.match(
-      /\/(runs|sessions|interrupts)\/([^/]+)\/?$/
-    );
-    if (!match)
-      throw new StudioReadError(
-        "invalid_target",
-        "Expected a Studio /runs/<id>, /sessions/<id>, or /interrupts/<id> URL."
-      );
-    if (entity && match[1] !== entity)
-      throw new StudioReadError(
-        "invalid_target",
-        "URL entity type does not match the command."
-      );
-    let id;
-    try {
-      id = decodeURIComponent(match[2] ?? "");
-    } catch {
-      throw new StudioReadError(
-        "invalid_target",
-        "Invalid entity ID encoding."
-      );
-    }
-    validateId(id);
-    const selection = {};
-    for (const key of [
-      "tab",
-      "sessionTab",
-      "call",
-      "branch",
-      "node",
-      "event",
-      "trace",
-      "detailView"
-    ]) {
-      const value = url.searchParams.get(key);
-      if (value) selection[key] = value;
-    }
-    url.search = "";
-    url.hash = "";
-    return {
-      entity: match[1],
-      id,
-      url: url.toString(),
-      selection
-    };
-  }
-  if (!entity)
-    throw new StudioReadError(
-      "invalid_target",
-      "inspect requires a Studio entity URL. Use runs/sessions/interrupts get for a bare ID."
-    );
-  validateId(input);
-  return { entity, id: input };
-};
-var validateId = (id) => {
-  if (!id || id.length > 512 || id === "." || id === ".." || /[\s/?#\\]/.test(id) || [...id].some(
-    (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127
-  )) {
-    throw new StudioReadError("invalid_target", "Invalid Studio entity ID.");
-  }
-};
-var secretKey = /(?:password|secret|authorization|cookie|apikey|credential|resumetoken|accesstoken|refreshtoken|bearertoken|sessiontoken|authtoken)/i;
-var contentKey = /^(?:input|output|inputs|outputs|inputmessages|outputmessages|messages|prompt|systemprompt|question|response|result|latestresult|arguments|args|content|text|options)$/i;
-var sanitizeStudioData = (value, includeContent, key = "") => {
-  const normalized = key.replace(/[^a-z0-9]/gi, "");
-  const keyPolicy = normalized.toLowerCase() === "apikey" && value && typeof value === "object" && "scopes" in value && "mode" in value;
-  if (secretKey.test(normalized) && !keyPolicy) return "[REDACTED]";
-  if (!includeContent && contentKey.test(normalized))
-    return "[CONTENT OMITTED: use --include-content]";
-  if (typeof value === "string")
-    return value.replace(/ktyx_(?:test|live)_[^\s"'<>]+/g, "[REDACTED]").replace(/Bearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [REDACTED]");
-  if (Array.isArray(value))
-    return value.map((item) => sanitizeStudioData(item, includeContent));
-  if (value && typeof value === "object")
-    return Object.fromEntries(
-      Object.entries(value).map(([name, item]) => [
-        name,
-        sanitizeStudioData(item, includeContent, name)
-      ])
-    );
-  return value;
-};
-var summarizeEvidence = (events) => {
-  const findings = events.flatMap((event) => {
-    const payload = event.payload;
-    const error = payload.error && typeof payload.error === "object" && !Array.isArray(payload.error) ? payload.error : {};
-    const suspension = event.type === "span.failed" && (error.controlFlow === true || ["GraphInterrupt", "NodeInterrupt", "ParallelChildWaiting"].includes(
-      String(error.name)
-    ));
-    const failed = !suspension && (/(?:failed|error|fault)/i.test(event.type) || payload.outcome === "fault" || payload.status === "failed");
-    const warning = suspension || /(?:retry|cancel|interrupt|limit_reached|denied|suspended|waiting)/i.test(
-      event.type
-    ) || ["denied", "cancelled"].includes(String(payload.outcome));
-    if (!failed && !warning) return [];
-    return [
-      {
-        eventId: event.id,
-        runId: event.runId,
-        nodeId: event.nodeId,
-        type: event.type,
-        occurredAt: event.occurredAt,
-        severity: failed ? "error" : "context",
-        evidence: payload
-      }
-    ];
-  });
-  return {
-    findings,
-    note: "Evidence, not an automated root-cause verdict. Interrupts, cancellations, and retries can be expected; correlate with the final run outcome. Missing telemetry does not prove an action did not occur."
-  };
-};
-var formatReadOutput = (value, json) => {
-  if (json) return JSON.stringify(value);
-  return JSON.stringify(value, null, 2);
-};
-
 // src/studio/read-command.ts
-var integer = (min, max) => (value) => {
+var integer3 = (min, max) => (value) => {
   if (!/^\d+$/.test(value) || !Number.isSafeInteger(Number(value)) || Number(value) < min || Number(value) > max) {
-    throw new import_commander2.InvalidArgumentError(
+    throw new import_commander4.InvalidArgumentError(
       `Expected an integer between ${min} and ${max}.`
     );
   }
@@ -1749,7 +2816,7 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
   const detailOptions = (command) => common(command).option(
     "--event-limit <count>",
     "Maximum latest events and diagnostic findings (1\u201310000).",
-    integer(1, 1e4),
+    integer3(1, 1e4),
     100
   ).option(
     "--focus-selection",
@@ -1802,7 +2869,7 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
     "--environment <name>",
     "Environment filter; overrides the connection default."
   ).addOption(
-    new import_commander2.Option(
+    new import_commander4.Option(
       "--range <range>",
       "Studio time range: 1h, 24h, 7d, 30d, all (or Studio preset name)."
     ).choices([
@@ -1847,7 +2914,7 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
       ).option(
         "--event-limit <count>",
         "Maximum timeline steps returned per run (1\u201310000).",
-        integer(1, 1e4),
+        integer3(1, 1e4),
         100
       ).action(
         async (leftInput, rightInput, options) => {
@@ -1947,13 +3014,13 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
     }
     const list = filters(
       group.command("list").description("Read a bounded, filtered page.")
-    ).option("--limit <count>", "Page size (1\u2013100).", integer(1, 100), 25).option(
+    ).option("--limit <count>", "Page size (1\u2013100).", integer3(1, 100), 25).option(
       "--cursor <offset>",
       "Numeric offset returned as nextCursor.",
-      integer(0, Number.MAX_SAFE_INTEGER),
+      integer3(0, Number.MAX_SAFE_INTEGER),
       0
     ).option("--status <status>", "Entity status filter.").option("--session <id>", "Session filter.").option("--user <id>", "User filter.").option("--tenant <id>", "Tenant filter.").option("--model <name>", "Model filter.").option("--node <id>", "Node filter.").option("--tool <name>", "Tool name filter.").addOption(
-      new import_commander2.Option("--tool-outcome <outcome>", "Tool outcome filter.").choices([
+      new import_commander4.Option("--tool-outcome <outcome>", "Tool outcome filter.").choices([
         "success",
         "denied",
         "fault",
@@ -2082,14 +3149,14 @@ var defaultStudioRuntime = {
 var portParser = (value) => {
   const port = Number(value);
   if (!Number.isInteger(port) || port < 1 || port > 65535) {
-    throw new import_commander3.InvalidArgumentError(
+    throw new import_commander5.InvalidArgumentError(
       "Port must be an integer between 1 and 65535."
     );
   }
   return port;
 };
 var withHome = (command) => command.addOption(
-  new import_commander3.Option("--home <path>", "Local Studio state directory.").default(
+  new import_commander5.Option("--home <path>", "Local Studio state directory.").default(
     defaultStudioHome()
   )
 );
@@ -2099,7 +3166,7 @@ var configureOutput = (command, runtime) => command.configureOutput({
 });
 var createStudioCommand = (runtime = defaultStudioRuntime) => {
   const studio = configureOutput(
-    new import_commander3.Command("studio").description("Run and manage Kortyx Studio.").showHelpAfterError().addHelpText(
+    new import_commander5.Command("studio").description("Run and manage Kortyx Studio.").showHelpAfterError().addHelpText(
       "after",
       "\nLocal state and generated credentials are stored in ~/.kortyx/studio by default."
     ),
@@ -2142,7 +3209,7 @@ var createStudioCommand = (runtime = defaultStudioRuntime) => {
       "Generate unpersisted credentials for a self-hosted deployment.",
       false
     ).addOption(
-      new import_commander3.Option("--format <format>", "Credential output format.").choices(["human", "dotenv"]).default("human")
+      new import_commander5.Option("--format <format>", "Credential output format.").choices(["human", "dotenv"]).default("human")
     ).option(
       "--service-name <name>",
       "SDK service name used by dotenv output.",
@@ -2184,20 +3251,29 @@ var createStudioCommand = (runtime = defaultStudioRuntime) => {
   ).action(async ({ home, confirm }) => {
     await resetStudio(home, confirm, runtime);
   });
-  registerStudioReadCommands(studio, (message) => runtime.log(message));
+  registerStudioReadCommands(
+    studio,
+    (message) => runtime.log(message),
+    runtime.request
+  );
+  registerStudioEvalCommands(
+    studio,
+    (message) => runtime.log(message),
+    runtime.request
+  );
   return studio;
 };
 
 // src/workflow-calls.ts
-var import_node_path3 = require("path");
+var import_node_path5 = require("path");
 var import_typescript = __toESM(require("typescript"));
 function discoverWorkflowCalls(entry, snapshots) {
-  const configPath = import_typescript.default.findConfigFile((0, import_node_path3.dirname)(entry), import_typescript.default.sys.fileExists);
+  const configPath = import_typescript.default.findConfigFile((0, import_node_path5.dirname)(entry), import_typescript.default.sys.fileExists);
   const config = configPath ? import_typescript.default.readConfigFile(configPath, import_typescript.default.sys.readFile).config : {};
   const parsed = import_typescript.default.parseJsonConfigFileContent(
     config ?? {},
     import_typescript.default.sys,
-    configPath ? (0, import_node_path3.dirname)(configPath) : (0, import_node_path3.dirname)(entry)
+    configPath ? (0, import_node_path5.dirname)(configPath) : (0, import_node_path5.dirname)(entry)
   );
   const program = import_typescript.default.createProgram([entry], {
     ...parsed.options,
@@ -2238,7 +3314,7 @@ function discoverWorkflowCalls(entry, snapshots) {
     );
   };
   const factoryBindings = /* @__PURE__ */ new WeakMap();
-  function resolve4(e, bindings, seen = /* @__PURE__ */ new Set()) {
+  function resolve6(e, bindings, seen = /* @__PURE__ */ new Set()) {
     if (!e) return;
     e = unwrap(e);
     if (seen.has(e)) return;
@@ -2246,18 +3322,18 @@ function discoverWorkflowCalls(entry, snapshots) {
     if (import_typescript.default.isIdentifier(e)) {
       const s = symbol(e);
       const bound = s && bindings.get(s);
-      if (bound) return resolve4(bound, bindings, seen);
+      if (bound) return resolve6(bound, bindings, seen);
       const d = s?.valueDeclaration;
       if (d && (import_typescript.default.isVariableDeclaration(d) || import_typescript.default.isPropertyAssignment(d)) && d.initializer && (!import_typescript.default.isVariableDeclaration(d) || import_typescript.default.isVariableDeclarationList(d.parent) && (d.parent.flags & import_typescript.default.NodeFlags.Const) !== 0))
-        return resolve4(d.initializer, bindings, seen);
+        return resolve6(d.initializer, bindings, seen);
     }
     if (import_typescript.default.isPropertyAccessExpression(e)) {
       const found = property(e.expression, e.name.text, bindings, seen);
-      if (found) return resolve4(found, bindings, seen);
+      if (found) return resolve6(found, bindings, seen);
       const owner = symbol(e.expression);
       const declaration = symbol(e.name)?.valueDeclaration;
       if (owner && owner.flags & import_typescript.default.SymbolFlags.Module && declaration && import_typescript.default.isVariableDeclaration(declaration))
-        return resolve4(declaration.initializer, bindings, seen);
+        return resolve6(declaration.initializer, bindings, seen);
       return e;
     }
     if (import_typescript.default.isElementAccessExpression(e) && e.argumentExpression && import_typescript.default.isStringLiteralLike(e.argumentExpression)) {
@@ -2267,7 +3343,7 @@ function discoverWorkflowCalls(entry, snapshots) {
         bindings,
         seen
       );
-      return found ? resolve4(found, bindings, seen) : e;
+      return found ? resolve6(found, bindings, seen) : e;
     }
     if (import_typescript.default.isCallExpression(e)) {
       const d = symbol(
@@ -2281,12 +3357,12 @@ function discoverWorkflowCalls(entry, snapshots) {
             const key = symbol(p.name);
             const arg = e.arguments[i] ?? p.initializer;
             if (key && arg)
-              bound.set(key, resolve4(arg, bindings, new Set(seen)) ?? arg);
+              bound.set(key, resolve6(arg, bindings, new Set(seen)) ?? arg);
           }
         });
         const returned = import_typescript.default.isBlock(fn.body) ? fn.body.statements.filter(import_typescript.default.isReturnStatement).map((r) => r.expression) : [fn.body];
         if (returned.length === 1 && returned[0]) {
-          const value = resolve4(returned[0], bound, seen);
+          const value = resolve6(returned[0], bound, seen);
           if (value) factoryBindings.set(value, bound);
           return value;
         }
@@ -2295,30 +3371,30 @@ function discoverWorkflowCalls(entry, snapshots) {
     return e;
   }
   function property(e, key, bindings, seen = /* @__PURE__ */ new Set()) {
-    const obj = resolve4(e, bindings, seen);
+    const obj = resolve6(e, bindings, seen);
     if (!obj || !import_typescript.default.isObjectLiteralExpression(obj)) return;
     const scoped = factoryBindings.get(obj) ?? bindings;
     for (const p of [...obj.properties].reverse()) {
       if (import_typescript.default.isSpreadAssignment(p)) {
-        const spread = resolve4(p.expression, scoped, new Set(seen));
+        const spread = resolve6(p.expression, scoped, new Set(seen));
         if (!spread || !import_typescript.default.isObjectLiteralExpression(spread)) return void 0;
         const found = property(p.expression, key, scoped, new Set(seen));
         if (found) return found;
       } else if (p.name && (import_typescript.default.isIdentifier(p.name) || import_typescript.default.isStringLiteralLike(p.name)) && p.name.text === key) {
         if (import_typescript.default.isPropertyAssignment(p))
-          return resolve4(p.initializer, scoped, new Set(seen));
+          return resolve6(p.initializer, scoped, new Set(seen));
         if (import_typescript.default.isShorthandPropertyAssignment(p))
-          return resolve4(p.name, scoped, new Set(seen));
+          return resolve6(p.name, scoped, new Set(seen));
       }
     }
     return void 0;
   }
   function literal(e, bindings) {
-    const value = resolve4(e, bindings);
+    const value = resolve6(e, bindings);
     return value && import_typescript.default.isStringLiteralLike(value) ? value.text : void 0;
   }
   function workflowId(e, bindings) {
-    const value = resolve4(e, bindings);
+    const value = resolve6(e, bindings);
     if (!value) return;
     if (import_typescript.default.isStringLiteralLike(value)) return value.text;
     if (import_typescript.default.isCallExpression(value) && api(value.expression, "defineWorkflow"))
@@ -2334,19 +3410,19 @@ function discoverWorkflowCalls(entry, snapshots) {
     if (d && import_typescript.default.isBindingElement(d) && (d.propertyName?.getText() ?? d.name.getText()) === "useWorkflow") {
       const decl = d.parent.parent;
       if (import_typescript.default.isVariableDeclaration(decl) && decl.initializer) {
-        const value2 = resolve4(decl.initializer, /* @__PURE__ */ new Map());
+        const value2 = resolve6(decl.initializer, /* @__PURE__ */ new Map());
         return !!value2 && import_typescript.default.isCallExpression(value2) && api(value2.expression, "createWorkflowHooks");
       }
     }
     if (import_typescript.default.isPropertyAccessExpression(e) && e.name.text === "useWorkflow") {
-      const value2 = resolve4(e.expression, /* @__PURE__ */ new Map());
+      const value2 = resolve6(e.expression, /* @__PURE__ */ new Map());
       return !!value2 && import_typescript.default.isCallExpression(value2) && api(value2.expression, "createWorkflowHooks");
     }
-    const value = resolve4(e, /* @__PURE__ */ new Map());
+    const value = resolve6(e, /* @__PURE__ */ new Map());
     return !!value && value !== e && api(value, "useWorkflow");
   }
   function functionFor(e) {
-    const value = resolve4(e, /* @__PURE__ */ new Map());
+    const value = resolve6(e, /* @__PURE__ */ new Map());
     if (value && (import_typescript.default.isArrowFunction(value) || import_typescript.default.isFunctionExpression(value)))
       return value;
     const d = symbol(
@@ -2369,7 +3445,7 @@ function discoverWorkflowCalls(entry, snapshots) {
         target.toolDiscovery.warnings.push(warning);
       warnings.add(warning);
     };
-    const value = resolve4(expression, bindings);
+    const value = resolve6(expression, bindings);
     if (!value || seen.has(value)) {
       unresolved();
       return;
@@ -2414,8 +3490,8 @@ function discoverWorkflowCalls(entry, snapshots) {
       bindings
     );
     const schema = property(value, "inputSchema", bindings);
-    const fields = resolve4(property(schema, "properties", bindings), bindings);
-    const requiredValue = resolve4(
+    const fields = resolve6(property(schema, "properties", bindings), bindings);
+    const requiredValue = resolve6(
       property(schema, "required", bindings),
       bindings
     );
@@ -2504,7 +3580,7 @@ function discoverWorkflowCalls(entry, snapshots) {
               if (!arg) return;
               if (import_typescript.default.isIdentifier(p.name)) {
                 const s = symbol(p.name);
-                if (s) bound.set(s, resolve4(arg, bindings) ?? arg);
+                if (s) bound.set(s, resolve6(arg, bindings) ?? arg);
               } else if (import_typescript.default.isObjectBindingPattern(p.name)) {
                 for (const b of p.name.elements) {
                   const value = property(
@@ -2514,7 +3590,7 @@ function discoverWorkflowCalls(entry, snapshots) {
                   );
                   const s = symbol(b.name);
                   if (s && value)
-                    bound.set(s, resolve4(value, bindings) ?? value);
+                    bound.set(s, resolve6(value, bindings) ?? value);
                 }
               }
             });
@@ -2529,7 +3605,7 @@ function discoverWorkflowCalls(entry, snapshots) {
   for (const source of program.getSourceFiles().filter(local)) {
     const walk = (n) => {
       if (import_typescript.default.isCallExpression(n) && api(n.expression, "defineWorkflow")) {
-        const obj = resolve4(n.arguments[0], /* @__PURE__ */ new Map());
+        const obj = resolve6(n.arguments[0], /* @__PURE__ */ new Map());
         const id = obj && literal(property(obj, "id", /* @__PURE__ */ new Map()), /* @__PURE__ */ new Map());
         if (id && known.has(id) && obj && import_typescript.default.isObjectLiteralExpression(obj))
           definitions.set(id, [...definitions.get(id) ?? [], obj]);
@@ -2599,24 +3675,24 @@ var readPackageName = async (cwd) => {
   let current = cwd;
   for (; ; ) {
     try {
-      const raw = await (0, import_promises4.readFile)((0, import_node_path4.join)(current, "package.json"), "utf8");
+      const raw = await (0, import_promises5.readFile)((0, import_node_path6.join)(current, "package.json"), "utf8");
       const parsed = JSON.parse(raw);
       if (isRecord(parsed) && typeof parsed.name === "string") {
         return parsed.name;
       }
     } catch {
     }
-    const parent = (0, import_node_path4.dirname)(current);
+    const parent = (0, import_node_path6.dirname)(current);
     if (parent === current) return void 0;
     current = parent;
   }
 };
 var loadEnvFiles = async (cwd) => {
-  for (const filename of [".env", ".env.local"]) {
-    const path = (0, import_node_path4.join)(cwd, filename);
+  for (const filename of [".env.local", ".env"]) {
+    const path = (0, import_node_path6.join)(cwd, filename);
     let raw;
     try {
-      raw = await (0, import_promises4.readFile)(path, "utf8");
+      raw = await (0, import_promises5.readFile)(path, "utf8");
     } catch {
       continue;
     }
@@ -2642,7 +3718,7 @@ var resolveOptions = async (options) => {
   if (!options.entry) {
     throw new Error("Missing required option: --entry <path>.");
   }
-  const entry = (0, import_node_path4.isAbsolute)(options.entry) ? options.entry : (0, import_node_path4.resolve)(options.cwd, options.entry);
+  const entry = (0, import_node_path6.isAbsolute)(options.entry) ? options.entry : (0, import_node_path6.resolve)(options.cwd, options.entry);
   const environment = options.environment ?? process.env.KORTYX_TELEMETRY_ENVIRONMENT ?? process.env.NODE_ENV ?? "development";
   const serviceName = options.serviceName ?? process.env.KORTYX_TELEMETRY_SERVICE_NAME ?? await readPackageName(options.cwd) ?? "kortyx-app";
   return {
@@ -2656,19 +3732,19 @@ var resolveOptions = async (options) => {
   };
 };
 var createFrameworkMarkerStubs = async () => {
-  const root = await (0, import_promises4.mkdtemp)((0, import_node_path4.join)((0, import_node_os3.tmpdir)(), "kortyx-cli-stubs-"));
+  const root = await (0, import_promises5.mkdtemp)((0, import_node_path6.join)((0, import_node_os3.tmpdir)(), "kortyx-cli-stubs-"));
   for (const moduleName of ["server-only", "client-only"]) {
-    const directory = (0, import_node_path4.join)(root, moduleName);
-    await (0, import_promises4.mkdir)(directory, { recursive: true });
-    await (0, import_promises4.writeFile)((0, import_node_path4.join)(directory, "index.js"), "", "utf8");
-    await (0, import_promises4.writeFile)(
-      (0, import_node_path4.join)(directory, "package.json"),
+    const directory = (0, import_node_path6.join)(root, moduleName);
+    await (0, import_promises5.mkdir)(directory, { recursive: true });
+    await (0, import_promises5.writeFile)((0, import_node_path6.join)(directory, "index.js"), "", "utf8");
+    await (0, import_promises5.writeFile)(
+      (0, import_node_path6.join)(directory, "package.json"),
       JSON.stringify({ name: moduleName, version: "0.0.0", main: "index.js" }),
       "utf8"
     );
   }
   const previousNodePath = process.env.NODE_PATH;
-  process.env.NODE_PATH = previousNodePath ? `${root}${import_node_path4.delimiter}${previousNodePath}` : root;
+  process.env.NODE_PATH = previousNodePath ? `${root}${import_node_path6.delimiter}${previousNodePath}` : root;
   const moduleWithPrivateInit = import_node_module.default;
   moduleWithPrivateInit._initPaths?.();
   return {
@@ -2676,48 +3752,71 @@ var createFrameworkMarkerStubs = async () => {
       if (previousNodePath === void 0) delete process.env.NODE_PATH;
       else process.env.NODE_PATH = previousNodePath;
       moduleWithPrivateInit._initPaths?.();
-      await (0, import_promises4.rm)(root, { recursive: true, force: true });
+      await (0, import_promises5.rm)(root, { recursive: true, force: true });
     }
   };
 };
-var importEntry = async (entry) => {
+var openEntry = async (entry) => {
   const stubs = await createFrameworkMarkerStubs();
   let unregisterPaths;
+  let unregisterEsm;
+  const unregisterTsx = (0, import_api.register)();
+  const release = async () => {
+    unregisterPaths?.();
+    unregisterTsx();
+    await unregisterEsm?.();
+    await stubs.cleanup();
+  };
   try {
-    const tsconfig = await findNearestFile((0, import_node_path4.dirname)(entry), "tsconfig.json");
+    const tsconfig = await findNearestFile((0, import_node_path6.dirname)(entry), "tsconfig.json");
+    unregisterEsm = (0, import_api2.register)(tsconfig ? { tsconfig } : {});
     unregisterPaths = tsconfig ? await registerTsconfigPaths(tsconfig) : void 0;
-    const module2 = (0, import_api.require)(entry, (0, import_node_path4.join)((0, import_node_path4.dirname)(entry), "kortyx-cli.cjs"));
+    const module2 = (0, import_node_module.createRequire)((0, import_node_path6.join)((0, import_node_path6.dirname)(entry), "kortyx-cli.cjs"))(entry);
     if (!isRecord(module2)) {
       throw new Error(`Entry module did not export an object: ${entry}`);
     }
-    return module2;
+    return { module: module2, release };
+  } catch (error) {
+    await release();
+    throw error;
+  }
+};
+var importEntry = async (entry) => {
+  const loaded = await openEntry(entry);
+  try {
+    return loaded.module;
   } finally {
-    unregisterPaths?.();
-    await stubs.cleanup();
+    await loaded.release();
   }
 };
 var findNearestFile = async (startDirectory, filename) => {
   let current = startDirectory;
   for (; ; ) {
-    const candidate = (0, import_node_path4.join)(current, filename);
+    const candidate = (0, import_node_path6.join)(current, filename);
     try {
-      await (0, import_promises4.access)(candidate);
+      await (0, import_promises5.access)(candidate);
       return candidate;
     } catch {
-      const parent = (0, import_node_path4.dirname)(current);
+      const parent = (0, import_node_path6.dirname)(current);
       if (parent === current) return void 0;
       current = parent;
     }
   }
 };
 var registerTsconfigPaths = async (tsconfigPath) => {
-  const raw = JSON.parse(await (0, import_promises4.readFile)(tsconfigPath, "utf8"));
+  const parsed = import_typescript2.default.parseConfigFileTextToJson(
+    tsconfigPath,
+    await (0, import_promises5.readFile)(tsconfigPath, "utf8")
+  );
+  if (parsed.error)
+    throw new Error(`Invalid TypeScript configuration: ${tsconfigPath}`);
+  const raw = parsed.config;
   if (!isRecord(raw)) return () => void 0;
   const compilerOptions = isRecord(raw.compilerOptions) ? raw.compilerOptions : void 0;
   const paths = isRecord(compilerOptions?.paths) ? compilerOptions.paths : void 0;
   if (!paths) return () => void 0;
-  const tsconfigDirectory = (0, import_node_path4.dirname)(tsconfigPath);
-  const baseUrl = typeof compilerOptions?.baseUrl === "string" ? (0, import_node_path4.resolve)(tsconfigDirectory, compilerOptions.baseUrl) : tsconfigDirectory;
+  const tsconfigDirectory = (0, import_node_path6.dirname)(tsconfigPath);
+  const baseUrl = typeof compilerOptions?.baseUrl === "string" ? (0, import_node_path6.resolve)(tsconfigDirectory, compilerOptions.baseUrl) : tsconfigDirectory;
   const mappings = Object.entries(paths).flatMap(([pattern, rawTargets]) => {
     if (!Array.isArray(rawTargets)) return [];
     return rawTargets.filter((target) => typeof target === "string").map((target) => ({ pattern, target }));
@@ -2746,7 +3845,7 @@ var registerTsconfigPaths = async (tsconfigPath) => {
         request.length - mapping.patternSuffix.length
       ) : "";
       const mapped = mapping.wildcard ? `${mapping.targetPrefix}${wildcardValue}${mapping.targetSuffix}` : mapping.target;
-      for (const candidate of candidateModulePaths((0, import_node_path4.resolve)(baseUrl, mapped))) {
+      for (const candidate of candidateModulePaths((0, import_node_path6.resolve)(baseUrl, mapped))) {
         try {
           return originalResolve.call(this, candidate, parent, isMain, options);
         } catch {
@@ -2767,10 +3866,10 @@ var candidateModulePaths = (basePath) => [
   `${basePath}.cts`,
   `${basePath}.js`,
   `${basePath}.jsx`,
-  (0, import_node_path4.join)(basePath, "index.ts"),
-  (0, import_node_path4.join)(basePath, "index.tsx"),
-  (0, import_node_path4.join)(basePath, "index.js"),
-  (0, import_node_path4.join)(basePath, "index.jsx")
+  (0, import_node_path6.join)(basePath, "index.ts"),
+  (0, import_node_path6.join)(basePath, "index.tsx"),
+  (0, import_node_path6.join)(basePath, "index.js"),
+  (0, import_node_path6.join)(basePath, "index.jsx")
 ];
 var valueFromModule = (module2, exportName) => {
   if (exportName) {
@@ -2800,7 +3899,7 @@ var projectSnapshots = async (value, options) => {
   }
   const knownWorkflowIds = workflows.map((workflow) => workflow.id);
   return workflows.map(
-    (workflow) => (0, import_agent.projectWorkflowTopology)({
+    (workflow) => (0, import_agent2.projectWorkflowTopology)({
       workflow,
       environment: options.environment,
       service: options.service,
@@ -2928,12 +4027,30 @@ var runTopologyPush = async (rawOptions) => {
   printResults(results, options);
 };
 var createCliProgram = () => {
-  const program = new import_commander4.Command().name("kortyx").description("Build, observe, and operate Kortyx projects.").showHelpAfterError().addHelpText(
+  const program = new import_commander6.Command().name("kortyx").description("Build, observe, and operate Kortyx projects.").showHelpAfterError().addHelpText(
     "after",
     '\nRun "kortyx <command> --help" for detailed command options.'
   );
   program.addCommand(createStudioCommand());
   program.addCommand(createConnectionsCommand());
+  let releaseEvalEntry;
+  program.addCommand(
+    createLocalEvalsCommand({
+      load: async (entry, envFiles) => {
+        Object.assign(
+          process.env,
+          await loadEvalEnvironment(process.cwd(), envFiles)
+        );
+        const loaded = await openEntry(entry);
+        releaseEvalEntry = loaded.release;
+        return loaded.module;
+      },
+      release: async () => {
+        await releaseEvalEntry?.();
+        releaseEvalEntry = void 0;
+      }
+    })
+  );
   const topology = program.command("topology").description("Publish deterministic workflow topology to Studio.");
   topology.command("push").description("Project and publish workflow topology.").requiredOption(
     "--entry <path>",
@@ -2971,7 +4088,7 @@ var main = async () => {
   try {
     await program.parseAsync(process.argv);
   } catch (error) {
-    if (error instanceof import_commander4.CommanderError && error.exitCode === 0) return;
+    if (error instanceof import_commander6.CommanderError && error.exitCode === 0) return;
     if (error instanceof StudioReadError) {
       console.error(
         process.argv.includes("--json") ? JSON.stringify(error.toJSON()) : `[${error.code}] ${error.message}`