@kortyx/cli 0.13.0 → 0.15.0

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## [0.15.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.14.0...cli-v0.15.0) (2026-10-08)
+
+
+### Features
+
+* **telemetry:** preserve complete private error diagnostics ([#290](https://github.com/kortyx-io/kortyx/issues/290)) ([e1c5a88](https://github.com/kortyx-io/kortyx/commit/e1c5a88c494016fd84a1af40579412fcfde31271))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @kortyx/agent bumped to 0.31.1
+    * @kortyx/core bumped to 0.12.0
+    * @kortyx/telemetry-contracts bumped to 0.15.0
+
+## [0.14.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.13.0...cli-v0.14.0) (2026-10-07)
+
+
+### Features
+
+* **evals:** group suite runs and expose CLI results ([#284](https://github.com/kortyx-io/kortyx/issues/284)) ([146fd71](https://github.com/kortyx-io/kortyx/commit/146fd71a88587e5591dd0511a4c0cb57969f9d07))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @kortyx/agent bumped to 0.31.0
+
 ## [0.13.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.12.0...cli-v0.13.0) (2026-10-07)
 
 

package/dist/index.js

@@ -23,7 +23,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/index.ts
-var import_promises5 = require("fs/promises");
+var import_promises6 = require("fs/promises");
 var import_node_module = __toESM(require("module"));
 var import_node_os3 = require("os");
 var import_node_path6 = require("path");
@@ -148,7 +148,7 @@ var StudioApiTransport = class {
     }
     if (!response.ok) {
       await response.body?.cancel();
-      const message = response.status === 401 ? "Invalid, expired, or revoked Studio read key." : response.status === 403 ? "The API key lacks studio:read or the operation's required permission (eval:run for execution)." : response.status === 404 ? "Entity not found in this connection's project. Check the URL and selected connection." : `Studio API returned HTTP ${response.status}.`;
+      const message = response.status === 401 ? "Invalid, expired, or revoked Studio read key." : response.status === 403 ? "The API key lacks studio:read or the operation's required permission (diagnostics:read for diagnostics; eval:run for execution)." : response.status === 404 ? "Entity not found in this connection's project. Check the URL and selected connection." : `Studio API returned HTTP ${response.status}.`;
       throw new StudioReadError(
         "api_error",
         message,
@@ -208,7 +208,7 @@ var StudioApiTransport = class {
 };
 var StudioReadClient = class extends StudioApiTransport {
   async get(path, schema, query = {}) {
-    if (!/^\/v1\/studio\/(?:context|catalogs|workflows|(?:runs|sessions|interrupts)(?:\/[^/?#]+)?)$/.test(
+    if (!/^\/v1\/studio\/(?:context|catalogs|workflows|(?:runs|sessions|interrupts)(?:\/[^/?#]+)?|diagnostics\/[a-f0-9-]{36})$/.test(
       path
     ) || /(?:^|\/)(?:\.|\.\.|%2e(?:%2e)?)(?:\/|$)/i.test(path)) {
       throw new StudioReadError(
@@ -236,6 +236,7 @@ x-api-env: &api-env
 x-bootstrap-keys: &bootstrap-keys
   KORTYX_TELEMETRY_API_KEY: \${KORTYX_TELEMETRY_API_KEY}
   KORTYX_STUDIO_API_KEY: \${KORTYX_STUDIO_API_KEY}
+  KORTYX_STUDIO_ENABLE_DIAGNOSTICS: \${KORTYX_STUDIO_ENABLE_DIAGNOSTICS:-0}
   KORTYX_STUDIO_ENABLE_EVALS: \${KORTYX_STUDIO_ENABLE_EVALS:-0}
   KORTYX_STUDIO_ENABLE_REVIEWS: \${KORTYX_STUDIO_ENABLE_REVIEWS:-0}
 
@@ -1430,7 +1431,9 @@ var parseEvalRunTarget = (input) => {
     } catch {
       throw new StudioReadError("invalid_target", "Invalid eval run URL.");
     }
-    const match = locator.pathname.match(/\/evals\/runs\/([^/]+)\/?$/);
+    const match = locator.pathname.match(
+      /\/evals\/(?:runs|evaluations)\/([^/]+)\/?$/
+    );
     if (!match || !["http:", "https:"].includes(locator.protocol) || locator.username || locator.password)
       throw new StudioReadError(
         "invalid_target",
@@ -1446,6 +1449,59 @@ var parseEvalRunTarget = (input) => {
   return { id, url };
 };
 var StudioEvalClient = class extends StudioApiTransport {
+  evaluations() {
+    return this.requestJson(
+      "GET",
+      "/v1/studio/evals/evaluations",
+      import_evals.StudioEvaluationHistorySchema
+    );
+  }
+  evaluation(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "GET",
+      `/v1/studio/evals/evaluations/${id}`,
+      import_evals.StudioEvaluationDetailSchema
+    );
+  }
+  evaluationResults(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "GET",
+      `/v1/studio/evals/evaluations/${id}/results`,
+      import_evals.StudioEvaluationResultsSchema
+    );
+  }
+  startEvaluation(input) {
+    const parsed = import_evals.StudioEvaluationStartRequestSchema.safeParse(input);
+    if (!parsed.success)
+      throw new StudioReadError(
+        "invalid_eval_request",
+        "Invalid evaluation selection or execution limits."
+      );
+    return this.requestJson(
+      "POST",
+      "/v1/studio/evals/evaluations",
+      import_zod3.z.object({ id: import_zod3.z.uuid() }),
+      {},
+      parsed.data
+    );
+  }
+  cancelEvaluation(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "POST",
+      `/v1/studio/evals/evaluations/${id}/cancel`,
+      import_zod3.z.object({ ok: import_zod3.z.literal(true) })
+    );
+  }
+  validateId(id) {
+    if (!import_zod3.z.uuid().safeParse(id).success)
+      throw new StudioReadError(
+        "invalid_target",
+        "Expected an evaluation UUID."
+      );
+  }
   targets() {
     return this.requestJson(
       "GET",
@@ -1961,11 +2017,65 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
       options
     );
   });
+  const terminal = (status) => status !== "queued" && status !== "running";
+  const resultExitCode = (status) => status === "passed" ? 0 : status === "failed" ? 1 : status === "cancelled" ? 130 : 2;
+  const waitForEvaluation = async (client, id, timeout) => {
+    const deadline = Date.now() + timeout * 1e3;
+    for (; ; ) {
+      const { run: run2 } = await client.evaluation(id);
+      if (terminal(run2.status)) return run2;
+      if (Date.now() >= deadline)
+        throw new StudioReadError(
+          "eval_wait_timeout",
+          `Evaluation ${id} is still running. Inspect it with evals runs get; waiting did not cancel it.`
+        );
+      await new Promise(
+        (resolve6) => setTimeout(resolve6, Math.min(2e3, deadline - Date.now()))
+      );
+    }
+  };
+  const readEvaluation = async (client, id, includeContent) => {
+    const { run: run2 } = await client.evaluationResults(id);
+    return includeContent ? run2 : { ...run2, suites: run2.suites.map(compactDetail) };
+  };
+  const waitOptions = (command) => command.option(
+    "--timeout <seconds>",
+    "Maximum time to wait (1\u20137200 seconds); does not cancel execution.",
+    integer2(7200),
+    1800
+  );
   const runs = evals.command("runs").description("Start and inspect saved eval runs.");
   selectionOptions(
-    runs.command("start <suite-id>").description(
-      "Enqueue once and return immediately; does not wait for grades."
+    runs.command("start [suite-id]").description(
+      "Start an evaluation containing all or selected suites; optionally wait for results."
     )
+  ).option("--all", "Run every suite registered on the selected application.").option(
+    "--suite <id>",
+    "Select a suite; repeat for several suites.",
+    (value, previous) => [...previous, value],
+    []
+  ).option("--name <name>", "Display name for this evaluation run.").option(
+    "--source <source>",
+    "Trigger: manual, deployment, schedule or ci.",
+    (value) => {
+      if (!["manual", "deployment", "schedule", "ci"].includes(value))
+        throw new import_commander3.InvalidArgumentError(
+          "Expected manual, deployment, schedule or ci."
+        );
+      return value;
+    },
+    "manual"
+  ).option("--commit <sha>", "Deployed application commit.").option("--deployment-url <url>", "Deployment or CI job URL.").option(
+    "--idempotency-key <key>",
+    "Reuse a matching saved evaluation after a trigger retry."
+  ).option(
+    "--wait",
+    "Wait for final results and return a pass/fail exit code."
+  ).option(
+    "--timeout <seconds>",
+    "Maximum wait in seconds (1\u20137200); execution continues after timeout.",
+    integer2(7200),
+    1800
   ).option(
     "--case <id>",
     "Select a case; repeat for multiple cases.",
@@ -1986,39 +2096,102 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
     },
     "studio"
   ).action(async (suiteId, options) => {
-    const { connection, client, target, suite, canRun } = await select(
-      suiteId,
-      options
+    const ids = [...suiteId ? [suiteId] : [], ...options.suite ?? []];
+    if (!options.all && !ids.length || options.all && ids.length || new Set(ids).size !== ids.length)
+      throw new StudioReadError(
+        "invalid_suite_selection",
+        "Choose --all or unique suite IDs (positional or repeated --suite)."
+      );
+    const data = await discover(options);
+    const matches = data.targets.filter(
+      (target2) => target2.manifest && (options.all || ids.every(
+        (id) => target2.manifest.suites.some((suite) => suite.id === id)
+      ))
     );
-    if (!canRun)
+    const target = matches[0];
+    if (matches.length !== 1 || !target?.manifest)
+      throw new StudioReadError(
+        "invalid_suite_selection",
+        "Select one available application with --target and --environment; all selected suites must belong to it."
+      );
+    if (!data.canRun)
       throw new StudioReadError(
         "eval_forbidden",
         "This Studio key requires eval:run to execute suites."
       );
+    const suites2 = options.all ? target.manifest.suites : ids.map(
+      (id) => target.manifest.suites.find((suite) => suite.id === id)
+    );
     const caseIds = options.case ?? [];
-    if (new Set(caseIds).size !== caseIds.length || caseIds.some((id) => !suite.cases.some((item) => item.id === id)) || (caseIds.length || suite.cases.length) * options.repetitions > 100)
+    if (caseIds.length && (options.all || suites2.length !== 1))
       throw new StudioReadError(
         "invalid_case_selection",
-        "Select unique existing case IDs and at most 100 total attempts."
+        "--case requires exactly one selected suite."
       );
-    const run2 = await client.start({
+    if (new Set(caseIds).size !== caseIds.length || caseIds.some(
+      (id) => !suites2[0]?.cases.some((item) => item.id === id)
+    ) || suites2.some(
+      (suite) => (caseIds.length || suite.cases.length) * options.repetitions > 100
+    ))
+      throw new StudioReadError(
+        "invalid_case_selection",
+        "Select unique existing cases and at most 100 attempts per suite."
+      );
+    const run2 = await data.client.startEvaluation({
       targetId: target.id,
+      selection: options.all ? "all" : "selected",
+      suites: suites2.map((suite) => ({
+        suiteId: suite.id,
+        suiteRevision: target.revisions[suite.id] ?? "",
+        ...caseIds.length ? { caseIds } : {}
+      })),
       judge: options.judge ?? "studio",
-      suiteId,
-      suiteRevision: target.revisions[suiteId] ?? "",
       repetitions: options.repetitions,
       concurrency: options.concurrency,
-      ...caseIds.length ? { caseIds } : {}
-    });
-    print(
-      {
-        connection: connection.name,
-        id: run2.id,
-        status: "queued",
-        studioUrl: connection.studioUrl ? `${connection.studioUrl}/evals/runs/${run2.id}` : null
+      name: options.name,
+      metadata: {
+        source: options.source ?? "manual",
+        commit: options.commit,
+        deploymentUrl: options.deploymentUrl
       },
-      options
-    );
+      idempotencyKey: options.idempotencyKey
+    });
+    const studioUrl = data.connection.studioUrl ? `${data.connection.studioUrl}/evals/evaluations/${run2.id}` : null;
+    if (!options.wait) {
+      print(
+        {
+          connection: data.connection.name,
+          id: run2.id,
+          status: "queued",
+          studioUrl
+        },
+        options
+      );
+      return;
+    }
+    try {
+      const completed = await waitForEvaluation(
+        data.client,
+        run2.id,
+        options.timeout
+      );
+      print(
+        {
+          connection: data.connection.name,
+          studioUrl,
+          run: await readEvaluation(
+            data.client,
+            run2.id,
+            options.includeContent ?? false
+          )
+        },
+        options
+      );
+      process.exitCode = resultExitCode(completed.status);
+    } catch (error) {
+      process.exitCode = 2;
+      throw error;
+    }
   });
   common(
     runs.command("list").description("List the latest 100 saved runs in this project.")
@@ -2027,12 +2200,12 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
     "Filter results; defaults to the connection environment."
   ).action(async (options) => {
     const { connection, client } = await clientFor(options);
-    const data = await client.runs();
+    const data = await client.evaluations();
     const environment = options.environment ?? connection.environment;
     print(
       {
         connection: connection.name,
-        runs: data.runs.filter((run2) => !environment || run2.environment === environment).map(summary)
+        runs: data.runs.filter((run2) => !environment || run2.environment === environment).map((run2) => run2)
       },
       options
     );
@@ -2044,15 +2217,59 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
   ).action(async (input, options) => {
     const target = parseEvalRunTarget(input);
     const { connection, client } = await clientFor(options, target.url);
-    const { run: run2 } = await client.run(target.id);
+    let run2;
+    try {
+      run2 = await readEvaluation(
+        client,
+        target.id,
+        options.includeContent ?? false
+      );
+    } catch (error) {
+      if (!(error instanceof StudioReadError) || error.status !== 404)
+        throw error;
+      const legacy = await client.run(target.id);
+      run2 = options.includeContent ? legacy.run : compactDetail(legacy.run);
+    }
     print(
       {
         connection: connection.name,
-        run: options.includeContent ? run2 : compactDetail(run2)
+        run: run2
       },
       options
     );
   });
+  waitOptions(
+    common(
+      runs.command("wait <id-or-url>").description(
+        "Wait for a grouped evaluation and print its final suite/case results."
+      )
+    )
+  ).action(async (input, options) => {
+    const target = parseEvalRunTarget(input);
+    const { connection, client } = await clientFor(options, target.url);
+    try {
+      const completed = await waitForEvaluation(
+        client,
+        target.id,
+        options.timeout
+      );
+      print(
+        {
+          connection: connection.name,
+          run: await readEvaluation(
+            client,
+            target.id,
+            options.includeContent ?? false
+          )
+        },
+        options
+      );
+      process.exitCode = resultExitCode(completed.status);
+    } catch (error) {
+      process.exitCode = 2;
+      throw error;
+    }
+  });
   common(
     runs.command("cancel <id-or-url>").description(
       "Request cooperative cancellation of a queued or running eval."
@@ -2060,7 +2277,14 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
   ).action(async (input, options) => {
     const target = parseEvalRunTarget(input);
     const { connection, client } = await clientFor(options, target.url);
-    const data = await client.cancel(target.id);
+    let data;
+    try {
+      data = await client.cancelEvaluation(target.id);
+    } catch (error) {
+      if (!(error instanceof StudioReadError) || error.status !== 404)
+        throw error;
+      data = await client.cancel(target.id);
+    }
     print({ connection: connection.name, id: target.id, ...data }, options);
   });
 }
@@ -2349,6 +2573,8 @@ var resetStudio = async (home, confirm, runtime) => {
 };
 
 // src/studio/read-command.ts
+var import_node_crypto2 = require("crypto");
+var import_promises5 = require("fs/promises");
 var import_telemetry_contracts5 = require("@kortyx/telemetry-contracts");
 var import_commander4 = require("commander");
 
@@ -2712,6 +2938,80 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
       )
     };
   };
+  const diagnostics = studio.command("diagnostics").description("Inspect or download a retained private error diagnostic.");
+  for (const action of ["get", "download"]) {
+    common(
+      diagnostics.command(action).argument("<diagnostic-id>").option(
+        "--environment <name>",
+        "Diagnostic environment (or connection default)."
+      ).option(
+        "--output <path>",
+        "Write a diagnostic JSON file without overwriting an existing file."
+      )
+    ).action(async (id, options) => {
+      if (!/^[a-f0-9-]{36}$/i.test(id))
+        throw new StudioReadError(
+          "invalid_diagnostic_id",
+          "Expected a diagnostic UUID."
+        );
+      const { connection, client } = await clientFor(options);
+      const environment = options.environment ?? connection.environment;
+      if (!environment)
+        throw new StudioReadError(
+          "environment_required",
+          "Provide --environment for diagnostic retrieval."
+        );
+      const value = await client.get(
+        `/v1/studio/diagnostics/${id}`,
+        import_telemetry_contracts5.DiagnosticResponseSchema,
+        {
+          env: environment,
+          ...action === "download" ? { download: "1" } : {}
+        }
+      );
+      if (action === "get") {
+        const { content, ...summary2 } = value;
+        print(
+          {
+            ...summary2,
+            capture: {
+              status: content?.capture.status ?? value.manifest.captureStatus,
+              omissions: content?.capture.omissions.length ?? 0,
+              redactions: content?.capture.redactions.length ?? 0
+            },
+            ...options.includeContent ? { content } : {},
+            note: "Private diagnostic access requires diagnostics:read. Use download for the complete redacted artifact."
+          },
+          options
+        );
+        return;
+      }
+      if (value.state !== "available" || !value.content)
+        throw new StudioReadError(
+          "diagnostic_incomplete",
+          `Diagnostic is ${value.state}; received ${value.receivedParts} of ${value.manifest.partCount} parts.`
+        );
+      const bytes = JSON.stringify(value.content);
+      if ((0, import_node_crypto2.createHash)("sha256").update(bytes).digest("hex") !== value.contentChecksum || Buffer.byteLength(bytes) !== value.contentByteLength)
+        throw new StudioReadError(
+          "diagnostic_integrity_failed",
+          "Diagnostic download failed integrity verification."
+        );
+      if (options.output) {
+        await (0, import_promises5.writeFile)(options.output, bytes, { flag: "wx", mode: 384 });
+        print(
+          {
+            diagnosticId: id,
+            state: value.state,
+            file: options.output,
+            bytes: value.contentByteLength,
+            checksum: value.contentChecksum
+          },
+          options
+        );
+      } else log(bytes);
+    });
+  }
   const inspect = async (input, options, entity) => {
     const target = parseStudioTarget(input, entity);
     const { connection, client } = await clientFor(options, target.url);
@@ -3100,7 +3400,7 @@ var registerStudioReadCommands = (studio, log, request = fetch) => {
 };
 
 // src/studio/runtime.ts
-var import_node_crypto2 = require("crypto");
+var import_node_crypto3 = require("crypto");
 var import_node_net = require("net");
 var stringOutput = (value) => typeof value === "string" ? value : "";
 var isCommandNotFound = (error) => error instanceof Error && ("code" in error && error.code === "ENOENT" || error.message.includes("ENOENT"));
@@ -3139,7 +3439,7 @@ var defaultStudioRuntime = {
   run,
   portAvailable,
   now: () => (/* @__PURE__ */ new Date()).toISOString(),
-  random: (bytes) => (0, import_node_crypto2.randomBytes)(bytes).toString("base64url"),
+  random: (bytes) => (0, import_node_crypto3.randomBytes)(bytes).toString("base64url"),
   request: fetch,
   log: (message = "") => console.log(message),
   error: (message = "") => console.error(message)
@@ -3675,7 +3975,7 @@ var readPackageName = async (cwd) => {
   let current = cwd;
   for (; ; ) {
     try {
-      const raw = await (0, import_promises5.readFile)((0, import_node_path6.join)(current, "package.json"), "utf8");
+      const raw = await (0, import_promises6.readFile)((0, import_node_path6.join)(current, "package.json"), "utf8");
       const parsed = JSON.parse(raw);
       if (isRecord(parsed) && typeof parsed.name === "string") {
         return parsed.name;
@@ -3692,7 +3992,7 @@ var loadEnvFiles = async (cwd) => {
     const path = (0, import_node_path6.join)(cwd, filename);
     let raw;
     try {
-      raw = await (0, import_promises5.readFile)(path, "utf8");
+      raw = await (0, import_promises6.readFile)(path, "utf8");
     } catch {
       continue;
     }
@@ -3732,12 +4032,12 @@ var resolveOptions = async (options) => {
   };
 };
 var createFrameworkMarkerStubs = async () => {
-  const root = await (0, import_promises5.mkdtemp)((0, import_node_path6.join)((0, import_node_os3.tmpdir)(), "kortyx-cli-stubs-"));
+  const root = await (0, import_promises6.mkdtemp)((0, import_node_path6.join)((0, import_node_os3.tmpdir)(), "kortyx-cli-stubs-"));
   for (const moduleName of ["server-only", "client-only"]) {
     const directory = (0, import_node_path6.join)(root, moduleName);
-    await (0, import_promises5.mkdir)(directory, { recursive: true });
-    await (0, import_promises5.writeFile)((0, import_node_path6.join)(directory, "index.js"), "", "utf8");
-    await (0, import_promises5.writeFile)(
+    await (0, import_promises6.mkdir)(directory, { recursive: true });
+    await (0, import_promises6.writeFile)((0, import_node_path6.join)(directory, "index.js"), "", "utf8");
+    await (0, import_promises6.writeFile)(
       (0, import_node_path6.join)(directory, "package.json"),
       JSON.stringify({ name: moduleName, version: "0.0.0", main: "index.js" }),
       "utf8"
@@ -3752,7 +4052,7 @@ var createFrameworkMarkerStubs = async () => {
       if (previousNodePath === void 0) delete process.env.NODE_PATH;
       else process.env.NODE_PATH = previousNodePath;
       moduleWithPrivateInit._initPaths?.();
-      await (0, import_promises5.rm)(root, { recursive: true, force: true });
+      await (0, import_promises6.rm)(root, { recursive: true, force: true });
     }
   };
 };
@@ -3794,7 +4094,7 @@ var findNearestFile = async (startDirectory, filename) => {
   for (; ; ) {
     const candidate = (0, import_node_path6.join)(current, filename);
     try {
-      await (0, import_promises5.access)(candidate);
+      await (0, import_promises6.access)(candidate);
       return candidate;
     } catch {
       const parent = (0, import_node_path6.dirname)(current);
@@ -3806,7 +4106,7 @@ var findNearestFile = async (startDirectory, filename) => {
 var registerTsconfigPaths = async (tsconfigPath) => {
   const parsed = import_typescript2.default.parseConfigFileTextToJson(
     tsconfigPath,
-    await (0, import_promises5.readFile)(tsconfigPath, "utf8")
+    await (0, import_promises6.readFile)(tsconfigPath, "utf8")
   );
   if (parsed.error)
     throw new Error(`Invalid TypeScript configuration: ${tsconfigPath}`);
@@ -4093,7 +4393,7 @@ var main = async () => {
       console.error(
         process.argv.includes("--json") ? JSON.stringify(error.toJSON()) : `[${error.code}] ${error.message}`
       );
-      process.exitCode = 1;
+      process.exitCode ??= 1;
       return;
     }
     const failure = (0, import_errors.serializeFailure)(error);