@kortyx/cli 0.13.0 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.14.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.13.0...cli-v0.14.0) (2026-10-07)
+
+
+### Features
+
+* **evals:** group suite runs and expose CLI results ([#284](https://github.com/kortyx-io/kortyx/issues/284)) ([146fd71](https://github.com/kortyx-io/kortyx/commit/146fd71a88587e5591dd0511a4c0cb57969f9d07))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @kortyx/agent bumped to 0.31.0
+
 ## [0.13.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.12.0...cli-v0.13.0) (2026-10-07)
 
 

package/dist/index.js

@@ -1430,7 +1430,9 @@ var parseEvalRunTarget = (input) => {
     } catch {
       throw new StudioReadError("invalid_target", "Invalid eval run URL.");
     }
-    const match = locator.pathname.match(/\/evals\/runs\/([^/]+)\/?$/);
+    const match = locator.pathname.match(
+      /\/evals\/(?:runs|evaluations)\/([^/]+)\/?$/
+    );
     if (!match || !["http:", "https:"].includes(locator.protocol) || locator.username || locator.password)
       throw new StudioReadError(
         "invalid_target",
@@ -1446,6 +1448,59 @@ var parseEvalRunTarget = (input) => {
   return { id, url };
 };
 var StudioEvalClient = class extends StudioApiTransport {
+  evaluations() {
+    return this.requestJson(
+      "GET",
+      "/v1/studio/evals/evaluations",
+      import_evals.StudioEvaluationHistorySchema
+    );
+  }
+  evaluation(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "GET",
+      `/v1/studio/evals/evaluations/${id}`,
+      import_evals.StudioEvaluationDetailSchema
+    );
+  }
+  evaluationResults(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "GET",
+      `/v1/studio/evals/evaluations/${id}/results`,
+      import_evals.StudioEvaluationResultsSchema
+    );
+  }
+  startEvaluation(input) {
+    const parsed = import_evals.StudioEvaluationStartRequestSchema.safeParse(input);
+    if (!parsed.success)
+      throw new StudioReadError(
+        "invalid_eval_request",
+        "Invalid evaluation selection or execution limits."
+      );
+    return this.requestJson(
+      "POST",
+      "/v1/studio/evals/evaluations",
+      import_zod3.z.object({ id: import_zod3.z.uuid() }),
+      {},
+      parsed.data
+    );
+  }
+  cancelEvaluation(id) {
+    this.validateId(id);
+    return this.requestJson(
+      "POST",
+      `/v1/studio/evals/evaluations/${id}/cancel`,
+      import_zod3.z.object({ ok: import_zod3.z.literal(true) })
+    );
+  }
+  validateId(id) {
+    if (!import_zod3.z.uuid().safeParse(id).success)
+      throw new StudioReadError(
+        "invalid_target",
+        "Expected an evaluation UUID."
+      );
+  }
   targets() {
     return this.requestJson(
       "GET",
@@ -1961,11 +2016,65 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
       options
     );
   });
+  const terminal = (status) => status !== "queued" && status !== "running";
+  const resultExitCode = (status) => status === "passed" ? 0 : status === "failed" ? 1 : status === "cancelled" ? 130 : 2;
+  const waitForEvaluation = async (client, id, timeout) => {
+    const deadline = Date.now() + timeout * 1e3;
+    for (; ; ) {
+      const { run: run2 } = await client.evaluation(id);
+      if (terminal(run2.status)) return run2;
+      if (Date.now() >= deadline)
+        throw new StudioReadError(
+          "eval_wait_timeout",
+          `Evaluation ${id} is still running. Inspect it with evals runs get; waiting did not cancel it.`
+        );
+      await new Promise(
+        (resolve6) => setTimeout(resolve6, Math.min(2e3, deadline - Date.now()))
+      );
+    }
+  };
+  const readEvaluation = async (client, id, includeContent) => {
+    const { run: run2 } = await client.evaluationResults(id);
+    return includeContent ? run2 : { ...run2, suites: run2.suites.map(compactDetail) };
+  };
+  const waitOptions = (command) => command.option(
+    "--timeout <seconds>",
+    "Maximum time to wait (1\u20137200 seconds); does not cancel execution.",
+    integer2(7200),
+    1800
+  );
   const runs = evals.command("runs").description("Start and inspect saved eval runs.");
   selectionOptions(
-    runs.command("start <suite-id>").description(
-      "Enqueue once and return immediately; does not wait for grades."
+    runs.command("start [suite-id]").description(
+      "Start an evaluation containing all or selected suites; optionally wait for results."
     )
+  ).option("--all", "Run every suite registered on the selected application.").option(
+    "--suite <id>",
+    "Select a suite; repeat for several suites.",
+    (value, previous) => [...previous, value],
+    []
+  ).option("--name <name>", "Display name for this evaluation run.").option(
+    "--source <source>",
+    "Trigger: manual, deployment, schedule or ci.",
+    (value) => {
+      if (!["manual", "deployment", "schedule", "ci"].includes(value))
+        throw new import_commander3.InvalidArgumentError(
+          "Expected manual, deployment, schedule or ci."
+        );
+      return value;
+    },
+    "manual"
+  ).option("--commit <sha>", "Deployed application commit.").option("--deployment-url <url>", "Deployment or CI job URL.").option(
+    "--idempotency-key <key>",
+    "Reuse a matching saved evaluation after a trigger retry."
+  ).option(
+    "--wait",
+    "Wait for final results and return a pass/fail exit code."
+  ).option(
+    "--timeout <seconds>",
+    "Maximum wait in seconds (1\u20137200); execution continues after timeout.",
+    integer2(7200),
+    1800
   ).option(
     "--case <id>",
     "Select a case; repeat for multiple cases.",
@@ -1986,39 +2095,102 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
     },
     "studio"
   ).action(async (suiteId, options) => {
-    const { connection, client, target, suite, canRun } = await select(
-      suiteId,
-      options
+    const ids = [...suiteId ? [suiteId] : [], ...options.suite ?? []];
+    if (!options.all && !ids.length || options.all && ids.length || new Set(ids).size !== ids.length)
+      throw new StudioReadError(
+        "invalid_suite_selection",
+        "Choose --all or unique suite IDs (positional or repeated --suite)."
+      );
+    const data = await discover(options);
+    const matches = data.targets.filter(
+      (target2) => target2.manifest && (options.all || ids.every(
+        (id) => target2.manifest.suites.some((suite) => suite.id === id)
+      ))
     );
-    if (!canRun)
+    const target = matches[0];
+    if (matches.length !== 1 || !target?.manifest)
+      throw new StudioReadError(
+        "invalid_suite_selection",
+        "Select one available application with --target and --environment; all selected suites must belong to it."
+      );
+    if (!data.canRun)
       throw new StudioReadError(
         "eval_forbidden",
         "This Studio key requires eval:run to execute suites."
       );
+    const suites2 = options.all ? target.manifest.suites : ids.map(
+      (id) => target.manifest.suites.find((suite) => suite.id === id)
+    );
     const caseIds = options.case ?? [];
-    if (new Set(caseIds).size !== caseIds.length || caseIds.some((id) => !suite.cases.some((item) => item.id === id)) || (caseIds.length || suite.cases.length) * options.repetitions > 100)
+    if (caseIds.length && (options.all || suites2.length !== 1))
+      throw new StudioReadError(
+        "invalid_case_selection",
+        "--case requires exactly one selected suite."
+      );
+    if (new Set(caseIds).size !== caseIds.length || caseIds.some(
+      (id) => !suites2[0]?.cases.some((item) => item.id === id)
+    ) || suites2.some(
+      (suite) => (caseIds.length || suite.cases.length) * options.repetitions > 100
+    ))
       throw new StudioReadError(
         "invalid_case_selection",
-        "Select unique existing case IDs and at most 100 total attempts."
+        "Select unique existing cases and at most 100 attempts per suite."
       );
-    const run2 = await client.start({
+    const run2 = await data.client.startEvaluation({
       targetId: target.id,
+      selection: options.all ? "all" : "selected",
+      suites: suites2.map((suite) => ({
+        suiteId: suite.id,
+        suiteRevision: target.revisions[suite.id] ?? "",
+        ...caseIds.length ? { caseIds } : {}
+      })),
       judge: options.judge ?? "studio",
-      suiteId,
-      suiteRevision: target.revisions[suiteId] ?? "",
       repetitions: options.repetitions,
       concurrency: options.concurrency,
-      ...caseIds.length ? { caseIds } : {}
-    });
-    print(
-      {
-        connection: connection.name,
-        id: run2.id,
-        status: "queued",
-        studioUrl: connection.studioUrl ? `${connection.studioUrl}/evals/runs/${run2.id}` : null
+      name: options.name,
+      metadata: {
+        source: options.source ?? "manual",
+        commit: options.commit,
+        deploymentUrl: options.deploymentUrl
       },
-      options
-    );
+      idempotencyKey: options.idempotencyKey
+    });
+    const studioUrl = data.connection.studioUrl ? `${data.connection.studioUrl}/evals/evaluations/${run2.id}` : null;
+    if (!options.wait) {
+      print(
+        {
+          connection: data.connection.name,
+          id: run2.id,
+          status: "queued",
+          studioUrl
+        },
+        options
+      );
+      return;
+    }
+    try {
+      const completed = await waitForEvaluation(
+        data.client,
+        run2.id,
+        options.timeout
+      );
+      print(
+        {
+          connection: data.connection.name,
+          studioUrl,
+          run: await readEvaluation(
+            data.client,
+            run2.id,
+            options.includeContent ?? false
+          )
+        },
+        options
+      );
+      process.exitCode = resultExitCode(completed.status);
+    } catch (error) {
+      process.exitCode = 2;
+      throw error;
+    }
   });
   common(
     runs.command("list").description("List the latest 100 saved runs in this project.")
@@ -2027,12 +2199,12 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
     "Filter results; defaults to the connection environment."
   ).action(async (options) => {
     const { connection, client } = await clientFor(options);
-    const data = await client.runs();
+    const data = await client.evaluations();
     const environment = options.environment ?? connection.environment;
     print(
       {
         connection: connection.name,
-        runs: data.runs.filter((run2) => !environment || run2.environment === environment).map(summary)
+        runs: data.runs.filter((run2) => !environment || run2.environment === environment).map((run2) => run2)
       },
       options
     );
@@ -2044,15 +2216,59 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
   ).action(async (input, options) => {
     const target = parseEvalRunTarget(input);
     const { connection, client } = await clientFor(options, target.url);
-    const { run: run2 } = await client.run(target.id);
+    let run2;
+    try {
+      run2 = await readEvaluation(
+        client,
+        target.id,
+        options.includeContent ?? false
+      );
+    } catch (error) {
+      if (!(error instanceof StudioReadError) || error.status !== 404)
+        throw error;
+      const legacy = await client.run(target.id);
+      run2 = options.includeContent ? legacy.run : compactDetail(legacy.run);
+    }
     print(
       {
         connection: connection.name,
-        run: options.includeContent ? run2 : compactDetail(run2)
+        run: run2
       },
       options
     );
   });
+  waitOptions(
+    common(
+      runs.command("wait <id-or-url>").description(
+        "Wait for a grouped evaluation and print its final suite/case results."
+      )
+    )
+  ).action(async (input, options) => {
+    const target = parseEvalRunTarget(input);
+    const { connection, client } = await clientFor(options, target.url);
+    try {
+      const completed = await waitForEvaluation(
+        client,
+        target.id,
+        options.timeout
+      );
+      print(
+        {
+          connection: connection.name,
+          run: await readEvaluation(
+            client,
+            target.id,
+            options.includeContent ?? false
+          )
+        },
+        options
+      );
+      process.exitCode = resultExitCode(completed.status);
+    } catch (error) {
+      process.exitCode = 2;
+      throw error;
+    }
+  });
   common(
     runs.command("cancel <id-or-url>").description(
       "Request cooperative cancellation of a queued or running eval."
@@ -2060,7 +2276,14 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
   ).action(async (input, options) => {
     const target = parseEvalRunTarget(input);
     const { connection, client } = await clientFor(options, target.url);
-    const data = await client.cancel(target.id);
+    let data;
+    try {
+      data = await client.cancelEvaluation(target.id);
+    } catch (error) {
+      if (!(error instanceof StudioReadError) || error.status !== 404)
+        throw error;
+      data = await client.cancel(target.id);
+    }
     print({ connection: connection.name, id: target.id, ...data }, options);
   });
 }
@@ -4093,7 +4316,7 @@ var main = async () => {
       console.error(
         process.argv.includes("--json") ? JSON.stringify(error.toJSON()) : `[${error.code}] ${error.message}`
       );
-      process.exitCode = 1;
+      process.exitCode ??= 1;
       return;
     }
     const failure = (0, import_errors.serializeFailure)(error);