@kortyx/cli 0.12.0 → 0.13.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [0.13.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.12.0...cli-v0.13.0) (2026-10-07)
+
+
+### Features
+
+* **api:** add typed security and tenant database extensions ([#260](https://github.com/kortyx-io/kortyx/issues/260)) ([47e4928](https://github.com/kortyx-io/kortyx/commit/47e49286f12e4470b3a327e609a65c99678b359f))
+* **evals:** diagnose setup and guide first workflow runs ([#275](https://github.com/kortyx-io/kortyx/issues/275)) ([01dbf80](https://github.com/kortyx-io/kortyx/commit/01dbf803e65eec03187de9eaa316e4294f4f8dc5))
+
+
+### Bug Fixes
+
+* **security:** remediate production dependency risks ([#267](https://github.com/kortyx-io/kortyx/issues/267)) ([319daa1](https://github.com/kortyx-io/kortyx/commit/319daa1716a8684c332842bc6b6e9bb9474219e7))
+* **studio:** adopt native Drizzle migrations safely ([98238a7](https://github.com/kortyx-io/kortyx/commit/98238a770a900017dd4956f1afc56fddba21d371))
+* **studio:** preserve legacy updater access ([#283](https://github.com/kortyx-io/kortyx/issues/283)) ([fb733d8](https://github.com/kortyx-io/kortyx/commit/fb733d848ac7b6e42851de810aa2f039abb535b2))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @kortyx/agent bumped to 0.30.0
+    * @kortyx/telemetry-contracts bumped to 0.14.0
+
 ## [0.12.0](https://github.com/kortyx-io/kortyx/compare/cli-v0.11.4...cli-v0.12.0) (2026-10-03)
 
 

package/README.md

@@ -447,3 +447,17 @@ Studio draws discovered call/return links before traffic exists. The **Observed
 `kortyx topology push` discovers shared tool definitions attached via `useTool({tool, input})` and `useReason({tools})` through local imports, custom hooks, and statically bound factories. Discovery does not execute nodes, tool factories or MCP discovery. The configured entry is still imported to obtain the workflow registry.
 
 Published node capabilities contain names, descriptions, calling mode, safe input-field summaries and discovery freshness. Dynamic attachments produce an unresolved warning rather than an empty-tools claim. Studio merges real observed tools and shows execution outcomes/durations separately from cached reuse. `--dry-run --json` exposes the discovered attachments and status without publishing.
+
+## Diagnose eval setup
+
+```sh
+kortyx studio evals doctor --connection staging --target catalog --suite catalog-smoke
+```
+
+Checks Studio access, execution permission, target/environment, authenticated
+consumer manifest/suites and advertised judge compatibility. Discovery GET only;
+no workflows, model calls or saved runs are started. Consumer GET wrappers may
+authenticate a test actor. Failures include actionable remedies. Use `--judge app`
+for a code judge or `--json` for a versioned report; failures exit 1. A successful
+check still needs a representative run to verify tool/provider access and saved
+results. See the [first eval guide](https://kortyx.io/docs/studio/first-eval).

package/dist/index.js

@@ -257,6 +257,7 @@ services:
   db-init:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
+    user: "1000:1000"
     environment:
       <<: [*api-env, *bootstrap-keys]
     command: >
@@ -270,6 +271,7 @@ services:
   api:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
+    user: "1000:1000"
     environment:
       <<: *api-env
       NODE_ENV: production
@@ -329,6 +331,9 @@ services:
     image: \${KORTYX_API_IMAGE_REF:-\${KORTYX_API_IMAGE:-ghcr.io/kortyx-io/kortyx-api}:\${KORTYX_STUDIO_IMAGE_TAG:-latest}}
     pull_policy: \${KORTYX_STUDIO_PULL_POLICY:-always}
     restart: unless-stopped
+    # The updater alone needs root for the host Docker socket and ownership-safe
+    # writes to the bind-mounted Studio state directory.
+    user: "0:0"
     command: ["node", "apps/api/dist/updater.js", "serve", "\${KORTYX_STUDIO_STATE_DIR}"]
     volumes:
       - type: bind
@@ -832,7 +837,7 @@ var createConnectionsCommand = (log = console.log, request = fetch) => {
         "/v1/studio/context",
         import_telemetry_contracts2.StudioContextResponseSchema
       );
-      if (!context.apiKey.scopes.includes("studio:read"))
+      if (!context.apiKey?.scopes.includes("studio:read"))
         throw new StudioReadError(
           "missing_scope",
           "The API key lacks studio:read permission."
@@ -1490,6 +1495,162 @@ var StudioEvalClient = class extends StudioApiTransport {
   }
 };
 
+// src/studio/eval-doctor.ts
+var advice = {
+  environment_forbidden: {
+    message: "Target environment is not allowed in this Studio project.",
+    remedy: "Allow the target environment in the project, or correct the target's environment label."
+  },
+  environment_unavailable: {
+    message: "Studio could not check the target environment.",
+    remedy: "Inspect Studio API database connectivity and private server logs."
+  },
+  endpoint_not_found: {
+    message: "Application endpoint returned HTTP 404.",
+    remedy: "Ensure the eval route is mounted and enabled in the deployed application; check the target URL and reverse proxy route."
+  },
+  endpoint_unauthorized: {
+    message: "Application endpoint rejected discovery credentials.",
+    remedy: "Match the consumer handler service key to the Studio target. If the app authenticates a test actor during discovery, also check that actor's credentials and permissions."
+  },
+  endpoint_http_error: {
+    message: "Application endpoint returned an unsuccessful HTTP status.",
+    remedy: "Inspect the consumer and proxy logs. Check deployment configuration and any app-owned test-identity initialization."
+  },
+  endpoint_unreachable: {
+    message: "Studio API could not fetch the application manifest.",
+    remedy: "Check API-to-consumer networking, DNS, TLS and timeouts. From Docker Desktop use host.docker.internal for a host app; redirects are not followed."
+  },
+  manifest_invalid: {
+    message: "Application returned an empty, oversized or incompatible manifest.",
+    remedy: "Mount createEvalRouteHandler on the exact target URL and use compatible SDK/Studio releases. Check whether a proxy returned HTML instead of JSON."
+  }
+};
+function buildEvalDoctorReport(data, options) {
+  const checks = [
+    {
+      id: "studio_access",
+      status: "passed",
+      message: "Authenticated Studio discovery (studio:read)."
+    },
+    {
+      id: "execution_permission",
+      status: data.canRun ? "passed" : "failed",
+      message: data.canRun ? "Studio key has eval:run." : "Studio key lacks eval:run.",
+      ...!data.canRun ? {
+        remedy: "Grant eval:run to this project key. For local bootstrap rerun with KORTYX_STUDIO_ENABLE_EVALS=1 and the existing stored key."
+      } : {}
+    }
+  ];
+  const targets = data.targets.filter(
+    (target) => (!options.target || target.id === options.target) && (!options.environment || target.environment === options.environment)
+  );
+  checks.push({
+    id: "target_selection",
+    status: targets.length ? "passed" : "failed",
+    message: targets.length ? `${targets.length} matching application target(s).` : "No application target matches this connection and selection.",
+    ...!targets.length ? {
+      remedy: "Register the target with this key's organization/project and environment. Mount KORTYX_EVAL_TARGETS_FILE on the Studio API, restart it, and check --target/--environment and the connection's default environment."
+    } : {}
+  });
+  for (const target of targets) {
+    const prefix = `${target.id} (${target.environment})`;
+    const diagnostic = target.diagnostic;
+    const environmentFailed = diagnostic?.code === "environment_forbidden" || diagnostic?.code === "environment_unavailable";
+    checks.push({
+      id: `${target.id}:environment`,
+      status: environmentFailed ? "failed" : target.manifest || diagnostic ? "passed" : "skipped",
+      message: `${prefix}: ${environmentFailed ? advice[diagnostic.code].message : target.manifest || diagnostic ? "target environment allowed." : "environment check unavailable on this API."}`,
+      ...environmentFailed ? { remedy: advice[diagnostic.code].remedy } : {}
+    });
+    if (!target.manifest) {
+      const guidance = diagnostic && !environmentFailed ? advice[diagnostic.code] : void 0;
+      checks.push({
+        id: `${target.id}:manifest`,
+        status: environmentFailed ? "skipped" : "failed",
+        ...diagnostic ? { code: diagnostic.code } : {},
+        message: `${prefix}: ${environmentFailed ? "discovery skipped because the environment check failed." : guidance?.message ?? "consumer unavailable; this API did not supply a failure category."}${diagnostic?.httpStatus ? ` (HTTP ${diagnostic.httpStatus})` : ""}`,
+        ...!environmentFailed ? {
+          remedy: guidance?.remedy ?? "Check the deployed consumer route, matching service key and API-to-consumer reachability. Upgrade the Studio API for detailed discovery diagnostics."
+        } : {}
+      });
+      checks.push({
+        id: `${target.id}:judge`,
+        status: "skipped",
+        message: `${prefix}: judge compatibility cannot be checked without a manifest.`
+      });
+      continue;
+    }
+    const suites = target.manifest.suites;
+    const selectedSuite = options.suite ? suites.find((suite) => suite.id === options.suite) : void 0;
+    const suitesReady = options.suite ? Boolean(selectedSuite) : suites.length > 0;
+    checks.push({
+      id: `${target.id}:manifest`,
+      status: suitesReady ? "passed" : "failed",
+      message: `${prefix}: authenticated manifest valid; ${suites.length} registered suite(s).${options.suite ? ` Requested suite ${options.suite} ${selectedSuite ? "found" : "missing"}.` : ""}`,
+      ...!suitesReady ? {
+        remedy: "Pass the intended suite to createEvals({ suites: [...] }), deploy the application and refresh discovery. Suites are fetched from the consumer; topology publication does not register them."
+      } : {}
+    });
+    const judgeReady = options.judge === "studio" ? Boolean(data.studioJudge && target.manifest.studioJudging) : Boolean(target.manifest.judge);
+    checks.push({
+      id: `${target.id}:judge`,
+      status: judgeReady ? "passed" : "failed",
+      message: `${prefix}: ${options.judge} judge ${judgeReady ? "configured and advertised" : "unavailable"}.`,
+      ...!judgeReady ? {
+        remedy: options.judge === "app" ? "Provide a code judge to createEvals in the consumer, or explicitly select --judge studio." : !data.studioJudge ? "Configure KORTYX_EVAL_JUDGE_MODEL and KORTYX_EVAL_JUDGE_API_KEY on the Studio API and restart it, or explicitly select an available --judge app." : "Update the consumer SDK to advertise Studio judging support, or select an available --judge app."
+      } : {}
+    });
+  }
+  return {
+    schemaVersion: 1,
+    status: checks.some((check) => check.status === "failed") ? "failed" : "passed",
+    checks
+  };
+}
+function evalDoctorFailure(error) {
+  const safe = error instanceof StudioReadError ? error : null;
+  const connectionError = safe && [
+    "invalid_config",
+    "invalid_connection",
+    "not_configured",
+    "unknown_connection",
+    "invalid_key_env",
+    "missing_key",
+    "invalid_key",
+    "invalid_url",
+    "incompatible_studio_api",
+    "schema_mismatch",
+    "connection_failed"
+  ].includes(safe.code);
+  return {
+    schemaVersion: 1,
+    status: "failed",
+    checks: [
+      {
+        id: "studio_access",
+        status: "failed",
+        ...safe ? { code: safe.code } : {},
+        message: connectionError ? safe.message : safe?.status === 401 ? "Studio rejected the project key (HTTP 401)." : safe?.status === 403 ? "Studio discovery requires studio:read (HTTP 403)." : safe?.status === 404 ? "Studio eval discovery endpoint was not found (HTTP 404)." : "Could not resolve the Studio connection or read a compatible discovery response.",
+        remedy: "Check the selected connection, its API URL and project key, network access, and compatible CLI/Studio releases. No consumer or provider credentials belong in this CLI connection."
+      }
+    ]
+  };
+}
+function formatEvalDoctorReport(report) {
+  const symbols = { passed: "\u2713", failed: "\u2717", skipped: "\u2013" };
+  return [
+    "Kortyx Evals \xB7 Setup check",
+    ...report.checks.flatMap((check) => [
+      `  ${symbols[check.status]} ${check.message}`,
+      ...check.remedy ? [`    \u2192 ${check.remedy}`] : []
+    ]),
+    "",
+    report.status === "passed" ? "Configuration checks passed. Run one representative suite to verify the test identity, tools, model access and saved results." : "Setup checks failed. Resolve the failures and run doctor again.",
+    "No workflow or judge calls were started. Consumer GET discovery may run app-owned authentication logic."
+  ].join("\n").replace(/\p{Cc}/gu, (character) => character === "\n" ? character : "");
+}
+
 // src/studio/read-output.ts
 var parseStudioTarget = (input, entity) => {
   if (/^[a-z][a-z0-9+.-]*:/i.test(input)) {
@@ -1709,6 +1870,36 @@ function registerStudioEvalCommands(studio, log, request = fetch) {
     "--environment <name>",
     "Filter targets; defaults to the connection environment."
   );
+  selectionOptions(
+    evals.command("doctor").description(
+      "Check deployment wiring without starting workflows or model calls."
+    )
+  ).option("--suite <id>", "Check that a specific suite is registered.").option(
+    "--judge <location>",
+    "Judge to check: studio (default) or app.",
+    (value) => {
+      if (value !== "studio" && value !== "app")
+        throw new import_commander3.InvalidArgumentError("Expected studio or app.");
+      return value;
+    },
+    "studio"
+  ).action(async (options) => {
+    let report;
+    try {
+      const { connection, client } = await clientFor(options);
+      report = buildEvalDoctorReport(await client.targets(), {
+        ...options,
+        environment: options.environment ?? connection.environment,
+        judge: options.judge ?? "studio"
+      });
+    } catch (error) {
+      report = evalDoctorFailure(error);
+    }
+    log(
+      options.json ? JSON.stringify(report) : formatEvalDoctorReport(report)
+    );
+    if (report.status === "failed") process.exitCode = 1;
+  });
   selectionOptions(
     suites.command("list").description("List targets, suites, revisions and case IDs.")
   ).action(async (options) => {