activeagent 1.9.3 → 1.9.5

data/lib/active_agent/cli/eval_commands.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module ActiveAgent
+  class CLI
+    # `activeagent evals ...`. list, runs and run go through the MCP facade's
+    # evaluation tools; import posts a version-1 report to the collector with
+    # ActiveAgent::Evals::Publisher; report reads one back from the
+    # collector's export endpoint.
+    module EvalCommands
+      # How `evals report` asks GET <mount>/api/evaluation_reports/:run_id
+      # for each format. The JSON form is the version-1 envelope the
+      # collector takes, so `evals report --format json` then `evals import`
+      # copies a run between installs.
+      REPORT_FORMATS = {
+        "json" => { accept: "application/json", query: nil },
+        "html" => { accept: "text/html", query: "format=html" },
+        # GitHub-flavored Markdown, for a pull request comment:
+        # `activeagent evals report RUN --format md | gh pr comment 12 --body-file -`
+        "md" => { accept: "text/markdown", query: "format=md" }
+      }.freeze
+
+      RUN_FINISHED = %w[complete failed].freeze
+      DEFAULT_WAIT_TIMEOUT = 30 * 60
+      DEFAULT_POLL_INTERVAL = 5
+      FAILURES_SHOWN = 20
+
+      private
+
+      def evals_list(argv)
+        arguments = {}
+        argv = parse(argv, "evals list [--agent SLUG] [--limit N]") do |p|
+          p.on("--agent AGENT", "Only this agent's evaluations, by slug or id") { |v| arguments[:agent] = v }
+          p.on("--limit N", "At most N evaluations (max 50)") { |v| arguments[:limit] = positive_integer(v, "--limit") }
+        end
+        return EXIT_SUCCESS unless argv
+
+        evaluations = client.tool("evaluations_list", arguments).fetch("evaluations", [])
+        emit(evaluations: evaluations) do
+          table(evaluations, {
+            "ID" => ->(e) { e["id"] },
+            "NAME" => ->(e) { e["name"] },
+            "AGENT" => ->(e) { e.dig("agent", "slug") },
+            "SCENARIOS" => ->(e) { e["scenario_count"] },
+            "RUNS" => ->(e) { e["run_count"] },
+            "LATEST" => ->(e) { e.dig("latest_run", "status") },
+            "PASSED" => ->(e) { passed(e["latest_run"]) }
+          }, empty: "No evaluations.")
+        end
+      end
+
+      def evals_runs(argv)
+        argv = parse(argv, "evals runs EVALUATION")
+        return EXIT_SUCCESS unless argv
+
+        evaluation_id = resolve_evaluation(require_argument(argv, "EVALUATION (an id or name)", "evals runs EVALUATION"))
+        payload = client.tool("evaluations_get", evaluation_id: evaluation_id)
+        runs = payload.fetch("runs", [])
+        emit(evaluation: payload["evaluation"], runs: runs) do
+          @out.puts "#{payload.dig("evaluation", "name")} (evaluation #{evaluation_id})"
+          table(runs, {
+            "RUN" => ->(r) { r["id"] },
+            "#" => ->(r) { r["number"] },
+            "STATUS" => ->(r) { r["status"] },
+            "PASSED" => ->(r) { passed(r) },
+            "SCORE" => ->(r) { r["average_score"] },
+            "CREATED" => ->(r) { r["created_at"] },
+            # The handle `evals report` takes.
+            "RUN_ID" => ->(r) { r["run_id"] }
+          }, empty: "No runs yet.")
+        end
+      end
+
+      # Starts a run. With --wait (implied by --min-pass-rate), polls it to
+      # the end and exits EXIT_GATE when it failed or passed fewer scenarios
+      # than the threshold, so a CI step fails the build.
+      def evals_run(argv)
+        arguments = {}
+        wait = false
+        min_pass_rate = nil
+        timeout = DEFAULT_WAIT_TIMEOUT
+        interval = DEFAULT_POLL_INTERVAL
+        argv = parse(argv, "evals run EVALUATION [--models a,b] [--wait] [--min-pass-rate 0.8]") do |p|
+          p.on("--models LIST", "Candidate models to compare, comma-separated") { |v| arguments[:models] = list_option(v) }
+          p.on("--keys LIST", "Replay only these scenario keys, comma-separated") { |v| arguments[:keys] = list_option(v) }
+          p.on("--group GROUP", "Replay only this scenario group") { |v| arguments[:group] = v }
+          p.on("--sandbox ID", "Replay against this ready checkout sandbox") { |v| arguments[:sandbox_id] = v }
+          p.on("--wait", "Wait for the run to finish") { wait = true }
+          p.on("--min-pass-rate RATE", "Fail (exit #{EXIT_GATE}) below this pass rate: 0.8 or 80%; implies --wait") do |v|
+            min_pass_rate = pass_rate_option(v)
+            wait = true
+          end
+          p.on("--timeout SECONDS", "Give up waiting after this long (default #{DEFAULT_WAIT_TIMEOUT})") { |v| timeout = positive_integer(v, "--timeout") }
+          p.on("--interval SECONDS", "Poll this often (default #{DEFAULT_POLL_INTERVAL})") { |v| interval = positive_integer(v, "--interval") }
+        end
+        return EXIT_SUCCESS unless argv
+
+        evaluation_id = resolve_evaluation(require_argument(argv, "EVALUATION (an id or name)", "evals run EVALUATION"))
+        started = client.tool("evaluations_run", arguments.merge(evaluation_id: evaluation_id))
+        run = started.fetch("run", {})
+        @err.puts "Started run #{run["id"]} of #{started.dig("evaluation", "name")} (evaluation #{evaluation_id})." unless @options[:json]
+        unless wait
+          return emit(started) { @out.puts "Status: #{run["status"]}. Follow it with `activeagent evals runs #{evaluation_id}`." }
+        end
+
+        finished = wait_for_run(evaluation_id, run["id"], timeout: timeout, interval: interval)
+        gate(finished, min_pass_rate)
+      end
+
+      # Without --format, a terminal gets Markdown to read and anything else
+      # (a pipe, a file, -o) gets the JSON envelope, so scripts see JSON.
+      # RUN is the run's run_id: the RUN_ID column of `evals runs`, which is
+      # a published run's own run_id and a dashboard run's
+      # dashboard-run-<id>-<created_at>.
+      def evals_report(argv)
+        format = nil
+        output = nil
+        argv = parse(argv, "evals report RUN [--format json|html|md] [-o FILE]") do |p|
+          p.on("--format FORMAT", REPORT_FORMATS.keys,
+               "json (a version-1 envelope), html or md; default md on a terminal, json otherwise") { |v| format = v }
+          p.on("-o", "--output FILE", "Write to FILE instead of standard output") { |v| output = v }
+        end
+        return EXIT_SUCCESS unless argv
+
+        run_id = require_argument(argv, "RUN (the run's run_id, the RUN_ID of `evals runs`)", "evals report RUN")
+        format ||= !output && @out.respond_to?(:tty?) && @out.tty? ? "md" : "json"
+        request = REPORT_FORMATS.fetch(format)
+        response = client.get("/api/evaluation_reports/#{URI.encode_www_form_component(run_id).gsub("+", "%20")}",
+                              accept: request[:accept], query: request[:query],
+                              missing: "No report for run #{run_id}; `activeagent evals runs EVALUATION` shows each run's RUN_ID",
+                              unavailable: "This dashboard has no report export (GET <mount>/api/evaluation_reports/:run_id); " \
+                                           "it predates actionagent 1.9.3")
+        body = response.body.to_s
+        body = JSON.pretty_generate(report_envelope(body)) if format == "json"
+        write_output(body, output) { "Wrote #{format} report for run #{run_id} to #{output}." }
+      end
+
+      # Posts a saved report to the collector. FILE is a version-1 envelope
+      # (what `evals report --format json` writes) or a bare Report#to_h
+      # hash, whose identities then come from the options.
+      def evals_import(argv)
+        identities = {}
+        argv = parse(argv, "evals import FILE [--run-id ID --source NAME --agent NAME --suite NAME]") do |p|
+          p.on("--run-id ID", "The report's run_id (kept across retries)") { |v| identities["run_id"] = v }
+          p.on("--source NAME", "The application that ran it") { |v| identities["source"] = v }
+          p.on("--agent NAME", "The agent's name") { |v| identities["agent_name"] = v }
+          p.on("--suite NAME", "The suite's name") { |v| identities["suite"] = v }
+        end
+        return EXIT_SUCCESS unless argv
+
+        path = require_argument(argv, "FILE", "evals import FILE")
+        document = read_json_file(path)
+        envelope = document.key?("version") && document.key?("report") ? document : { "report" => document }
+        if envelope.key?("version") && envelope["version"] != 1
+          raise Error.new("#{path} is a version #{envelope["version"].inspect} envelope; this CLI sends version 1", exit_code: EXIT_USAGE)
+        end
+        envelope = envelope.merge(identities)
+        envelope["run_id"] ||= envelope["report"].is_a?(Hash) ? envelope["report"].dig("metadata", "run_id") : nil
+
+        receipt = client.publisher.call(report: envelope["report"], run_id: envelope["run_id"], source: envelope["source"],
+                                        agent_name: envelope["agent_name"], suite: envelope["suite"])
+        emit(receipt) do
+          @out.puts "#{receipt["duplicate"] ? "Already stored" : "Imported"} run #{receipt["run_id"]} " \
+                    "(evaluation #{receipt["evaluation_id"]}, run #{receipt["id"]})."
+        end
+      rescue ArgumentError => e
+        raise Error.new("#{e.message}; give it with --run-id, --source, --agent or --suite", exit_code: EXIT_USAGE)
+      rescue ActiveAgent::Evals::Publisher::Error => e
+        raise Error.new(e.message, status: e.status)
+      end
+
+      # An evaluation's id, from an id or its exact name among the first 50
+      # evaluations the key can reach.
+      def resolve_evaluation(value)
+        return Integer(value, 10) if value.match?(/\A\d+\z/)
+
+        matches = client.tool("evaluations_list", limit: 50).fetch("evaluations", []).select { |e| e["name"] == value }
+        case matches.size
+        when 1 then matches.first["id"]
+        when 0 then raise Error, "No evaluation named #{value.inspect}; `activeagent evals list` shows them"
+        else
+          raise Error.new("#{matches.size} evaluations are named #{value.inspect} (ids #{matches.map { |e| e["id"] }.join(", ")}); " \
+                          "pass the id", exit_code: EXIT_USAGE)
+        end
+      end
+
+      def wait_for_run(evaluation_id, run_id, timeout:, interval:)
+        deadline = @clock.call + timeout
+        loop do
+          payload = client.tool("evaluation_runs_get", evaluation_id: evaluation_id, run_id: run_id, failed_only: true, limit: FAILURES_SHOWN)
+          return payload if RUN_FINISHED.include?(payload.dig("run", "status"))
+
+          if @clock.call >= deadline
+            raise Error, "Run #{run_id} is still #{payload.dig("run", "status")} after #{timeout}s; " \
+                         "raise --timeout or follow it with `activeagent evals runs #{evaluation_id}`"
+          end
+          @sleeper.call(interval)
+        end
+      end
+
+      def gate(payload, min_pass_rate)
+        run = payload.fetch("run", {})
+        rate = pass_rate(run)
+        failed = run["status"] == "failed"
+        below = min_pass_rate && (rate.nil? || rate < min_pass_rate)
+        verdict = failed || below ? "fail" : "pass"
+        failures = payload.fetch("results", [])
+
+        emit(payload.merge("pass_rate" => rate, "min_pass_rate" => min_pass_rate, "verdict" => verdict)) do
+          @out.puts "Run #{run["id"]} #{run["status"]}: #{passed(run) || "no results"}" \
+                    "#{rate ? format(" (%.1f%%)", rate * 100) : ""}#{min_pass_rate ? format(", threshold %.1f%%", min_pass_rate * 100) : ""}."
+          @out.puts "Error: #{run["error_message"]}" if run["error_message"]
+          unless failures.empty?
+            @out.puts "Failed results:"
+            table(failures, {
+              "SCENARIO" => ->(r) { r["scenario_key"] },
+              "MODEL" => ->(r) { r["model"] },
+              "STATUS" => ->(r) { r["status"] },
+              "FAULT" => ->(r) { r["fault"] },
+              "TRACE" => ->(r) { r["trace_id"] }
+            })
+          end
+          @out.puts "Report: activeagent evals report #{run["run_id"]}" if run["run_id"]
+          @out.puts(verdict == "pass" ? "PASS" : "FAIL")
+        end
+        verdict == "pass" ? EXIT_SUCCESS : EXIT_GATE
+      end
+
+      def pass_rate(run)
+        evaluated = run["samples_evaluated"].to_i
+        evaluated.positive? ? run["samples_passed"].to_f / evaluated : nil
+      end
+
+      def passed(run)
+        return unless run.is_a?(Hash) && run["samples_evaluated"]
+
+        "#{run["samples_passed"].to_i}/#{run["samples_evaluated"]}"
+      end
+
+      def pass_rate_option(value)
+        rate = value.end_with?("%") ? Float(value.delete_suffix("%")) / 100 : Float(value)
+        raise ArgumentError unless rate.between?(0, 1)
+
+        rate
+      rescue ArgumentError, TypeError
+        raise Error.new("--min-pass-rate must be a rate between 0 and 1, or a percentage such as 80%", exit_code: EXIT_USAGE)
+      end
+
+      # The collector's export must be the envelope `evals import` sends back.
+      def report_envelope(body)
+        envelope = JSON.parse(body)
+        unless envelope.is_a?(Hash) && envelope["version"] == 1 && envelope["report"].is_a?(Hash)
+          raise Error, "The dashboard's report is not a version-1 envelope"
+        end
+        envelope
+      rescue JSON::ParserError
+        raise Error.new("The dashboard returned invalid JSON for the report"), cause: nil
+      end
+    end
+
+    include EvalCommands
+  end
+end

data/lib/active_agent/cli/resource_commands.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  class CLI
+    # `activeagent catalogs|agents ...`, through the MCP facade's dashboard
+    # tools: scenario catalogs are listed, imported (or the import
+    # previewed) and exported as the dashboard's Catalogs page does it, and
+    # agents are listed with their versions.
+    module ResourceCommands
+      CATALOG_FORMATS = %w[yaml json].freeze
+      CATALOG_MODES = %w[replace merge].freeze
+
+      private
+
+      def catalogs_list(argv)
+        argv = parse(argv, "catalogs list")
+        return EXIT_SUCCESS unless argv
+
+        catalogs = client.tool("catalogs_list").fetch("catalogs", [])
+        emit(catalogs: catalogs) do
+          table(catalogs, {
+            "KEY" => ->(c) { c["key"] },
+            "NAME" => ->(c) { c["name"] },
+            "PRODUCTS" => ->(c) { c["product_count"] },
+            "SCENARIOS" => ->(c) { c["scenario_count"] },
+            "SOURCE" => ->(c) { c["source_path"] || c["source_kind"] },
+            "UPDATED" => ->(c) { c["updated_at"] }
+          }, empty: "No scenario catalogs.")
+        end
+      end
+
+      # Imports a catalog's document (YAML, or the JSON `catalogs export
+      # --format json` writes, which YAML reads). Products, sets and
+      # scenarios are matched by key, so importing the same file twice
+      # changes nothing; --mode merge keeps what the document leaves out.
+      # --preview shows what the import would change and writes nothing.
+      def catalogs_import(argv)
+        arguments = {}
+        preview = false
+        argv = parse(argv, "catalogs import FILE [--name KEY] [--mode replace|merge] [--preview]") do |p|
+          p.on("--name KEY", "The catalog key when the document sets no `catalog:`") { |v| arguments[:name] = v }
+          p.on("--mode MODE", CATALOG_MODES, "replace (the default) makes the catalog the document; merge keeps what it leaves out") do |v|
+            arguments[:mode] = v
+          end
+          p.on("--preview", "Show what the import would change, without applying it") { preview = true }
+        end
+        return EXIT_SUCCESS unless argv
+
+        path = require_argument(argv, "FILE", "catalogs import FILE")
+        arguments[:document] = read_text_file(path)
+        return catalogs_preview(arguments) if preview
+
+        payload = client.tool("catalogs_import", arguments)
+        emit(payload) do
+          catalog = payload.fetch("catalog", {})
+          @out.puts "Imported catalog #{catalog["key"]} (#{catalog["product_count"]} products, #{catalog["scenario_count"]} scenarios)."
+          Array(payload["products"]).each do |product|
+            sets = Array(product["sets"]).map { |set| "#{set["key"]} (#{set["scenario_count"]})" }
+            @out.puts "  #{product["key"]}#{" -> #{product["agent"]}" if product["agent"]}: #{sets.join(", ")}"
+          end
+        end
+      end
+
+      def catalogs_preview(arguments)
+        payload = client.tool("catalogs_preview", arguments)
+        emit(payload) do
+          catalog = payload.fetch("catalog", {})
+          # The catalog's own action is about its record (name, description);
+          # what its products, sets and scenarios would do is the change list.
+          verdict = { "create" => "created", "update" => "updated" }.fetch(catalog["action"], "kept as it is")
+          @out.puts "Preview (#{payload["mode"]}): catalog #{catalog["key"]} would be #{verdict}; " \
+                    "#{payload["scenario_count"]} scenarios in the document."
+          counts = payload.fetch("counts", {}).reject { |action, _| action == "unchanged" }
+          @out.puts "Changes: #{counts.empty? ? "none" : counts.map { |action, count| "#{count} #{action}" }.join(", ")}"
+          table(Array(payload["changes"]), {
+            "ACTION" => ->(c) { c["action"] },
+            "LEVEL" => ->(c) { c["level"] },
+            "PATH" => ->(c) { c["path"] }
+          }, empty: "Nothing would change.")
+        end
+      end
+
+      # Writes a catalog's canonical document from the dashboard's records:
+      # the whole catalog, or one product or set of it. The same document
+      # the Catalogs page's Export YAML downloads; --format json writes it
+      # as JSON instead.
+      def catalogs_export(argv)
+        arguments = {}
+        output = nil
+        argv = parse(argv, "catalogs export KEY [--product KEY [--set KEY]] [--format yaml|json] [-o FILE]") do |p|
+          p.on("--product KEY", "Only this product") { |v| arguments[:product] = v }
+          p.on("--set KEY", "Only this set of --product") { |v| arguments[:set] = v }
+          p.on("--format FORMAT", CATALOG_FORMATS, "yaml (the default; json when FILE ends in .json) or json") { |v| arguments[:format] = v }
+          p.on("-o", "--output FILE", "Write to FILE instead of standard output") { |v| output = v }
+        end
+        return EXIT_SUCCESS unless argv
+
+        arguments[:catalog] = require_argument(argv, "KEY", "catalogs export KEY")
+        arguments[:format] ||= output.to_s.end_with?(".json") ? "json" : "yaml"
+        payload = client.tool("catalogs_export", arguments)
+        if @options[:json]
+          @out.puts JSON.pretty_generate(payload)
+          return EXIT_SUCCESS
+        end
+
+        write_output(payload["document"].to_s, output) do
+          "Wrote catalog #{payload.dig("catalog", "key")} (#{payload["format"]}) to #{output}."
+        end
+      end
+
+      # The agents the key reaches, from the facade's agent:// resources.
+      def agents_list(argv)
+        argv = parse(argv, "agents list")
+        return EXIT_SUCCESS unless argv
+
+        resources = Array(client.mcp("resources/list")&.fetch("resources", []))
+        agents = resources.map do |resource|
+          { "slug" => resource["uri"].to_s.delete_prefix("agent://"), "name" => resource["name"], "description" => resource["description"] }
+        end
+        emit(agents: agents) do
+          table(agents, { "SLUG" => ->(a) { a["slug"] }, "NAME" => ->(a) { a["name"] }, "DESCRIPTION" => ->(a) { a["description"] } },
+                empty: "No agents.")
+        end
+      end
+
+      # An agent's versions, newest first, with what each changed.
+      def agents_versions(argv)
+        arguments = {}
+        argv = parse(argv, "agents versions AGENT [--limit N]") do |p|
+          p.on("--limit N", "At most N versions (max 50)") { |v| arguments[:limit] = positive_integer(v, "--limit") }
+        end
+        return EXIT_SUCCESS unless argv
+
+        arguments[:agent] = require_argument(argv, "AGENT (a slug or id)", "agents versions AGENT")
+        payload = client.tool("agents_versions", arguments)
+        emit(payload) do
+          agent = payload.fetch("agent", {})
+          @out.puts "#{agent["name"]} (#{agent["slug"]}): #{agent["version_count"]} versions"
+          table(payload.fetch("versions", []), {
+            "VERSION" => ->(v) { v["version_number"] },
+            "CREATED" => ->(v) { v["created_at"] },
+            "BY" => ->(v) { v["created_by"] },
+            "RELEASE" => ->(v) { v["release"] ? (v["revision"] || v["release_digest"].to_s[0, 12]) : nil },
+            "SUMMARY" => ->(v) { v["change_summary"] },
+            "CHANGED" => ->(v) { v["diff"].is_a?(Hash) ? v["diff"].keys.join(", ") : nil }
+          }, empty: "No versions.")
+          @out.puts "(more versions; raise --limit)" if payload["truncated"]
+        end
+      end
+    end
+
+    include ResourceCommands
+  end
+end

data/lib/active_agent/cli/session_commands.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  class CLI
+    # login, logout and status: the profile a command runs as, and whether
+    # the dashboard accepts it.
+    module SessionCommands
+      MCP_PROTOCOL_VERSION = "2025-03-26"
+
+      private
+
+      # Checks the key against the dashboard before storing it, unless
+      # --no-verify. The key comes from --key, ACTIVEAGENT_API_KEY, or a
+      # prompt (echo off on a terminal), so it need not sit in shell history.
+      def login(argv)
+        values = {}
+        verify = true
+        argv = parse(argv, "login --url URL [--key KEY] [--ingest-key KEY] [--profile NAME]") do |p|
+          p.on("--key KEY", "A dashboard API key (Settings -> API Keys); prompted for when omitted") { |v| values["api_key"] = v }
+          p.on("--ingest-key KEY", "The mount's ingest key, for evals import/report, when it differs from --key") { |v| values["ingest_key"] = v }
+          p.on("--[no-]verify", "Check the key against the dashboard first (default: on)") { |v| verify = v }
+        end
+        return EXIT_SUCCESS unless argv
+        raise Error.new("unexpected arguments: #{argv.join(" ")}", exit_code: EXIT_USAGE) unless argv.empty?
+
+        stored = @credentials[profile_name]
+        url = @options[:url] || env_value("ACTIVEAGENT_URL") || stored["url"]
+        raise Error.new("missing --url, the dashboard's mount (e.g. https://ops.example.com/activeagents)", exit_code: EXIT_USAGE) if url.to_s.empty?
+
+        values["url"] = Client.parse_url(url).to_s
+        values["api_key"] ||= env_value("ACTIVEAGENT_API_KEY") || prompt_for_key
+        values["ingest_key"] ||= env_value("ACTIVEAGENT_INGEST_KEY") || stored["ingest_key"]
+        raise Error.new("an API key is required", exit_code: EXIT_USAGE) if values["api_key"].to_s.strip.empty?
+        values["api_key"] = values["api_key"].strip
+
+        server = verify ? mcp_initialize(Client.new(url: values["url"], api_key: values["api_key"])) : nil
+        @credentials.save(profile_name, values)
+
+        emit(profile: profile_name, url: values["url"], verified: verify, server: server, credentials: @credentials.path) do
+          @out.puts "Logged in to #{values["url"]} as profile '#{profile_name}'#{" (not verified)" unless verify}."
+          @out.puts "Saved to #{@credentials.path}"
+        end
+      end
+
+      def logout(argv)
+        argv = parse(argv, "logout [--profile NAME]")
+        return EXIT_SUCCESS unless argv
+
+        removed = @credentials.delete(profile_name)
+        emit(profile: profile_name, removed: removed) do
+          @out.puts(removed ? "Removed profile '#{profile_name}'." : "No profile '#{profile_name}' was stored.")
+        end
+      end
+
+      # Two checks, one per key: the MCP facade's initialize with the API key,
+      # and Publisher#verify! against the report collector with the ingest
+      # key. Exits EXIT_FAILURE when either fails.
+      def status(argv)
+        argv = parse(argv, "status [--profile NAME]")
+        return EXIT_SUCCESS unless argv
+
+        client
+        checks = {
+          "dashboard" => check { { "server" => mcp_initialize(client) } },
+          "collector" => check { client.publisher.verify! && { "endpoint" => client.reports_endpoint } }
+        }
+        ok = checks.values.all? { |result| result["ok"] }
+
+        emit(profile: profile_name, url: client.url, ok: ok, checks: checks) do
+          @out.puts "Profile:   #{profile_name}"
+          @out.puts "Dashboard: #{client.url}"
+          @out.puts "API key:   #{checks["dashboard"]["ok"] ? "accepted (#{client.mcp_endpoint})" : "FAILED: #{checks["dashboard"]["error"]}"}"
+          @out.puts "Collector: #{checks["collector"]["ok"] ? "accepted (#{client.reports_endpoint})" : "FAILED: #{checks["collector"]["error"]}"}"
+        end
+        ok ? EXIT_SUCCESS : EXIT_FAILURE
+      end
+
+      def check
+        { "ok" => true }.merge(yield || {})
+      rescue Error, ActiveAgent::Evals::Publisher::Error => e
+        { "ok" => false, "error" => e.message }
+      end
+
+      # The facade's serverInfo, which an accepted key gets back.
+      def mcp_initialize(client)
+        result = client.mcp("initialize", protocolVersion: MCP_PROTOCOL_VERSION, capabilities: {},
+                                          clientInfo: { name: "activeagent-cli", version: self.class.version })
+        result.is_a?(Hash) ? result["serverInfo"] : nil
+      end
+
+      def prompt_for_key
+        if @input.respond_to?(:tty?) && @input.tty?
+          require "io/console"
+          @err.print "Dashboard API key: "
+          key = @input.noecho(&:gets)
+          @err.puts
+          key
+        else
+          @input.gets
+        end
+      end
+    end
+
+    include SessionCommands
+  end
+end

data/lib/active_agent/cli/trace_commands.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  class CLI
+    # `activeagent traces ...`. list and get read through the MCP facade's
+    # trace tools; export reads one trace whole as the payload trace ingest
+    # takes (traces_export), and import posts such a payload to
+    # <mount>/api/traces with the ingest key, so a trace moves between
+    # dashboards the way a run report does with `evals report` and `evals
+    # import`.
+    module TraceCommands
+      DURATION = /\A(\d+)([smhd]?)\z/
+      DURATION_MINUTES = { "" => 1, "m" => 1, "h" => 60, "d" => 24 * 60 }.freeze
+
+      private
+
+      def traces_list(argv)
+        arguments = {}
+        argv = parse(argv, "traces list [--agent NAME] [--since 24h] [--status error|ok] [--limit N]") do |p|
+          p.on("--agent AGENT", "An agent class name or a dashboard agent's slug") { |v| arguments[:agent] = v }
+          p.on("--since DURATION", "How far back: minutes, or 30m, 6h, 7d (default 24h)") { |v| arguments[:since_minutes] = since_minutes(v) }
+          p.on("--status STATUS", %w[error ok], "Only failed (error) or successful (ok) traces") { |v| arguments[:status] = v }
+          p.on("--service NAME", "Only this reporting service") { |v| arguments[:service] = v }
+          p.on("--limit N", "At most N traces (max 100)") { |v| arguments[:limit] = positive_integer(v, "--limit") }
+        end
+        return EXIT_SUCCESS unless argv
+        raise Error.new("unexpected arguments: #{argv.join(" ")}", exit_code: EXIT_USAGE) unless argv.empty?
+
+        payload = client.tool("traces_search", arguments)
+        traces = payload.fetch("traces", [])
+        emit(payload) do
+          table(traces, {
+            "TRACE" => ->(t) { t["short_id"] },
+            "TIME" => ->(t) { t["timestamp"] },
+            "AGENT" => ->(t) { [ t["agent"], t["action"] ].compact.join("#") },
+            "MODEL" => ->(t) { t["model"] },
+            "STATUS" => ->(t) { t["status"] },
+            "MS" => ->(t) { t["duration_ms"] },
+            "TOKENS" => ->(t) { t.dig("tokens", "total") }
+          }, empty: "No traces in the last #{payload["window_minutes"]} minutes.")
+          @out.puts "(more traces match; raise --limit or narrow the filters)" if payload["truncated"]
+        end
+      end
+
+      # One trace as the dashboard's trace page shows it: its spans, tool
+      # calls and errors, with long values cut (traces_get).
+      def traces_get(argv)
+        argv = parse(argv, "traces get TRACE")
+        return EXIT_SUCCESS unless argv
+
+        trace_id = require_argument(argv, "TRACE (a trace id, or its first 8 characters)", "traces get TRACE")
+        payload = client.tool("traces_get", trace_id: trace_id)
+        emit(payload) do
+          @out.puts "Trace #{payload["trace_id"]}: #{[ payload["agent"], payload["action"] ].compact.join("#")} " \
+                    "on #{payload["model"] || "-"} (#{payload["status"]}, #{payload["duration_ms"]} ms, " \
+                    "#{payload.dig("tokens", "total")} tokens)"
+          @out.puts "Error: #{payload["error"]}" if payload["error"]
+          table(Array(payload["spans"]), {
+            "SPAN" => ->(s) { s["span_id"] },
+            "TYPE" => ->(s) { s["type"] },
+            "NAME" => ->(s) { s["name"] },
+            "START" => ->(s) { s["start"] },
+            "MS" => ->(s) { s["duration"] },
+            "STATUS" => ->(s) { s["status"] }
+          }, empty: "No spans.")
+          omitted = payload["spans_omitted"].to_i
+          @out.puts "(#{omitted} more spans; `activeagent traces export #{payload["trace_id"]}` has them all)" if omitted.positive?
+        end
+      end
+
+      # Writes the trace, whole, as the payload POST <mount>/api/traces
+      # ingests ({ traces: [trace], sdk }), which `traces import` sends.
+      def traces_export(argv)
+        output = nil
+        argv = parse(argv, "traces export TRACE [-o FILE]") do |p|
+          p.on("-o", "--output FILE", "Write to FILE instead of standard output") { |v| output = v }
+        end
+        return EXIT_SUCCESS unless argv
+
+        trace_id = require_argument(argv, "TRACE (a trace id, or its first 8 characters)", "traces export TRACE")
+        payload = client.tool("traces_export", trace_id: trace_id)
+        write_output(JSON.pretty_generate(payload), output) do
+          "Wrote trace #{payload.dig("traces", 0, "trace_id")} to #{output}."
+        end
+      end
+
+      # Posts a saved trace payload to the dashboard's trace ingest with the
+      # ingest key. FILE is what `traces export` writes, or one trace object,
+      # or the batch a reporter would send. A trace_id the dashboard already
+      # holds is skipped there, so importing the same file twice stores
+      # nothing more.
+      def traces_import(argv)
+        argv = parse(argv, "traces import FILE")
+        return EXIT_SUCCESS unless argv
+
+        path = require_argument(argv, "FILE", "traces import FILE")
+        document = read_json_file(path)
+        payload = document.key?("traces") ? document : { "traces" => [ document ] }
+        traces = payload["traces"]
+        unless traces.is_a?(Array) && traces.any? && traces.all? { |trace| trace.is_a?(Hash) && trace["trace_id"].is_a?(String) }
+          raise Error.new("#{path} holds no traces: it is the { \"traces\": [...] } payload `traces export` writes, " \
+                          "or one trace with a trace_id", exit_code: EXIT_USAGE)
+        end
+        payload["sdk"] = { "name" => "activeagent-cli", "version" => self.class.version } unless payload["sdk"].is_a?(Hash) && !payload["sdk"].empty?
+
+        client.post("/api/traces", JSON.generate(payload))
+        trace_ids = traces.map { |trace| trace["trace_id"] }
+        emit(accepted: true, trace_ids: trace_ids, endpoint: client.traces_endpoint) do
+          @out.puts "Sent #{trace_ids.size} trace#{"s" unless trace_ids.size == 1} to #{client.traces_endpoint}: #{trace_ids.join(", ")}."
+        end
+      end
+
+      # Minutes in a duration such as 90, 30m, 6h or 7d; seconds round up
+      # to a whole minute.
+      def since_minutes(value)
+        match = DURATION.match(value.to_s.strip.downcase)
+        raise Error.new("--since must be minutes or a duration such as 30m, 6h or 7d", exit_code: EXIT_USAGE) unless match
+
+        amount = match[1].to_i
+        minutes = match[2] == "s" ? (amount / 60.0).ceil : amount * DURATION_MINUTES.fetch(match[2])
+        raise Error.new("--since must be at least a minute", exit_code: EXIT_USAGE) unless minutes.positive?
+
+        minutes
+      end
+    end
+
+    include TraceCommands
+  end
+end