llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/cli/option_groups.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Lists a command's flags by the `group:` each declares, since dry-cli-help
+      # prints them in one list. A command with no groups keeps the plain OPTIONS.
+      module OptionGroups
+        # Each group's heading, in the order they print.
+        HEADINGS = { general: "General Options", tui: "TUI Options", browser: "Browser Report Options" }.freeze
+
+        private
+
+        # Overrides `Dry::CLI::Help::Screens::Command#render_options`, which prints one
+        # list. The help row goes last among the general options.
+        #
+        # @return [Array<String>] the section's lines
+        def render_options
+          shown = command.options.reject { it.options.fetch(:hidden, false) && !include_hidden? }
+          return super if shown.none? { it.options[:group] }
+
+          groups = shown.zip(option_rows).group_by { |option, _row| option.options.fetch(:group, :general) }
+          HEADINGS.flat_map do |group, heading|
+            rows = groups.fetch(group, []).map(&:last)
+            rows << self.class::HELP if group == :general
+            [format.title(heading), *definitions(rows), ""]
+          end[0...-1]
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/progress_bars.rb

@@ -8,7 +8,8 @@ module LLM
       # - a flat list, the jobs indented under the headline, with no tree;
       # - the headline's bar, which counts every case, in yellow;
       # - a finished job keeps its bar, full, with the time it took where the
-      #   ETA was, instead of turning into a one-line summary;
+      #   ETA was, instead of turning into a one-line summary; red says what failed,
+      #   so no count follows it;
       # - on a screen too short for every job, a finished row stays for
       #   {LINGER} seconds, since a short job would otherwise leave the screen
       #   before anyone saw it full: at two cases, its last frame is 50%;
@@ -38,15 +39,26 @@ module LLM
         # Tells the terminal to show a frame only once all of it has arrived.
         # Terminals that do not know these ignore them.
         SYNC_START = "\e[?2026h"
+        # Ends the update {SYNC_START} began.
         SYNC_END = "\e[?2026l"
 
+        # Matches one color code, which takes no column, or one character, which does.
+        ANSI_OR_CHARACTER = /\e\[[\d;]*[A-Za-z]|./
+
         # Stands between a label's fixed columns and its model.
         MODEL_SEPARATOR = " | "
 
+        # Set by {.for}, to share the colors with whatever draws the models next.
+        #
+        # @return [ModelColors]
+        # @param value [ModelColors]
         attr_writer :model_colors
 
+        # Reaches into the console's private stream, clock and config, which dry-cli-ui offers no other way to pass along.
+        #
         # @param console [Dry::CLI::UI::Console] the command's ui, whose stream, clock and settings it draws with
         # @param colors [ModelColors] shared with whatever else draws the models
+        # @return [ProgressBars]
         def self.for(console, colors: ModelColors.new)
           new(console.send(:err), clock: console.send(:clock), config: console.send(:config)).tap { it.model_colors = colors }
         end
@@ -55,7 +67,7 @@ module LLM
         # the widest of its column, so that every model starts in one column:
         # `[01-bookkeeping-cfo]  effort=low t=0.2 | moonshotai/kimi-k3`.
         #
-        # @param runs [Array<Experiment>]
+        # @param runs [Array<Actions::Experiment>]
         # @return [Array<String>] in the order given
         def self.labels(runs)
           datasets = runs.map { "[#{it.dataset}]" }
@@ -66,27 +78,57 @@ module LLM
           end
         end
 
+        # Stops drawing while the block runs, so a question can use the screen, then
+        # starts again below it. The widget's own position no longer means anything by
+        # then, so nothing is overwritten.
+        #
+        # @yieldreturn [Object]
+        # @return [Object] what the block returns
+        def pausing
+          lock.synchronize { @paused = true }
+          yield
+        ensure
+          lock.synchronize do
+            @paused = false
+            @drawn = 0
+            redraw if live?
+          end
+        end
+
         private
 
+        # Overrides `Dry::CLI::UI::Widgets::Multi#label_width`: never under {LABEL_WIDTH}, and with room for the indent.
+        #
         # @return [Integer]
         def label_width = [LABEL_WIDTH, title.length, *jobs.map { it.label.length }].max + INDENT.length
 
+        # Overrides `Dry::CLI::UI::Widgets::Multi#rows`: a flat list, each job indented and its model painted.
+        #
+        # @return [Array<String>] the headline, then one row per job shown
         def rows
           width = label_width
           [headline(width), *visible.map { "#{INDENT}#{painted(row(it, width - INDENT.length))}" }]
         end
 
+        # Paints only what follows {MODEL_SEPARATOR}, leaving the meter's colors alone.
+        #
         # @param text [String] a job's row, whose label holds the width it was padded to
         # @return [String] the row with its model in that model's color
         def painted(text)
           text.sub(/(?<=#{Regexp.escape(MODEL_SEPARATOR)})\S+/) { terminal.pastel.public_send(model_colors[it], it) }
         end
 
+        # Stands in for a bar built without {.for}.
+        #
+        # @return [ModelColors]
         def model_colors = @model_colors ||= ModelColors.new
 
         # The running jobs and those just finished, in the order declared. The
         # finished get at most half the rows, newest first: when jobs end
         # faster than rows linger, the oldest must not hold the screen.
+        # Overrides `Dry::CLI::UI::Widgets::Multi#visible`.
+        #
+        # @return [Array<Dry::CLI::UI::Widgets::Multi::Job>]
         def visible
           return super unless crowded? && state == :running
 
@@ -96,12 +138,27 @@ module LLM
           jobs & (finished + jobs.select { it.state == :running }.first(slots - finished.size))
         end
 
+        # Whether a finished job still gets its row.
+        #
+        # @param job [Dry::CLI::UI::Widgets::Multi::Job]
+        # @param now [Float] by the clock
+        # @return [Boolean]
         def lingering?(job, now) = ENDED.include?(job.state) && now - finished_at(job) < LINGER
 
+        # Derived, since a job records when it started and how long it took.
+        #
+        # @param job [Dry::CLI::UI::Widgets::Multi::Job] one that has ended
+        # @return [Float] by the clock
         def finished_at(job) = job.started + job.seconds
 
+        # Overrides `Dry::CLI::UI::Widgets::Multi#redraw`, which clears the screen below
+        # first and flickers with many rows; clears only when the frame got shorter.
+        #
+        # @return [void]
         def redraw
-          drawn = rows
+          return if @paused
+
+          drawn = rows.map { clipped(it) }
           cursor = terminal.cursor
           up = @drawn.positive? ? cursor.up(@drawn) : ""
           lines = drawn.map { "\r#{it}#{cursor.clear_line_after}\n" }.join
@@ -110,23 +167,53 @@ module LLM
           @drawn = drawn.size
         end
 
+        # Cuts a row to the screen's width, keeping its color codes. A row that
+        # wrapped would take two lines while {#redraw} counts one, and each frame
+        # would then start lower than the last, leaving the old ones on the screen.
+        #
+        # @param text [String] a row, perhaps colored
+        # @return [String]
+        def clipped(text)
+          room = terminal.width - 1
+          text.scan(ANSI_OR_CHARACTER).select { it.start_with?("\e") || (room -= 1) >= 0 }.join
+        end
+
+        # Overrides `Dry::CLI::UI::Widgets::Multi#headline`: live, it keeps its bar to the
+        # end, in yellow, with the time taken in place of the ETA.
+        #
+        # @param width [Integer] the columns its title is padded to
+        # @return [String]
         def headline(width)
           return super unless live?
 
           finished = state != :running
           bar = full_meter(current, total, started, HEADLINE_COLOR, counts, (@seconds if finished))
-          "#{glyph(state)} #{title.ljust(width)} #{bar}#{finished ? noted(headline_note) : stopping}"
+          "#{glyph(state)} #{title.ljust(width)} #{bar}#{stopping unless finished}"
         end
 
+        # Overrides `Dry::CLI::UI::Widgets::Multi#row`, so that a finished bar stays full
+        # instead of turning into a one-line summary.
+        #
+        # @param job [Dry::CLI::UI::Widgets::Multi::Job]
+        # @param width [Integer] the columns its label is padded to
+        # @return [String]
         def row(job, width)
           return super unless ENDED.include?(job.state) && !spinner?(job) && live?
 
           handle = job.handle
           bar = full_meter(handle.current, handle.total, job.started, handle.color, handle.counts, job.seconds.to_f)
-          "#{glyph(job.state)} #{job.label.ljust(width)} #{bar}#{noted(note(job))}"
+          "#{glyph(job.state)} #{job.label.ljust(width)} #{bar}"
         end
 
         # {#meter}, with the time taken in place of the ETA once seconds are known.
+        #
+        # @param done [Integer]
+        # @param all [Integer, nil]
+        # @param since [Float] when the work started, by the clock
+        # @param color [Symbol, nil] the bar's own color, if any
+        # @param tally [Hash{Symbol => Integer}, nil] units by outcome
+        # @param seconds [Float, nil] the time taken; nil while the work runs
+        # @return [String]
         def full_meter(done, all, since, color, tally, seconds)
           return meter(done, all, since, color, tally) if seconds.nil?
 

data/lib/llm/evals/cli/render.rb

@@ -10,6 +10,12 @@ module LLM
         argument :input, required: true, desc: "JSON report file produced by run or compare --format json"
         option :output, aliases: ["-o"], desc: "HTML output path (default: log/<project>-llm-evals.report.html)"
 
+        # Writes the page beside the other reports, to the given path, or to STDOUT for `-o -`.
+        #
+        # @param input [String] the path of the saved JSON
+        # @param output [String, nil] what -o was given: nil or "" for the default file, "-", or a path
+        # @return [void]
+        # @raise [Error] on a report of an unknown format, version or provider
         def call(input:, output: nil, **)
           source = File.read(input)
           saved = LLM::Evals::Render::Reports::JSON.decode(source, providers: configuration.providers)
@@ -25,12 +31,6 @@ module LLM
           out.write(document) unless path
           status_box(:success, "HTML report written to #{relative(path)}", stream: err) if path
         end
-
-        private
-
-        def invocation = ui.send(:invocation)
-
-        def relative(path) = Pathname(path).relative_path_from(Dir.pwd).to_s
       end
     end
   end

data/lib/llm/evals/cli/report_dispatcher/console.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module LLM
+  module Evals
+    module CLI
+      class ReportDispatcher
+        # What an output may say and open, without knowing the command.
+        class Console
+          # Takes callables, so it can stand in for the command without holding it.
+          #
+          # @param out [IO] takes a document written to STDOUT
+          # @param status [#call] draws a status box on the error stream: level, text
+          # @param save_warnings [#call] saves what the run held back from the screen
+          # @param browser [#browse]
+          def initialize(out:, status:, save_warnings:, browser:)
+            @out = out
+            @status = status
+            @save_warnings = save_warnings
+            @browser = browser
+          end
+
+          # Says where a file went.
+          #
+          # @param label [String] such as "JSON report"
+          # @param path [String]
+          # @return [void]
+          def written(label, path) = @status.call(:success, "#{label} written to #{relative(path)}")
+
+          # A warning rather than a failure, since the report exists either way.
+          #
+          # @param target [String] what could not be opened, as it is named to the user
+          # @return [void]
+          def could_not_open(target) = @status.call(:warning, "could not open #{target}")
+
+          # Asks the runtime, which knows this platform's opener.
+          #
+          # @param url [String]
+          # @return [Boolean] whether a browser took it
+          def browse(url) = @browser.browse(url)
+
+          # Writes it as it is, with no newline added, so a piped document is unchanged.
+          #
+          # @param document [String]
+          # @return [void]
+          def print(document) = @out.write(document)
+
+          # Left to the command, which knows where its warnings belong.
+          #
+          # @return [void]
+          def save_warnings = @save_warnings.call
+
+          private
+
+          # As {Command#relative}, which is out of reach from here.
+          #
+          # @param path [String]
+          # @return [String] relative to the working directory
+          def relative(path) = Pathname(path).relative_path_from(Dir.pwd).to_s
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/report_dispatcher/html_output.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module LLM
+  module Evals
+    module CLI
+      class ReportDispatcher
+        # --format html: the page, from the same JSON a later `render` would read.
+        class HtmlOutput < Output
+          # Writes the page, and opens it unless it went to STDOUT.
+          #
+          # @see Output#write
+          # @param snapshot [Snapshot]
+          # @return [void]
+          def write(snapshot)
+            json = json_for(snapshot)
+            path = output_path(snapshot.output, "html")
+            document = pipeline.write_html(json:, path:)
+            path ? console.written("HTML report", path) : console.print(document)
+            console.save_warnings
+            open_page(path) if snapshot.open && path
+          end
+
+          private
+
+          # With autosave the JSON is saved and read back, so the page holds what the file holds.
+          #
+          # @param snapshot [Snapshot]
+          # @return [String] the JSON text
+          def json_for(snapshot)
+            return pipeline.write_json(**snapshot.document, path: nil) unless snapshot.results[:autosave]
+
+            path = pipeline.save_json(**snapshot.document)
+            console.written("JSON results", path)
+            File.read(path)
+          end
+
+          # Warns when no browser takes it; the page is on disk either way.
+          #
+          # @param path [String]
+          # @return [void]
+          def open_page(path)
+            console.could_not_open(path) unless console.browse(URI::DEFAULT_PARSER.escape("file://#{path}"))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/report_dispatcher/json_output.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      class ReportDispatcher
+        # --format json: the snapshot itself, saved beside the others unless -o
+        # names a place for it.
+        class JsonOutput < Output
+          # Writes the document, or prints it for `-o -`.
+          #
+          # @see Output#write
+          # @param snapshot [Snapshot]
+          # @return [void]
+          def write(snapshot)
+            if snapshot.output.nil? || snapshot.output.empty?
+              console.written("JSON report", pipeline.save_json(**snapshot.document))
+            else
+              path = output_path(snapshot.output, "json")
+              document = pipeline.write_json(**snapshot.document, path:)
+              path ? console.written("JSON report", path) : console.print(document)
+            end
+            console.save_warnings
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/report_dispatcher/output.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      class ReportDispatcher
+        # One way of delivering a run.
+        class Output
+          # Holds only collaborators; nothing is written until {#write}.
+          #
+          # @param pipeline [LLM::Evals::Render::Reports::Pipeline]
+          # @param console [Console]
+          # @param invocation [Dry::CLI::UI::Invocation] names the default files
+          def initialize(pipeline:, console:, invocation:)
+            @pipeline = pipeline
+            @console = console
+            @invocation = invocation
+          end
+
+          # Abstract; each subclass is one delivery.
+          #
+          # @param snapshot [Snapshot]
+          # @yield [report] draw the decoded report in the terminal, for an output that does
+          # @return [void]
+          # @raise [NotImplementedError] unless a subclass implements it
+          def write(snapshot, &) = raise NotImplementedError, "#{self.class} must implement #write"
+
+          private
+
+          attr_reader :pipeline, :console
+
+          # @!attribute [r] pipeline
+          #   Writes the files this output asks for.
+          #   @return [LLM::Evals::Render::Reports::Pipeline]
+
+          # @!attribute [r] console
+          #   What this output may say and open.
+          #   @return [Console]
+
+          # Where -o sends this format: the default report name without a path, and a
+          # path's .log ending swapped for the extension.
+          #
+          # @param output [String, nil] what -o was given
+          # @param extension [String]
+          # @return [String, nil] nil means STDOUT
+          def output_path(output, extension)
+            return nil if output == "-"
+            return pipeline.report_path(File.dirname(@invocation.report_path("")), extension) if output.nil? || output.empty?
+
+            @invocation.report_path(output).sub(/\.log\z/, ".#{extension}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/report_dispatcher/pipeline_output.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      class ReportDispatcher
+        # No format, or tui: the display and autosave the lineup configures.
+        class PipelineOutput < Output
+          # Lets the pipeline decide, then says what it wrote.
+          #
+          # @see Output#write
+          # @param snapshot [Snapshot]
+          # @yield [report] draw the decoded report in the terminal, when the display does
+          # @return [void]
+          def write(snapshot, &)
+            results = snapshot.results
+            result = pipeline.call(**snapshot.document, autosave: results[:autosave], display: results[:display], open: snapshot.open, &)
+            console.written("JSON results", result.json_path) if result.json_path
+            announce_page(result, snapshot.open) if result.html_path
+            console.save_warnings unless Config::TERMINAL_DISPLAYS.include?(results[:display]) # the terminal view has said so already
+          end
+
+          private
+
+          # Says where the page went, and warns when it would not open.
+          #
+          # @param result [LLM::Evals::Render::Reports::Pipeline::Result]
+          # @param open [Boolean] whether opening it was asked for
+          # @return [void]
+          def announce_page(result, open)
+            console.written("HTML report", result.html_path)
+            console.could_not_open(result.html_path) if open && !result.browser_opened
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/report_dispatcher.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Sends a finished run to the output its format asks for: an HTML page, a
+      # JSON document, or, with no format, whatever the lineup's results section
+      # configures. The command keeps the terminal view, which draws on its own
+      # streams, and hands it over as a block.
+      class ReportDispatcher
+        # What --format asks of results.display.
+        DISPLAYS = { "tui" => "tui", "none" => "none", "html" => "browser", "json" => "none" }.freeze
+
+        # One run on its way out.
+        #
+        # @!attribute [r] summaries
+        #   @return [Array<Actions::Experiment::Summary>]
+        # @!attribute [r] metrics
+        #   @return [Array<Scoring::Metric>]
+        # @!attribute [r] weighted_score
+        #   @return [Scoring::WeightedScore, nil]
+        # @!attribute [r] run_options
+        #   @return [Hash] what the saved report says about its run, results included
+        # @!attribute [r] results
+        #   @return [Hash{Symbol => Object}] the settled autosave, destination, temp_dir and display
+        # @!attribute [r] output
+        #   @return [String, nil] what -o was given
+        # @!attribute [r] open
+        #   @return [Boolean] whether to open what gets written
+        Snapshot = Data.define(:summaries, :metrics, :weighted_score, :run_options, :results, :output, :open) do
+          # Leaves out how and where it is delivered, which the saved JSON has no use for.
+          #
+          # @return [Hash] what {Render::Reports::Pipeline} serializes
+          def document = { summaries:, metrics:, weighted_score:, run_options: }
+        end
+
+        # Holds only collaborators; nothing is written until {#call}.
+        #
+        # @param config [Config]
+        # @param options [Hash] the parsed flags
+        # @param providers [Providers::Registry]
+        # @param runtime [Runtime::Environment]
+        # @param suite [String] names the files when the run has no project
+        # @param console [Console]
+        # @param invocation [Dry::CLI::UI::Invocation]
+        def initialize(config:, options:, providers:, runtime:, suite:, console:, invocation:)
+          @config = config
+          @options = options
+          @providers = providers
+          @runtime = runtime
+          @suite = suite
+          @console = console
+          @invocation = invocation
+        end
+
+        # Picks the output for the format and hands it the snapshot.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param metrics [Array<Scoring::Metric>]
+        # @param weighted_score [Scoring::WeightedScore, nil]
+        # @param run_options [Hash]
+        # @param output [String, nil] what -o was given: nil, "-", "" for the default file, or a path
+        # @param open [Boolean]
+        # @param format [String, nil] `tui`, `html` or `json`
+        # @param terminal [Proc] the block below
+        # @yield [report] draw the decoded report in the terminal
+        # @yieldparam report [LLM::Evals::Render::Reports::JSON::Loaded]
+        # @return [void]
+        def call(summaries, metrics:, weighted_score:, run_options:, output:, open:, format: nil, &terminal)
+          results = effective_results(format)
+          snapshot = Snapshot.new(summaries:,
+            metrics:,
+            weighted_score:,
+            output:,
+            open:,
+            results:,
+            run_options: run_options.merge(results:))
+          strategy(format).new(pipeline: pipeline(results, run_options), console: @console, invocation: @invocation)
+                          .write(snapshot, &terminal)
+        end
+
+        # Lets the command tell in advance whether the terminal view will draw.
+        #
+        # @param format [String, nil] `tui`, `html` or `json`
+        # @return [String] the display this run settles on
+        def display(format = nil) = effective_results(format)[:display]
+
+        private
+
+        # Anything but html or json leaves the choice to the lineup.
+        #
+        # @param format [String, nil] `tui`, `html` or `json`
+        # @return [Class<Output>]
+        def strategy(format)
+          case format
+          when "html" then HtmlOutput
+          when "json" then JsonOutput
+          else PipelineOutput
+          end
+        end
+
+        # The flags over the lineup's results section.
+        #
+        # @param format [String, nil] `tui`, `html` or `json`, which outranks --display
+        # @return [Hash{Symbol => Object}] autosave, destination, temp_dir and display
+        def effective_results(format)
+          configured = @config.results
+          results = {
+            autosave:    @options[:autosave].nil? ? configured.autosave : @options[:autosave],
+            destination: File.expand_path(@options[:destination] || configured.destination),
+            temp_dir:    File.expand_path(@options[:temp_dir] || configured.temp_dir),
+            display:     @options[:display] || configured.display_mode
+          }
+          results[:display] = DISPLAYS.fetch(format, results[:display]) if format
+          results
+        end
+
+        # Looks the project up under symbol and string keys alike, and falls back on the suite's name.
+        #
+        # @param results [Hash{Symbol => Object}] from {#effective_results}
+        # @param run_options [Hash]
+        # @return [LLM::Evals::Render::Reports::Pipeline]
+        def pipeline(results, run_options)
+          LLM::Evals::Render::Reports::Pipeline.new(providers: @providers,
+            runtime:     @runtime,
+            destination: results[:destination],
+            temp_dir:    results[:temp_dir],
+            project:     run_options.dig(:effective, :project) || run_options.dig("effective", "project") || @suite)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/run.rb

@@ -17,31 +17,19 @@ module LLM
           "-n                                             # show what would run, call nothing"
         ]
 
-        def call(models:, dataset:, dry_run:, **options)
-          config = lineup(options)
-          weigh(config)
-          contenders = contenders(config, models, options)
-          datasets = datasets(dataset)
-          return matrix(datasets, contenders) if dry_run
-
-          setup!(contenders, options)
-          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
-          summaries = execute(runs, concurrency: concurrency(options))
-          report(summaries,
-            output:      options[:output],
-            open:        options[:open],
-            scores:      options[:scores],
-            format:      options[:format],
-            config:,
-            options:,
-            run_options: serialize_run_options(config:, contenders:, datasets:, runs:, models:, dataset:, options:)) do |display_summaries|
-            display_summaries.each { breakdown(it, titled: display_summaries.size > 1) }
-          end
-        end
-
         private
 
+        # Overrides {ModelCommand#details}. A single experiment's breakdown needs no title.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @return [void]
+        def details(summaries) = summaries.each { breakdown(it, titled: summaries.size > 1) }
+
         # One row per case: its score, and whatever the scorer diagnosed.
+        #
+        # @param summary [Actions::Experiment::Summary]
+        # @param titled [Boolean] print its name above the table
+        # @return [void]
         def breakdown(summary, titled: false)
           status_box :info, summary.name if titled
           rows = summary.results.map do |result|
@@ -52,6 +40,9 @@ module LLM
 
         # The scorer's diagnostics, one per line, rendered without knowing
         # their shape: "field: value" for a Hash, each item for an Array.
+        #
+        # @param result [Results::RunResult]
+        # @return [String] the error instead, for a case whose call failed
         def diagnosis(result)
           return result.error if result.failed?