llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/render/reports/json.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "json"
-require "time"
 
 module LLM
   module Evals
@@ -10,161 +9,59 @@ module LLM
         # A portable snapshot of completed runs, suitable for rendering later
         # without repeating model calls or depending on the original case files.
         class JSON
+          # Identifies a document as one of these snapshots, so other JSON is refused.
           FORMAT = "llm-evals-report"
+          # The document's schema version; {Decoder} refuses any other.
           VERSION = 1
+          # A snapshot read back: what {HTML} and the terminal display are given.
+          #
+          # @!attribute [r] suite
+          #   @return [String] the suite's name
+          # @!attribute [r] generated_at
+          #   @return [String, nil] when the snapshot was written, ISO 8601
+          # @!attribute [r] run_options
+          #   @return [Hash{String => Object}] as the run recorded them, with string keys now
+          # @!attribute [r] summaries
+          #   @return [Array<Actions::Experiment::Summary>]
+          # @!attribute [r] metrics
+          #   @return [Array<Scoring::Metric>]
+          # @!attribute [r] weighted_score
+          #   @return [Scoring::WeightedScore, nil] nil when the lineup set no weights
           Loaded = Data.define(:suite, :generated_at, :run_options, :summaries, :metrics, :weighted_score)
+          # Stands in for the {Actions::Experiment} on a result read back; a snapshot keeps only its model.
+          #
+          # @!attribute [r] model
+          #   @return [Models::Model]
           SavedExperiment = Data.define(:model)
 
-          # @param summaries [Array<Experiment::Summary>]
-          # @param metrics [Array<Metric>]
-          # @param weighted_score [WeightedScore, nil]
+          # Serializes completed runs under the configured suite's name.
+          #
+          # @param summaries [Array<Actions::Experiment::Summary>]
+          # @param metrics [Array<Scoring::Metric>]
+          # @param weighted_score [Scoring::WeightedScore, nil]
+          # @param run_options [Hash]
           # @return [String]
           def self.dump(summaries, metrics:, weighted_score: nil, run_options: {})
-            document = {
-              format:       FORMAT,
-              version:      VERSION,
-              suite:        LLM::Evals.configuration.name,
-              generated_at: Time.now.iso8601,
-              run_options:  json_value(run_options),
-              metrics:      metrics.map { { name: it.name, description: it.description, aggregate: it.aggregate, primary: it.primary } },
-              weights:      weighted_score&.weights,
-              summaries:    summaries.map { summary(it) }
-            }
-            ::JSON.pretty_generate(document)
+            Encoder.new(summaries, metrics:, weighted_score:, run_options:, suite: LLM::Evals.configuration.name).call
           end
 
-          def self.json_value(value)
-            case value
-            when Hash then value.to_h { |key, item| [key.to_s, json_value(item)] }
-            when Array then value.map { json_value(it) }
-            when Symbol then value.to_s
-            else value
-            end
-          end
-          private_class_method :json_value
-
+          # Parses a snapshot back into runs. Providers are looked up by name, so
+          # the document's providers must still be configured.
+          #
           # @param source [String] report JSON, not a path
           # @param providers [Providers::Registry]
           # @return [Loaded]
+          # @raise [KeyError] when a required field is missing
           # @raise [Error] for an unknown format/version or provider
-          def self.decode(source, providers:)
-            document = ::JSON.parse(source)
-            unless document["format"] == FORMAT && document["version"] == VERSION
-              raise Error, "unsupported report format version (expected #{FORMAT} v#{VERSION})"
-            end
-
-            metrics = document.fetch("metrics").map do |metric|
-              Metric.new(name: metric.fetch("name"),
-                description: metric.fetch("description", ""),
-                aggregate: metric.fetch("aggregate", "mean").to_sym,
-                primary: metric.fetch("primary", false))
-            end
-            weights = document["weights"]
-            weighted_score = WeightedScore.new(weights, metrics: metrics.map(&:name)) if weights && !weights.empty?
-            Loaded.new(suite: document.fetch("suite", "llm-evals"),
-              generated_at: document["generated_at"],
-              run_options: document.fetch("run_options", {}),
-              summaries: document.fetch("summaries").map { summary_from(it, providers) },
-              metrics:,
-              weighted_score:)
-          rescue ::JSON::ParserError => e
-            raise Error, "invalid report JSON: #{e.message}"
-          end
+          def self.decode(source, providers:) = Decoder.new(providers:).call(source)
 
+          # Alias of {.decode}.
+          #
+          # @param source [String] report JSON, not a path
+          # @param providers [Providers::Registry]
+          # @return [Loaded]
+          # @raise [Error] for an unknown format/version or provider
           def self.load(source, providers:) = decode(source, providers:)
-
-          def self.summary(summary)
-            {
-              model:     { id: summary.model.id, provider: summary.model.provider.name, settings: summary.model.settings },
-              dataset:   summary.dataset,
-              name:      summary.name,
-              means:     summary.means,
-              errors:    summary.errors,
-              duration:  summary.duration,
-              permalink: summary.permalink,
-              usage:     summary.usage.to_h,
-              results:   summary.results.map { result(it) }
-            }
-          end
-          private_class_method :summary
-
-          def self.result(result)
-            {
-              case:     {
-                id:       result.case.id,
-                dataset:  result.case.dataset,
-                variant:  result.case.variant,
-                input:    result.case.input,
-                expected: result.case.expected,
-                metadata: result.case.metadata
-              },
-              reply:    result.reply && {
-                output:   result.reply.output,
-                raw:      result.reply.raw,
-                usage:    result.reply.usage.to_h,
-                error:    result.reply.error,
-                metadata: result.reply.metadata
-              },
-              score:    result.score && {
-                value:       result.score.value,
-                metrics:     result.score.metrics,
-                outcomes:    result.score.outcomes,
-                diagnostics: result.score.diagnostics
-              },
-              duration: result.duration,
-              error:    result.error
-            }
-          end
-          private_class_method :result
-
-          def self.summary_from(data, providers)
-            model_data = data.fetch("model")
-            provider = providers.find { it.name == model_data.fetch("provider") }
-            raise Error, "report uses unknown provider #{model_data.fetch("provider").inspect}" unless provider
-
-            model = Model.new(id: model_data.fetch("id"), provider:, settings: model_data.fetch("settings", {}))
-            placeholder_cases = data.fetch("results").map do |result_data|
-              case_data = result_data.fetch("case")
-              kase = Case.new(id: case_data.fetch("id"),
-                dataset: case_data.fetch("dataset"),
-                variant: case_data.fetch("variant"),
-                input: case_data.fetch("input"),
-                expected: case_data["expected"],
-                path: "",
-                metadata: (case_data["metadata"] || {}).transform_keys(&:to_sym))
-              [result_data, kase]
-            end
-            experiment = SavedExperiment.new(model:)
-            results = placeholder_cases.map do |result_data, kase|
-              reply_data = result_data["reply"]
-              reply = if reply_data
-                        Reply.new(output: reply_data["output"],
-                          raw: reply_data["raw"],
-                          usage: reply_data["usage"],
-                          error: reply_data["error"],
-                          metadata: reply_data["metadata"] || {})
-                      end
-              score_data = result_data["score"]
-              if score_data
-                score = Score.new(value: score_data.fetch("value"),
-                  metrics: score_data["metrics"] || {},
-                  outcomes: score_data["outcomes"] || {},
-                  diagnostics: score_data["diagnostics"] || {})
-              end
-              RunResult.new(experiment:, case: kase, reply:, score:, duration: result_data.fetch("duration", 0.0), error: result_data["error"])
-            end
-
-            Experiment::Summary.new(model:,
-              dataset: data.fetch("dataset"),
-              name: data.fetch("name"),
-              means: data.fetch("means"),
-              results:,
-              errors: data.fetch("errors", []),
-              duration: data.fetch("duration", 0.0),
-              permalink: data["permalink"],
-              usage: Usage.from(data["usage"]))
-          end
-          private_class_method :summary_from
         end
       end
     end

data/lib/llm/evals/render/reports/pipeline.rb

@@ -11,10 +11,25 @@ module LLM
         # Writes the canonical JSON snapshot and passes that snapshot to the
         # selected display. Temporary snapshots are removed after presentation.
         class Pipeline
+          # Where a run's reports went. A path is nil for anything not kept; `browser_opened`
+          # is nil when no attempt was made.
+          #
+          # @!attribute [r] json_path
+          #   @return [String, nil] the saved snapshot; nil when it was only temporary
+          # @!attribute [r] html_path
+          #   @return [String, nil]
+          # @!attribute [r] css_path
+          #   @return [String, nil]
+          # @!attribute [r] js_path
+          #   @return [String, nil]
+          # @!attribute [r] browser_opened
+          #   @return [Boolean, nil]
           Result = Data.define(:json_path, :html_path, :css_path, :js_path, :browser_opened)
 
+          # Reduces the project's name to a slug for use in file names.
+          #
           # @param providers [Providers::Registry]
-          # @param runtime [Runtime]
+          # @param runtime [Runtime::Environment]
           # @param destination [String]
           # @param temp_dir [String]
           # @param project [String]
@@ -26,14 +41,20 @@ module LLM
             @project = project.to_s.downcase.gsub(/[^a-z0-9]+/, "-").delete_prefix("-").delete_suffix("-")
           end
 
-          # @param summaries [Array<Experiment::Summary>]
-          # @param metrics [Array<Metric>]
-          # @param weighted_score [WeightedScore, nil]
+          # Decodes the snapshot back before showing it, so a live report and `render`
+          # display exactly the same thing. Does nothing without autosave or a display.
+          # `display` is none, tui, browser or both.
+          #
+          # @param summaries [Array<Actions::Experiment::Summary>]
+          # @param metrics [Array<Scoring::Metric>]
+          # @param weighted_score [Scoring::WeightedScore, nil]
           # @param run_options [Hash]
           # @param autosave [Boolean]
           # @param display [String]
           # @param open [Boolean]
           # @yield [report] display the decoded JSON in the terminal
+          # @yieldparam report [JSON::Loaded]
+          # @raise [Error] when the snapshot cannot be decoded
           # @return [Result]
           def call(summaries:, metrics:, weighted_score:, run_options:, autosave:, display:, open:)
             return Result.new(nil, nil, nil, nil, nil) unless autosave || display != "none"
@@ -44,24 +65,18 @@ module LLM
             return Result.new(json_path:, html_path: nil, css_path: nil, js_path: nil, browser_opened: nil) if display == "none"
 
             report = JSON.decode(File.read(json_path), providers: @providers)
-            assets = case display
-                     when "tui"
-                       yield report if block_given?
-                       [nil, nil, nil, nil]
-                     when "browser"
-                       render_browser(report, open:)
-                     else
-                       [nil, nil, nil, nil]
-                     end
-            html_path, css_path, js_path, browser_opened = assets
+            yield report if block_given? && Config::TERMINAL_DISPLAYS.include?(display)
+            html_path, css_path, js_path, browser_opened = render_browser(report, open:) if Config::BROWSER_DISPLAYS.include?(display)
             Result.new(json_path: autosave ? json_path : nil, html_path:, css_path:, js_path:, browser_opened:)
           ensure
             FileUtils.rm_f(json_path) if defined?(json_path) && json_path && !autosave
           end
 
-          # @param summaries [Array<Experiment::Summary>]
-          # @param metrics [Array<Metric>]
-          # @param weighted_score [WeightedScore, nil]
+          # The snapshot's document, to a file or back to the caller.
+          #
+          # @param summaries [Array<Actions::Experiment::Summary>]
+          # @param metrics [Array<Scoring::Metric>]
+          # @param weighted_score [Scoring::WeightedScore, nil]
           # @param run_options [Hash]
           # @param path [String, nil] nil returns JSON text to the caller
           # @return [String, nil] JSON text when no path was supplied
@@ -76,6 +91,8 @@ module LLM
             end
           end
 
+          # Rebuilds the HTML from a snapshot, standing alone with its assets embedded.
+          #
           # @param json [String] serialized run results
           # @param path [String, nil] nil returns HTML text to the caller
           # @return [String, nil] HTML text when no path was supplied
@@ -94,9 +111,9 @@ module LLM
           # Writes an explicitly requested JSON report under the configured
           # destination, using the same timestamped name as an autosaved result.
           #
-          # @param summaries [Array<Experiment::Summary>]
-          # @param metrics [Array<Metric>]
-          # @param weighted_score [WeightedScore, nil]
+          # @param summaries [Array<Actions::Experiment::Summary>]
+          # @param metrics [Array<Scoring::Metric>]
+          # @param weighted_score [Scoring::WeightedScore, nil]
           # @param run_options [Hash]
           # @return [String] the saved JSON path
           def save_json(summaries:, metrics:, weighted_score:, run_options:)
@@ -118,6 +135,12 @@ module LLM
 
           private
 
+          # Writes the page and its assets next to each other, and opens it when asked.
+          #
+          # @param report [JSON::Loaded]
+          # @param open [Boolean] whether to open the page in the browser
+          # @return [Array(String, String, String, Boolean, nil)] the HTML, CSS and JS paths, and
+          #   whether the browser opened; nil when `open` is false
           def render_browser(report, open:)
             FileUtils.mkdir_p(@destination)
             html_path = report_path(@destination, "html")
@@ -133,12 +156,24 @@ module LLM
             [html_path, css_path, js_path, browser_opened]
           end
 
+          # Timestamped, so results accumulate rather than replace each other.
+          #
+          # @param directory [String]
+          # @param stamp [String] see {#timestamp}
+          # @param extension [String]
+          # @return [String]
           def artifact_path(directory, stamp, extension)
             File.join(directory, "#{@project}.results.#{stamp}.#{extension}")
           end
 
+          # Down to the microsecond, so runs started together do not overwrite each other.
+          #
+          # @return [String]
           def timestamp = Time.now.strftime("%Y-%m-%d.%H%M%S%6N")
 
+          # Creates an empty file now to claim the name; {#call} removes it afterwards.
+          #
+          # @return [String] the path
           def temporary_json_path
             FileUtils.mkdir_p(@temp_dir)
             file = Tempfile.create(["#{@project}.results.", ".json"], @temp_dir)

data/lib/llm/evals/render/reports/terminal.rb

@@ -7,6 +7,8 @@ module LLM
         # Presents a decoded report in the terminal and optionally writes the
         # detailed view to a report file.
         class Terminal
+          # The collaborators are callables so the command keeps its drawing methods.
+          #
           # @param invocation [Dry::CLI::UI::Invocation]
           # @param writing_to [#call] redirects the command's table helpers
           # @param leaderboard [#call] draws the summary leaderboard
@@ -16,11 +18,16 @@ module LLM
             @leaderboard = leaderboard
           end
 
-          # @param summaries [Array<Experiment::Summary>]
+          # Draws the detail then the leaderboard. With a report file, both also go there;
+          # the screen then gets the leaderboard alone.
+          #
+          # @param summaries [Array<Actions::Experiment::Summary>]
           # @param output [String, nil]
           # @param scores [Boolean]
           # @param totals_mode [String] overall, variant or both
+          # @param details [Proc, nil] the block below
           # @yield [summaries] draw the per-case detail
+          # @yieldparam summaries [Array<Actions::Experiment::Summary>]
           # @return [String, nil] report file path
           def call(summaries:, output:, scores:, totals_mode:, &details)
             path = @invocation.report_path(output)

data/lib/llm/evals/render.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Reports on completed runs, built from a saved snapshot.
+    module Render
+    end
+  end
+end

data/lib/llm/evals/results/provider_response.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Results
+      # What a {Models::Model} returns from one call: the text, what it cost, and the
+      # provider's raw response for an adapter that needs more than the text.
+      #
+      # @!attribute [r] text
+      #   @return [String] the model's answer, never nil
+      # @!attribute [r] usage
+      #   @return [Stats::Usage] what the call cost
+      # @!attribute [r] raw
+      #   @return [RubyLLM::Message, Hash, nil] the provider's own response: OpenRouter's message, TypeSafe's answers by name
+      ProviderResponse = Data.define(:text, :usage, :raw) do
+        # Normalizes the text to a String and the usage to {Stats::Usage}.
+        #
+        # @param text [#to_s] the model's answer
+        # @param usage [Stats::Usage, Hash, nil] what the call cost; see {Stats::Usage.from}
+        # @param raw [RubyLLM::Message, Hash, nil] the provider's own response
+        def initialize(text:, usage: Stats::Usage.zero, raw: nil) = super(text: text.to_s, usage: Stats::Usage.from(usage), raw: raw)
+      end
+    end
+  end
+end

data/lib/llm/evals/results/reply.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Results
+      # What an {Actions::Adapter} returns for one case. `error` is the model's failure
+      # (it did not answer the question: no JSON, a refusal) and scores 0; a
+      # transport failure is not the model's and is raised instead, so it is
+      # counted as an error of the run rather than a wrong answer.
+      #
+      # @!attribute [r] output
+      #   @return [Object, nil] the adapter's parsed answer, usually a Hash; nil when the model failed
+      # @!attribute [r] raw
+      #   @return [String, nil] the reply as the model gave it
+      # @!attribute [r] usage
+      #   @return [Stats::Usage] what the call cost
+      # @!attribute [r] error
+      #   @return [String, nil] why the model failed to answer
+      # @!attribute [r] metadata
+      #   @return [Hash] anything else the adapter wants kept with the reply
+      Reply = Data.define(:output, :raw, :usage, :error, :metadata) do
+        # Normalizes the usage, so a Hash read back from a snapshot is accepted.
+        #
+        # @param output [Object, nil] the adapter's parsed answer
+        # @param raw [String, nil] the reply as the model gave it
+        # @param usage [Stats::Usage, Hash, nil] what the call cost; see {Stats::Usage.from}
+        # @param error [String, nil] why the model failed to answer
+        # @param metadata [Hash] anything else the adapter wants kept with the reply
+        def initialize(output: nil, raw: nil, usage: Stats::Usage.zero, error: nil, metadata: {})
+          super(output: output, raw: raw, usage: Stats::Usage.from(usage), error: error, metadata: metadata)
+        end
+
+        # @return [Boolean] whether the model failed to answer
+        def error? = !error.nil?
+      end
+    end
+  end
+end

data/lib/llm/evals/results/run_outcome.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Results
+      # What running a set of experiments produced, and how long it took as a whole.
+      #
+      # @!attribute [r] summaries
+      #   @return [Array<Actions::Experiment::Summary>] the experiments that finished
+      # @!attribute [r] seconds
+      #   @return [Float] wall-clock time for the whole run, which experiments overlap within
+      RunOutcome = Data.define(:summaries, :seconds)
+    end
+  end
+end

data/lib/llm/evals/results/run_result.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Results
+      # One case, run once: what came back and what it scored. `reply` is nil
+      # when the adapter raised, and `error` then holds the message.
+      #
+      # @!attribute [r] experiment
+      #   @return [Actions::Experiment, Render::Reports::JSON::SavedExperiment] what ran it; the latter when read from a snapshot
+      # @!attribute [r] case
+      #   @return [Datasets::Case] the case that ran
+      # @!attribute [r] reply
+      #   @return [Reply, nil] what the adapter returned; nil when it raised
+      # @!attribute [r] score
+      #   @return [Score, nil] what the reply scored; nil when there is nothing to score
+      # @!attribute [r] duration
+      #   @return [Float] seconds this case took
+      # @!attribute [r] error
+      #   @return [String, nil] the message of what the adapter raised
+      RunResult = Data.define(:experiment, :case, :reply, :score, :duration, :error) do
+        # Only `experiment` and `case` are required, since a failed run has little else.
+        #
+        # @param experiment [Actions::Experiment, Render::Reports::JSON::SavedExperiment] what ran it
+        # @param case [Datasets::Case] the case that ran
+        # @param reply [Reply, nil] what the adapter returned
+        # @param score [Score, nil] what the reply scored
+        # @param duration [Float] seconds this case took
+        # @param error [String, nil] the message of what the adapter raised
+        def initialize(experiment:, case:, reply: nil, score: nil, duration: 0.0, error: nil) = super
+
+        # @return [Boolean] whether the adapter raised rather than answered
+        def failed? = !error.nil?
+
+        # @return [Boolean] whether the case went wrong, by raising or by a reply the model failed
+        def errored? = failed? || reply&.error? == true
+      end
+    end
+  end
+end

data/lib/llm/evals/results/score.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Results
+      # What a {Scoring::Scorer} says about one reply. `value` is the case's overall
+      # score in 0..1; `metrics` holds one entry per {Scoring::Metric} the scorer
+      # declares, nil where the case does not apply; `outcomes` and
+      # `diagnostics` are free-form, keyed by field, and only ever displayed.
+      #
+      # @!attribute [r] value
+      #   @return [Float] the overall score, in 0..1
+      # @!attribute [r] metrics
+      #   @return [Hash{String => Float, nil}] by metric name
+      # @!attribute [r] outcomes
+      #   @return [Hash] free-form, keyed by field; only displayed
+      # @!attribute [r] diagnostics
+      #   @return [Hash] free-form, keyed by field; only displayed
+      Score = Data.define(:value, :metrics, :outcomes, :diagnostics) do
+        # Refuses a value outside 0..1, and turns metric names into Strings so that
+        # a fresh score and one read back from JSON look up the same way.
+        #
+        # @param value [Numeric] the overall score, in 0..1
+        # @param metrics [Hash{String, Symbol => Float, nil}] by metric name
+        # @param outcomes [Hash] free-form, keyed by field
+        # @param diagnostics [Hash] free-form, keyed by field
+        # @raise [ArgumentError] when the value is not a number within 0..1
+        def initialize(value:, metrics: {}, outcomes: {}, diagnostics: {})
+          raise ArgumentError, "score value #{value.inspect} is not within 0..1" unless value.is_a?(Numeric) && (0..1).cover?(value)
+
+          super(value: value.to_f, metrics: metrics.transform_keys(&:to_s), outcomes: outcomes, diagnostics: diagnostics)
+        end
+
+        # The score of a reply the model failed: 0 everywhere, the failure as a diagnostic.
+        #
+        # @param metrics [Array<Scoring::Metric>]
+        # @param error [String]
+        # @return [Score]
+        def self.failed(metrics, error)
+          new(value: 0.0, metrics: metrics.to_h { [it.name, 0.0] }, diagnostics: { "error" => error })
+        end
+      end
+    end
+  end
+end