llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/render/reports/html.rb

@@ -2,6 +2,7 @@
 
 require "cgi"
 require "erb"
+require "forwardable"
 require "json"
 
 module LLM
@@ -11,18 +12,31 @@ module LLM
         # Builds a self-contained HTML report for completed eval runs.
         # Only contenders with full, error-free coverage can lead the report.
         class HTML
+          extend Forwardable
+          include Format
+
           # Directory containing the report template and its embedded assets.
-          TEMPLATE_DIRECTORY = File.expand_path("../../templates", __dir__)
+          TEMPLATE_DIRECTORY = File.expand_path("../templates", __dir__)
 
+          # Shorthand for building one and rendering it.
+          #
           # @param report [JSON::Loaded] decoded from a saved JSON snapshot
+          # @param stylesheet_path [String, nil] link to this stylesheet; nil embeds it in the page
+          # @param script_path [String, nil] link to this script; nil embeds it in the page
           # @return [String]
           def self.render(report, stylesheet_path: nil, script_path: nil)
             new(report, stylesheet_path:, script_path:).render
           end
 
+          # What the page is built from, and the assets it links instead of embedding.
+          # `report` is a {JSON::Loaded}; the two paths are Strings, or nil to embed.
+          #
+          # @return [JSON::Loaded, String, nil]
           attr_reader :report, :stylesheet_path, :script_path
 
           # @param report [JSON::Loaded] decoded from a saved JSON snapshot
+          # @param stylesheet_path [String, nil] link to this stylesheet; nil embeds it in the page
+          # @param script_path [String, nil] link to this script; nil embeds it in the page
           def initialize(report, stylesheet_path: nil, script_path: nil)
             @report = report
             @stylesheet_path = stylesheet_path
@@ -33,220 +47,83 @@ module LLM
           #
           # @return [String]
           def render
-            health = health_summary
-            standings = global_standings
-            datasets = dataset_reports
+            health = view.health_summary
+            standings = view.global_standings
+            datasets = view.dataset_reports
             ERB.new(File.read(template_path), trim_mode: "-").result(binding)
           end
 
           private
 
-          # @return [String]
-          def report_title = "#{configuration_name} evaluation report"
-
-          # @return [String]
-          def configuration_name = report.suite
-
-          def summaries = report.summaries
-
-          def metrics = report.metrics
-
-          def weighted_score = report.weighted_score
-
-          # @return [Array<RunResult>]
-          def results = summaries.flat_map(&:results)
-
-          # @return [Array<RunResult>]
-          def successful_results = results.select { it.score && !it.failed? && !it.reply&.error? }
-
-          # @return [Hash{Symbol => Object}] health counts and cost for the template
-          def health_summary
-            failed = results.count { !it.score || it.failed? || it.reply&.error? }
-            usage = total_usage
-            {
-              successful:    successful_results.size,
-              total:         results.size,
-              failed:,
-              input_tokens:  tokens(usage.input_tokens),
-              output_tokens: tokens(usage.output_tokens),
-              cost:          money(usage.cost_usd)
-            }
-          end
-
-          # @return [Hash{Symbol => Object}] report entries and ranking after coverage checks
-          def global_standings
-            expected_keys = results.map { it.case.key }.uniq
-            entries = configuration_groups.map { |group| contender(group, expected_keys) }
-            complete = entries.select { it[:complete] }
-            assign_weighted_scores(complete)
-            ranked = complete.sort_by { ranking_value(it) || -1 }
-
-            {
-              entries:,
-              expected_count: expected_keys.length,
-              ranked:,
-              leader: ranked.last,
-              best_coverage: entries.max_by { it[:successful] }
-            }
-          end
-
-          # @return [Array<Hash{Symbol => Object}>] data for each dataset section
-          def dataset_reports
-            summaries.group_by(&:dataset).sort.map do |dataset, group|
-              case_keys = group.flat_map { it.results.map { it.case.key } }.uniq
-              entries = group.map { contender([it], case_keys) }
-              complete = entries.select { it[:complete] }
-              assign_weighted_scores(complete)
-
-              {
-                name:      dataset,
-                summaries: group,
-                case_keys:,
-                winner:    complete.max_by { ranking_value(it) || -1 },
-                entries:   entries.sort_by { ranking_value(it) || -1 }.reverse,
-                cases:     group.flat_map(&:results).group_by { it.case.key }.values.map(&:first)
-                                .sort_by { [it.case.variant, it.case.number || 0, it.case.id] }
-              }
-            end
-          end
-
-          # @param kase [Case]
-          # @param group [Array<Experiment::Summary>]
-          # @return [Array<Hash{Symbol => Object}]] runs and display values for one case
-          def case_results(kase, group)
-            group.flat_map(&:results).select { it.case.key == kase.key }.map do |run|
-              status = if run.failed?
-                         run.error
-                       elsif run.reply&.error?
-                         run.reply.error
-                       elsif run.score
-                         percent(run.score.value)
-                       else
-                         "No score"
-                       end
-              details = run.score&.diagnostics
-              details = { "error" => run.error } if run.failed?
-              { run:, status:, details: details || {} }
-            end
-          end
-
-          # @param group [Array<Experiment::Summary>]
-          # @param expected_keys [Array<String>]
-          # @return [Hash{Symbol => Object}] model coverage and score inputs
-          def contender(group, expected_keys)
-            group_results = group.flat_map(&:results).uniq { it.case.key }
-            successful = group_results.select { it.score && !it.failed? && !it.reply&.error? }
-            errors = group_results.count { !it.score || it.failed? || it.reply&.error? }
-            metric_values = metrics.to_h do |metric|
-              values = group_results.filter_map { it.score&.metrics&.fetch(metric.name, nil) }
-              [metric.name, mean(values)]
-            end
-            score_values = group_results.filter_map { it.score&.value unless it.failed? || it.reply&.error? }
-            cost = group.sum { it.usage.cost_usd }
-            row = metric_values.merge(
-              "score"  => mean(score_values),
-              "errors" => errors,
-              "time"   => group.sum(&:duration),
-              "cost"   => cost
-            )
-            successful_keys = successful.map { it.case.key }.uniq
-
-            {
-              model:         group.first.model,
-              successful:    successful_keys.length,
-              total:         expected_keys.length,
-              errors:,
-              cost:,
-              input_tokens:  group.sum { it.usage.input_tokens },
-              output_tokens: group.sum { it.usage.output_tokens },
-              row:,
-              complete:      expected_keys.length.positive? && successful_keys.sort == expected_keys.uniq.sort && errors.zero?
-            }
-          end
-
-          # @return [Array<Array<Experiment::Summary>>]
-          def configuration_groups
-            summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values
-          end
-
-          # @param entries [Array<Hash{Symbol => Object}>]
-          # @return [void]
-          def assign_weighted_scores(entries)
-            scores = if weighted_score
-                       weighted_score.call(entries.map { it[:row] })
-                     else
-                       entries.map { it[:row][primary_metric] }
-                     end
-            entries.zip(scores).each { |entry, score| entry[:ranking] = score }
-          end
-
-          # @param entry [Hash{Symbol => Object}]
-          # @return [Float, nil]
-          def ranking_value(entry) = entry[:ranking]
-
-          # @return [String]
-          def primary_metric = metrics.find(&:primary)&.name || "score"
-
-          # @return [String]
-          def weight_description
-            weighted_score ? "Weighted score from the lineup; chart order runs worst to best." : "Ranked by #{primary_metric}; chart order runs worst to best."
-          end
-
-          # @return [Usage]
-          def total_usage = summaries.map(&:usage).reduce(Usage.zero, :+)
+          # Template helpers that {ViewModel} answers; the ERB calls them through this binding.
+          def_delegators :view,
+            :report_title,
+            :configuration_name,
+            :summaries,
+            :weighted_score,
+            :case_results,
+            :ranking_value,
+            :primary_metric,
+            :weight_description
+
+          # The template's source of data, built once per report.
+          #
+          # @return [ViewModel]
+          def view = @view ||= ViewModel.new(report)
 
-          # @param model [Model]
+          # The id's first segment is called the provider here; an id with no slash fills both.
+          #
+          # @param model [Models::Model]
           # @return [Array<String>] HTML-escaped provider and model names
           def model_parts(model)
             provider, name = model.id.to_s.split("/", 2)
             [escape(provider), escape(name || provider)]
           end
 
-          # @param model [Model]
+          # For the template's model rows.
+          #
+          # @param model [Models::Model]
           # @return [String] escaped effort setting, or `default`
-          def effort_label(model) = escape(model.effort || "default")
+          def effort_label(model) = escape(model.effort_label)
 
-          # @param model [Model]
+          # For the template's model rows.
+          #
+          # @param model [Models::Model]
           # @return [String] escaped temperature setting, or `default`
-          def temperature_label(model) = escape(model.temperature.nil? ? "default" : model.temperature)
-
-          # @param values [Array<Numeric>]
-          # @return [Float, nil]
-          def mean(values) = values.empty? ? nil : values.sum.fdiv(values.size)
-
-          # @param value [Numeric, nil]
-          # @return [String]
-          def percent(value) = value.nil? ? "—" : format("%.1f%%", value * 100)
-
-          # @param value [Numeric]
-          # @return [String]
-          def money(value)
-            cents = (value.to_s.to_r * 100).ceil
-            format("$%<dollars>d.%<cents>02d", dollars: cents / 100, cents: cents % 100)
-          end
-
-          # @param value [Numeric]
-          # @return [String]
-          def tokens(value) = value.to_i.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+          def temperature_label(model) = escape(model.temperature_label)
 
+          # For the expected answer and case details on the page.
+          #
           # @param value [Object]
           # @return [String]
           def pretty(value) = ::JSON.pretty_generate(value)
 
+          # Everything the template prints from data goes through this.
+          #
           # @param value [Object]
           # @return [String] HTML-escaped text
           def escape(value) = CGI.escapeHTML(value.to_s)
 
+          # The packaged stylesheet's text, for embedding.
+          #
           # @return [String]
           def stylesheet = asset("html_report.css")
 
+          # A link when the report ships its stylesheet alongside; otherwise embedded, so the file stands alone.
+          #
+          # @return [String] an HTML tag
           def stylesheet_tag
             stylesheet_path ? %(<link rel="stylesheet" href="#{escape(stylesheet_path)}">) : "<style>#{stylesheet}</style>"
           end
 
+          # The packaged script's text, which sorts the tables, for embedding.
+          #
           # @return [String]
           def sorting_script = asset("html_report.js")
 
+          # A link when the report ships its script alongside; otherwise embedded, so the file stands alone.
+          #
+          # @return [String] an HTML tag
           def script_tag
             script_path ? %(<script src="#{escape(script_path)}"></script>) : "<script>#{sorting_script}</script>"
           end
@@ -257,6 +134,8 @@ module LLM
           # @return [String]
           def asset(name) = File.read(File.join(TEMPLATE_DIRECTORY, name))
 
+          # Where the page's ERB source lives.
+          #
           # @return [String]
           def template_path = File.join(TEMPLATE_DIRECTORY, "html_report.html.erb")
         end

data/lib/llm/evals/render/reports/json/decoder.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class JSON
+          # A snapshot's document back into runs, one mapper for each of {Encoder}'s.
+          class Decoder
+            # Looked up by name for each saved model.
+            #
+            # @param providers [Providers::Registry]
+            def initialize(providers:)
+              @providers = providers
+            end
+
+            # Parses a document, checking its format and version before reading anything else.
+            #
+            # @param source [String] report JSON, not a path
+            # @return [Loaded]
+            # @raise [KeyError] when a required field is missing
+            # @raise [Error] for an unknown format/version or provider
+            def call(source)
+              document = ::JSON.parse(source)
+              unless document["format"] == FORMAT && document["version"] == VERSION
+                raise Error, "unsupported report format version (expected #{FORMAT} v#{VERSION})"
+              end
+
+              metrics = document.fetch("metrics").map { metric(it) }
+              Loaded.new(suite: document.fetch("suite", "llm-evals"),
+                generated_at: document["generated_at"],
+                run_options: document.fetch("run_options", {}),
+                summaries: document.fetch("summaries").map { summary(it) },
+                metrics:,
+                weighted_score: weighted_score(document["weights"], metrics))
+            rescue ::JSON::ParserError => e
+              raise Error, "invalid report JSON: #{e.message}"
+            end
+
+            private
+
+            # @param data [Hash{String => Object}] one saved metric
+            # @return [Scoring::Metric]
+            def metric(data)
+              Scoring::Metric.new(name: data.fetch("name"),
+                description: data.fetch("description", ""),
+                aggregate: data.fetch("aggregate", "mean").to_sym,
+                primary: data.fetch("primary", false))
+            end
+
+            # Rebuilds the lineup's weights, when it set any.
+            #
+            # @param weights [Hash{String => Numeric}, nil] as saved
+            # @param metrics [Array<Scoring::Metric>] the snapshot's
+            # @return [Scoring::WeightedScore, nil] nil with no weights
+            def weighted_score(weights, metrics)
+              Scoring::WeightedScore.new(weights, metrics: metrics.map(&:name)) if weights && !weights.empty?
+            end
+
+            # One saved experiment, whose results all share a {SavedExperiment} in place of the real one.
+            #
+            # @param data [Hash{String => Object}] one saved summary
+            # @return [Actions::Experiment::Summary]
+            def summary(data)
+              model = model(data.fetch("model"))
+              experiment = SavedExperiment.new(model:)
+              Actions::Experiment::Summary.new(model:,
+                dataset: data.fetch("dataset"),
+                name: data.fetch("name"),
+                means: data.fetch("means"),
+                results: data.fetch("results").map { result(it, experiment) },
+                errors: data.fetch("errors", []),
+                duration: data.fetch("duration", 0.0),
+                permalink: data["permalink"],
+                usage: Stats::Usage.from(data["usage"]))
+            end
+
+            # @param data [Hash{String => Object}] the saved id, provider and settings
+            # @return [Models::Model]
+            # @raise [Error] when no configured provider has the saved name
+            def model(data)
+              provider = @providers.find { it.name == data.fetch("provider") }
+              raise Error, "report uses unknown provider #{data.fetch("provider").inspect}" unless provider
+
+              Models::Model.new(id: data.fetch("id"), provider:, settings: data.fetch("settings", {}))
+            end
+
+            # @param data [Hash{String => Object}] one saved run
+            # @param experiment [SavedExperiment] what ran it
+            # @return [Results::RunResult]
+            def result(data, experiment)
+              Results::RunResult.new(experiment:,
+                case: kase(data.fetch("case")),
+                reply: reply(data["reply"]),
+                score: score(data["score"]),
+                duration: data.fetch("duration", 0.0),
+                error: data["error"])
+            end
+
+            # The case's directory is not saved, so its path comes back empty.
+            #
+            # @param data [Hash{String => Object}] one saved case
+            # @return [Datasets::Case]
+            def kase(data)
+              Datasets::Case.new(id: data.fetch("id"),
+                dataset: data.fetch("dataset"),
+                variant: data.fetch("variant"),
+                input: data.fetch("input"),
+                expected: data["expected"],
+                path: "",
+                metadata: (data["metadata"] || {}).transform_keys(&:to_sym))
+            end
+
+            # @param data [Hash{String => Object}, nil] one saved reply
+            # @return [Results::Reply, nil] nil when the run saved none
+            def reply(data)
+              return unless data
+
+              Results::Reply.new(output: data["output"], raw: data["raw"], usage: data["usage"], error: data["error"], metadata: data["metadata"] || {})
+            end
+
+            # @param data [Hash{String => Object}, nil] one saved score
+            # @return [Results::Score, nil] nil when the run saved none
+            def score(data)
+              return unless data
+
+              Results::Score.new(value: data.fetch("value"),
+                metrics: data["metrics"] || {},
+                outcomes: data["outcomes"] || {},
+                diagnostics: data["diagnostics"] || {})
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/json/encoder.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "time"
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class JSON
+          # Completed runs into the snapshot's document. {Decoder} reads it back.
+          class Encoder
+            # Does nothing until {#call}.
+            #
+            # @param summaries [Array<Actions::Experiment::Summary>]
+            # @param metrics [Array<Scoring::Metric>]
+            # @param weighted_score [Scoring::WeightedScore, nil]
+            # @param run_options [Hash]
+            # @param suite [String]
+            def initialize(summaries, metrics:, weighted_score:, run_options:, suite:)
+              @summaries = summaries
+              @metrics = metrics
+              @weighted_score = weighted_score
+              @run_options = run_options
+              @suite = suite
+            end
+
+            # Stamps the document with the current time. Call stats are included for other
+            # readers of the file; {Decoder} ignores them and recalculates.
+            #
+            # @return [String]
+            def call
+              ::JSON.pretty_generate(
+                format:       FORMAT,
+                version:      VERSION,
+                suite:        @suite,
+                generated_at: Time.now.iso8601,
+                run_options:  plain(@run_options),
+                metrics:      @metrics.map { { name: it.name, description: it.description, aggregate: it.aggregate, primary: it.primary } },
+                weights:      @weighted_score&.weights,
+                call_stats:   Stats::CallStats.for_run(@summaries, @run_options).rows.map(&:to_h),
+                summaries:    @summaries.map { summary(it) }
+              )
+            end
+
+            private
+
+            # Turns symbols into strings at every depth, so what is written reads back the same.
+            #
+            # @param value [Object] run options, or anything inside them
+            # @return [Object] the same shape, with no Symbols
+            def plain(value)
+              case value
+              when Hash then value.to_h { |key, item| [key.to_s, plain(item)] }
+              when Array then value.map { plain(it) }
+              when Symbol then value.to_s
+              else value
+              end
+            end
+
+            # @param summary [Actions::Experiment::Summary]
+            # @return [Hash{Symbol => Object}]
+            def summary(summary)
+              {
+                model:     { id: summary.model.id, provider: summary.model.provider.name, settings: summary.model.settings },
+                dataset:   summary.dataset,
+                name:      summary.name,
+                means:     summary.means,
+                errors:    summary.errors,
+                duration:  summary.duration,
+                permalink: summary.permalink,
+                usage:     summary.usage.to_h,
+                results:   summary.results.map { result(it) }
+              }
+            end
+
+            # @param result [Results::RunResult]
+            # @return [Hash{Symbol => Object}]
+            def result(result)
+              { case: kase(result.case), reply: reply(result.reply), score: score(result.score), duration: result.duration, error: result.error }
+            end
+
+            # Omits the case's path, keeping the snapshot portable.
+            #
+            # @param kase [Datasets::Case]
+            # @return [Hash{Symbol => Object}]
+            def kase(kase)
+              { id: kase.id, dataset: kase.dataset, variant: kase.variant, input: kase.input, expected: kase.expected, metadata: kase.metadata }
+            end
+
+            # @param reply [Results::Reply, nil]
+            # @return [Hash{Symbol => Object}, nil] nil for a run that raised
+            def reply(reply)
+              reply && { output: reply.output, raw: reply.raw, usage: reply.usage.to_h, error: reply.error, metadata: reply.metadata }
+            end
+
+            # @param score [Results::Score, nil]
+            # @return [Hash{Symbol => Object}, nil] nil for an unscored run
+            def score(score)
+              score && { value: score.value, metrics: score.metrics, outcomes: score.outcomes, diagnostics: score.diagnostics }
+            end
+          end
+        end
+      end
+    end
+  end
+end