llm-evals 0.4.0

data/lib/llm/evals/providers/registry.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Providers
+      # The providers in use. A `vendors:` contender goes to the first
+      # provider that serves its id, asked last-registered first, so a
+      # suite's own transport beats the defaults and OpenRouter, which claims
+      # everything, is always last. A `direct:` contender goes to the
+      # provider it names, and a direct provider is never asked about an id.
+      class Registry
+        include Enumerable
+
+        # @return [Registry] TypeSafe, then OpenRouter for everything else
+        def self.default = new([OpenRouter.new, TypeSafe.new])
+
+        # @param providers [Array<Base>] the last one is asked first
+        def initialize(providers = [])
+          @providers = providers.dup
+        end
+
+        # Registering a provider under a name already taken replaces it, so a
+        # helper and a required file that both register the same provider
+        # leave one, not two that would be asked twice.
+        #
+        # @param provider [Base]
+        # @return [Base] the provider
+        def register(provider)
+          @providers.reject! { it.name == provider.name }
+          @providers << provider
+          provider
+        end
+
+        def each(&) = @providers.reverse.each(&)
+
+        # @param target [Contender, String] a contender, or a `vendors:` model id
+        # @return [Base]
+        # @raise [Error] when no provider serves it
+        def for(target)
+          return direct(target.provider) if target.is_a?(Contender) && target.direct?
+
+          model_id = target.is_a?(Contender) ? target.model : target
+          find { !it.direct? && it.serves?(model_id) } ||
+            raise(Error, "no provider serves #{model_id.inspect}; have #{reject(&:direct?).map(&:name).join(", ")}")
+        end
+
+        # @param name [Symbol, String] a key under `direct:`
+        # @return [Base]
+        # @raise [Error] when no direct provider has the name
+        def direct(name)
+          find { it.direct? && it.name == name.to_s } ||
+            raise(Error, "no direct provider #{name.to_s.inspect}; have #{direct_names.join(", ")}")
+        end
+
+        # @return [Array<String>] the names `direct:` may use
+        def direct_names = select(&:direct?).map(&:name)
+
+        # @param contenders [Array<Contender>]
+        # @return [Array<Base>] each provider the contenders use, once
+        def used_by(contenders) = contenders.map { self.for(it) }.uniq
+
+        # @param contenders [Array<Contender>]
+        # @return [Array<String>] the API keys the contenders' providers need
+        def keys_for(contenders) = used_by(contenders).flat_map(&:env_keys).uniq
+
+        # @param contender [Contender]
+        # @return [Model]
+        def model_for(contender) = self.for(contender).model_for(contender)
+
+        # One refused contender: who, and why.
+        Refusal = Data.define(:contender, :reason) do
+          def to_s = "#{contender.model} refused: #{reason}"
+        end
+
+        # Splits the contenders into those whose provider produces a request
+        # type the suite declares and those it does not. A refused contender
+        # is named, never run, and never scored 0: a model the suite cannot
+        # even ask is not a model that answered wrongly.
+        #
+        # @param contenders [Array<Contender>]
+        # @param request_types [Array<Symbol>] what the suite's config declares
+        # @return [Array(Array<Contender>, Array<Refusal>)] admitted, refused
+        def admit(contenders, request_types:)
+          admitted, refused = contenders.partition { request_types.include?(self.for(it).request_type) }
+          refusals = refused.map do |contender|
+            needs = self.for(contender).request_type
+            Refusal.new(contender: contender, reason: "needs #{needs}; this suite produces #{request_types.join(", ")}")
+          end
+          [admitted, refusals]
+        end
+
+        # @return [Array<String>] every effort any provider accepts, for the flag's help
+        def efforts = flat_map(&:efforts).uniq
+
+        # Asks each provider in use for its catalog, and names every model it
+        # does not list.
+        #
+        # @param contenders [Array<Contender>]
+        # @return [void]
+        # @raise [Error] naming every unknown model
+        def verify!(contenders)
+          unknown = contenders.uniq(&:key).group_by { self.for(it) }.flat_map { |provider, list| provider.unknown(list.map(&:model)) }
+          return if unknown.empty?
+
+          raise Error,
+            "unknown model#{"s" if unknown.size > 1} #{unknown.join(", ")}: not listed by its provider; " \
+            "check the id in .llm-evals.config.yml or -m"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/providers/type_safe.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Providers
+      # TypeSafe's own API, which serves Jev as typed questions (noul, choice,
+      # score) rather than chat: request type `typesafe`, reached only through
+      # a `direct: typesafe:` entry, whose ids read "typesafe/<name>". Jev on
+      # OpenRouter, `typesafe/jev-router` under `vendors:`, is a chat model
+      # and goes through {OpenRouter}; the contender carries which is meant.
+      # The gem knows this API's model list, key and pricing, enough for
+      # `verify` and the cost column; the request shape waits for the first
+      # suite that declares `request_types: [typesafe]`.
+      class TypeSafe < Base
+        PREFIX = "typesafe/"
+        URL = "https://api.typesafe.ai/v1/models"
+        KEY = "TYPESAFE_API_KEY"
+
+        # TypeSafe bills input tokens only: $0.042 per million for jev-latest
+        # and jev-preview (docs.typesafe.ai/models, read 2026-10-14). Output is free.
+        USD_PER_INPUT_TOKEN = 0.042 / 1_000_000
+
+        def name = "typesafe"
+        def request_type = :typesafe
+        def request_type_description = "typed questions: noul, choice, score"
+        def direct? = true
+        def prefix = PREFIX
+        def env_keys = [KEY]
+
+        # Jev takes no reasoning effort.
+        def efforts = []
+
+        # @return [Array<String>] the models, as "typesafe/<name>"
+        def catalog
+          Catalog.fetch(URL, "Authorization" => "Bearer #{ENV.fetch(KEY, nil)}").fetch("models").map { "#{PREFIX}#{it.fetch("name")}" }
+        end
+
+        # @param input_tokens [Integer]
+        # @return [Usage] with the cost at TypeSafe's input rate
+        def usage(input_tokens:, output_tokens: 0)
+          Usage.new(input_tokens: input_tokens, output_tokens: output_tokens, cost_usd: input_tokens * USD_PER_INPUT_TOKEN)
+        end
+
+        # @raise [Error] always: there is no typesafe consumer yet
+        def complete(model, system:, user:, json: true) # rubocop:disable Lint/UnusedMethodArgument
+          raise Error,
+            "#{model.id}: no typesafe consumer yet; the typed request shape is defined by the first suite " \
+            "that declares request_types: [typesafe]"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "erb"
+require "json"
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        # Builds a self-contained HTML report for completed eval runs.
+        # Only contenders with full, error-free coverage can lead the report.
+        class HTML
+          # Directory containing the report template and its embedded assets.
+          TEMPLATE_DIRECTORY = File.expand_path("../../templates", __dir__)
+
+          # @param report [JSON::Loaded] decoded from a saved JSON snapshot
+          # @return [String]
+          def self.render(report, stylesheet_path: nil, script_path: nil)
+            new(report, stylesheet_path:, script_path:).render
+          end
+
+          attr_reader :report, :stylesheet_path, :script_path
+
+          # @param report [JSON::Loaded] decoded from a saved JSON snapshot
+          def initialize(report, stylesheet_path: nil, script_path: nil)
+            @report = report
+            @stylesheet_path = stylesheet_path
+            @script_path = script_path
+          end
+
+          # Renders the template with report data and embedded assets.
+          #
+          # @return [String]
+          def render
+            health = health_summary
+            standings = global_standings
+            datasets = dataset_reports
+            ERB.new(File.read(template_path), trim_mode: "-").result(binding)
+          end
+
+          private
+
+          # @return [String]
+          def report_title = "#{configuration_name} evaluation report"
+
+          # @return [String]
+          def configuration_name = report.suite
+
+          def summaries = report.summaries
+
+          def metrics = report.metrics
+
+          def weighted_score = report.weighted_score
+
+          # @return [Array<RunResult>]
+          def results = summaries.flat_map(&:results)
+
+          # @return [Array<RunResult>]
+          def successful_results = results.select { it.score && !it.failed? && !it.reply&.error? }
+
+          # @return [Hash{Symbol => Object}] health counts and cost for the template
+          def health_summary
+            failed = results.count { !it.score || it.failed? || it.reply&.error? }
+            usage = total_usage
+            {
+              successful:    successful_results.size,
+              total:         results.size,
+              failed:,
+              input_tokens:  tokens(usage.input_tokens),
+              output_tokens: tokens(usage.output_tokens),
+              cost:          money(usage.cost_usd)
+            }
+          end
+
+          # @return [Hash{Symbol => Object}] report entries and ranking after coverage checks
+          def global_standings
+            expected_keys = results.map { it.case.key }.uniq
+            entries = configuration_groups.map { |group| contender(group, expected_keys) }
+            complete = entries.select { it[:complete] }
+            assign_weighted_scores(complete)
+            ranked = complete.sort_by { ranking_value(it) || -1 }
+
+            {
+              entries:,
+              expected_count: expected_keys.length,
+              ranked:,
+              leader: ranked.last,
+              best_coverage: entries.max_by { it[:successful] }
+            }
+          end
+
+          # @return [Array<Hash{Symbol => Object}>] data for each dataset section
+          def dataset_reports
+            summaries.group_by(&:dataset).sort.map do |dataset, group|
+              case_keys = group.flat_map { it.results.map { it.case.key } }.uniq
+              entries = group.map { contender([it], case_keys) }
+              complete = entries.select { it[:complete] }
+              assign_weighted_scores(complete)
+
+              {
+                name:      dataset,
+                summaries: group,
+                case_keys:,
+                winner:    complete.max_by { ranking_value(it) || -1 },
+                entries:   entries.sort_by { ranking_value(it) || -1 }.reverse,
+                cases:     group.flat_map(&:results).group_by { it.case.key }.values.map(&:first)
+                                .sort_by { [it.case.variant, it.case.number || 0, it.case.id] }
+              }
+            end
+          end
+
+          # @param kase [Case]
+          # @param group [Array<Experiment::Summary>]
+          # @return [Array<Hash{Symbol => Object}]] runs and display values for one case
+          def case_results(kase, group)
+            group.flat_map(&:results).select { it.case.key == kase.key }.map do |run|
+              status = if run.failed?
+                         run.error
+                       elsif run.reply&.error?
+                         run.reply.error
+                       elsif run.score
+                         percent(run.score.value)
+                       else
+                         "No score"
+                       end
+              details = run.score&.diagnostics
+              details = { "error" => run.error } if run.failed?
+              { run:, status:, details: details || {} }
+            end
+          end
+
+          # @param group [Array<Experiment::Summary>]
+          # @param expected_keys [Array<String>]
+          # @return [Hash{Symbol => Object}] model coverage and score inputs
+          def contender(group, expected_keys)
+            group_results = group.flat_map(&:results).uniq { it.case.key }
+            successful = group_results.select { it.score && !it.failed? && !it.reply&.error? }
+            errors = group_results.count { !it.score || it.failed? || it.reply&.error? }
+            metric_values = metrics.to_h do |metric|
+              values = group_results.filter_map { it.score&.metrics&.fetch(metric.name, nil) }
+              [metric.name, mean(values)]
+            end
+            score_values = group_results.filter_map { it.score&.value unless it.failed? || it.reply&.error? }
+            cost = group.sum { it.usage.cost_usd }
+            row = metric_values.merge(
+              "score"  => mean(score_values),
+              "errors" => errors,
+              "time"   => group.sum(&:duration),
+              "cost"   => cost
+            )
+            successful_keys = successful.map { it.case.key }.uniq
+
+            {
+              model:         group.first.model,
+              successful:    successful_keys.length,
+              total:         expected_keys.length,
+              errors:,
+              cost:,
+              input_tokens:  group.sum { it.usage.input_tokens },
+              output_tokens: group.sum { it.usage.output_tokens },
+              row:,
+              complete:      expected_keys.length.positive? && successful_keys.sort == expected_keys.uniq.sort && errors.zero?
+            }
+          end
+
+          # @return [Array<Array<Experiment::Summary>>]
+          def configuration_groups
+            summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values
+          end
+
+          # @param entries [Array<Hash{Symbol => Object}>]
+          # @return [void]
+          def assign_weighted_scores(entries)
+            scores = if weighted_score
+                       weighted_score.call(entries.map { it[:row] })
+                     else
+                       entries.map { it[:row][primary_metric] }
+                     end
+            entries.zip(scores).each { |entry, score| entry[:ranking] = score }
+          end
+
+          # @param entry [Hash{Symbol => Object}]
+          # @return [Float, nil]
+          def ranking_value(entry) = entry[:ranking]
+
+          # @return [String]
+          def primary_metric = metrics.find(&:primary)&.name || "score"
+
+          # @return [String]
+          def weight_description
+            weighted_score ? "Weighted score from the lineup; chart order runs worst to best." : "Ranked by #{primary_metric}; chart order runs worst to best."
+          end
+
+          # @return [Usage]
+          def total_usage = summaries.map(&:usage).reduce(Usage.zero, :+)
+
+          # @param model [Model]
+          # @return [Array<String>] HTML-escaped provider and model names
+          def model_parts(model)
+            provider, name = model.id.to_s.split("/", 2)
+            [escape(provider), escape(name || provider)]
+          end
+
+          # @param model [Model]
+          # @return [String] escaped effort setting, or `default`
+          def effort_label(model) = escape(model.effort || "default")
+
+          # @param model [Model]
+          # @return [String] escaped temperature setting, or `default`
+          def temperature_label(model) = escape(model.temperature.nil? ? "default" : model.temperature)
+
+          # @param values [Array<Numeric>]
+          # @return [Float, nil]
+          def mean(values) = values.empty? ? nil : values.sum.fdiv(values.size)
+
+          # @param value [Numeric, nil]
+          # @return [String]
+          def percent(value) = value.nil? ? "—" : format("%.1f%%", value * 100)
+
+          # @param value [Numeric]
+          # @return [String]
+          def money(value)
+            cents = (value.to_s.to_r * 100).ceil
+            format("$%<dollars>d.%<cents>02d", dollars: cents / 100, cents: cents % 100)
+          end
+
+          # @param value [Numeric]
+          # @return [String]
+          def tokens(value) = value.to_i.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+
+          # @param value [Object]
+          # @return [String]
+          def pretty(value) = ::JSON.pretty_generate(value)
+
+          # @param value [Object]
+          # @return [String] HTML-escaped text
+          def escape(value) = CGI.escapeHTML(value.to_s)
+
+          # @return [String]
+          def stylesheet = asset("html_report.css")
+
+          def stylesheet_tag
+            stylesheet_path ? %(<link rel="stylesheet" href="#{escape(stylesheet_path)}">) : "<style>#{stylesheet}</style>"
+          end
+
+          # @return [String]
+          def sorting_script = asset("html_report.js")
+
+          def script_tag
+            script_path ? %(<script src="#{escape(script_path)}"></script>) : "<script>#{sorting_script}</script>"
+          end
+
+          # Reads one packaged asset for embedding in the standalone report.
+          #
+          # @param name [String] report asset filename
+          # @return [String]
+          def asset(name) = File.read(File.join(TEMPLATE_DIRECTORY, name))
+
+          # @return [String]
+          def template_path = File.join(TEMPLATE_DIRECTORY, "html_report.html.erb")
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/json.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        # A portable snapshot of completed runs, suitable for rendering later
+        # without repeating model calls or depending on the original case files.
+        class JSON
+          FORMAT = "llm-evals-report"
+          VERSION = 1
+          Loaded = Data.define(:suite, :generated_at, :run_options, :summaries, :metrics, :weighted_score)
+          SavedExperiment = Data.define(:model)
+
+          # @param summaries [Array<Experiment::Summary>]
+          # @param metrics [Array<Metric>]
+          # @param weighted_score [WeightedScore, nil]
+          # @return [String]
+          def self.dump(summaries, metrics:, weighted_score: nil, run_options: {})
+            document = {
+              format:       FORMAT,
+              version:      VERSION,
+              suite:        LLM::Evals.configuration.name,
+              generated_at: Time.now.iso8601,
+              run_options:  json_value(run_options),
+              metrics:      metrics.map { { name: it.name, description: it.description, aggregate: it.aggregate, primary: it.primary } },
+              weights:      weighted_score&.weights,
+              summaries:    summaries.map { summary(it) }
+            }
+            ::JSON.pretty_generate(document)
+          end
+
+          def self.json_value(value)
+            case value
+            when Hash then value.to_h { |key, item| [key.to_s, json_value(item)] }
+            when Array then value.map { json_value(it) }
+            when Symbol then value.to_s
+            else value
+            end
+          end
+          private_class_method :json_value
+
+          # @param source [String] report JSON, not a path
+          # @param providers [Providers::Registry]
+          # @return [Loaded]
+          # @raise [Error] for an unknown format/version or provider
+          def self.decode(source, providers:)
+            document = ::JSON.parse(source)
+            unless document["format"] == FORMAT && document["version"] == VERSION
+              raise Error, "unsupported report format version (expected #{FORMAT} v#{VERSION})"
+            end
+
+            metrics = document.fetch("metrics").map do |metric|
+              Metric.new(name: metric.fetch("name"),
+                description: metric.fetch("description", ""),
+                aggregate: metric.fetch("aggregate", "mean").to_sym,
+                primary: metric.fetch("primary", false))
+            end
+            weights = document["weights"]
+            weighted_score = WeightedScore.new(weights, metrics: metrics.map(&:name)) if weights && !weights.empty?
+            Loaded.new(suite: document.fetch("suite", "llm-evals"),
+              generated_at: document["generated_at"],
+              run_options: document.fetch("run_options", {}),
+              summaries: document.fetch("summaries").map { summary_from(it, providers) },
+              metrics:,
+              weighted_score:)
+          rescue ::JSON::ParserError => e
+            raise Error, "invalid report JSON: #{e.message}"
+          end
+
+          def self.load(source, providers:) = decode(source, providers:)
+
+          def self.summary(summary)
+            {
+              model:     { id: summary.model.id, provider: summary.model.provider.name, settings: summary.model.settings },
+              dataset:   summary.dataset,
+              name:      summary.name,
+              means:     summary.means,
+              errors:    summary.errors,
+              duration:  summary.duration,
+              permalink: summary.permalink,
+              usage:     summary.usage.to_h,
+              results:   summary.results.map { result(it) }
+            }
+          end
+          private_class_method :summary
+
+          def self.result(result)
+            {
+              case:     {
+                id:       result.case.id,
+                dataset:  result.case.dataset,
+                variant:  result.case.variant,
+                input:    result.case.input,
+                expected: result.case.expected,
+                metadata: result.case.metadata
+              },
+              reply:    result.reply && {
+                output:   result.reply.output,
+                raw:      result.reply.raw,
+                usage:    result.reply.usage.to_h,
+                error:    result.reply.error,
+                metadata: result.reply.metadata
+              },
+              score:    result.score && {
+                value:       result.score.value,
+                metrics:     result.score.metrics,
+                outcomes:    result.score.outcomes,
+                diagnostics: result.score.diagnostics
+              },
+              duration: result.duration,
+              error:    result.error
+            }
+          end
+          private_class_method :result
+
+          def self.summary_from(data, providers)
+            model_data = data.fetch("model")
+            provider = providers.find { it.name == model_data.fetch("provider") }
+            raise Error, "report uses unknown provider #{model_data.fetch("provider").inspect}" unless provider
+
+            model = Model.new(id: model_data.fetch("id"), provider:, settings: model_data.fetch("settings", {}))
+            placeholder_cases = data.fetch("results").map do |result_data|
+              case_data = result_data.fetch("case")
+              kase = Case.new(id: case_data.fetch("id"),
+                dataset: case_data.fetch("dataset"),
+                variant: case_data.fetch("variant"),
+                input: case_data.fetch("input"),
+                expected: case_data["expected"],
+                path: "",
+                metadata: (case_data["metadata"] || {}).transform_keys(&:to_sym))
+              [result_data, kase]
+            end
+            experiment = SavedExperiment.new(model:)
+            results = placeholder_cases.map do |result_data, kase|
+              reply_data = result_data["reply"]
+              reply = if reply_data
+                        Reply.new(output: reply_data["output"],
+                          raw: reply_data["raw"],
+                          usage: reply_data["usage"],
+                          error: reply_data["error"],
+                          metadata: reply_data["metadata"] || {})
+                      end
+              score_data = result_data["score"]
+              if score_data
+                score = Score.new(value: score_data.fetch("value"),
+                  metrics: score_data["metrics"] || {},
+                  outcomes: score_data["outcomes"] || {},
+                  diagnostics: score_data["diagnostics"] || {})
+              end
+              RunResult.new(experiment:, case: kase, reply:, score:, duration: result_data.fetch("duration", 0.0), error: result_data["error"])
+            end
+
+            Experiment::Summary.new(model:,
+              dataset: data.fetch("dataset"),
+              name: data.fetch("name"),
+              means: data.fetch("means"),
+              results:,
+              errors: data.fetch("errors", []),
+              duration: data.fetch("duration", 0.0),
+              permalink: data["permalink"],
+              usage: Usage.from(data["usage"]))
+          end
+          private_class_method :summary_from
+        end
+      end
+    end
+  end
+end