llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/providers/type_safe.rb

@@ -1,52 +1,153 @@
 # frozen_string_literal: true
 
+require "json"
+require "net/http"
+require "openssl"
+
 module LLM
   module Evals
     module Providers
       # TypeSafe's own API, which serves Jev as typed questions (noul, choice,
       # score) rather than chat: request type `typesafe`, reached only through
-      # a `direct: typesafe:` entry, whose ids read "typesafe/<name>". Jev on
-      # OpenRouter, `typesafe/jev-router` under `vendors:`, is a chat model
+      # a `typesafe:` section, whose ids read "typesafe/<name>". Jev on
+      # OpenRouter, `typesafe/jev-router` under `openrouter:`, is a chat model
       # and goes through {OpenRouter}; the contender carries which is meant.
-      # The gem knows this API's model list, key and pricing, enough for
-      # `verify` and the cost column; the request shape waits for the first
-      # suite that declares `request_types: [typesafe]`.
+      # A suite's adapter reaches it through {Models::Model#ask}.
       class TypeSafe < Base
+        # The namespace of this API's model ids.
         PREFIX = "typesafe/"
+        # The model list, which {#catalog} reads; {Client::URL} takes the questions.
         URL = "https://api.typesafe.ai/v1/models"
+        # The environment variable holding the API key.
         KEY = "TYPESAFE_API_KEY"
 
         # TypeSafe bills input tokens only: $0.042 per million for jev-latest
         # and jev-preview (docs.typesafe.ai/models, read 2026-10-14). Output is free.
         USD_PER_INPUT_TOKEN = 0.042 / 1_000_000
 
+        # An answer TypeSafe refused, with its status, for {Models::Throttle} to judge.
+        class HTTPError < Error
+          attr_reader :status, :retry_after
+
+          # @!attribute [r] status
+          #   @return [Integer] the HTTP status code
+
+          # @!attribute [r] retry_after
+          #   @return [Float, nil] seconds from the `retry-after` header, when it is numeric
+
+          # Reads the status and hint off the response, and keeps at most 500 characters
+          # of the body in the message.
+          #
+          # @param response [Net::HTTPResponse]
+          def initialize(response)
+            @status = response.code.to_i
+            @retry_after = Float(response["retry-after"], exception: false) if response["retry-after"]
+            super("TypeSafe answered #{@status}: #{response.body.to_s[0, 500]}")
+          end
+        end
+
+        # Overrides {Base#name}.
+        #
+        # @return [String] "typesafe"
         def name = "typesafe"
+        # Overrides {Base#request_type}: only suites declaring `typesafe` admit this.
+        #
+        # @return [Symbol] `:typesafe`
         def request_type = :typesafe
+        # Overrides {Base#request_type_description}.
+        #
+        # @return [String]
         def request_type_description = "typed questions: noul, choice, score"
+        # Overrides {Base#direct?}: reached only through a `typesafe:` section.
+        #
+        # @return [Boolean] true
         def direct? = true
+        # Overrides {Base#prefix}.
+        #
+        # @return [String] {PREFIX}
         def prefix = PREFIX
+        # Overrides {Base#env_keys}.
+        #
+        # @return [Array<String>] just {KEY}
         def env_keys = [KEY]
 
         # Jev takes no reasoning effort.
+        #
+        # @return [Array<String>] always empty
         def efforts = []
 
+        # Overrides {Base#catalog}, with the key as a bearer token.
+        #
         # @return [Array<String>] the models, as "typesafe/<name>"
+        # @raise [Error] when the list cannot be read
+        # @raise [KeyError] when the body has no `models`, or an entry no `name`
         def catalog
           Catalog.fetch(URL, "Authorization" => "Bearer #{ENV.fetch(KEY, nil)}").fetch("models").map { "#{PREFIX}#{it.fetch("name")}" }
         end
 
-        # @param input_tokens [Integer]
-        # @return [Usage] with the cost at TypeSafe's input rate
+        # Prices a call at TypeSafe's input rate; output tokens are counted but free.
+        #
+        # @param input_tokens [Integer] tokens billed
+        # @param output_tokens [Integer] tokens returned, which cost nothing
+        # @return [Stats::Usage] with the cost at TypeSafe's input rate
         def usage(input_tokens:, output_tokens: 0)
-          Usage.new(input_tokens: input_tokens, output_tokens: output_tokens, cost_usd: input_tokens * USD_PER_INPUT_TOKEN)
+          Stats::Usage.new(input_tokens: input_tokens, output_tokens: output_tokens, cost_usd: input_tokens * USD_PER_INPUT_TOKEN)
         end
 
-        # @raise [Error] always: there is no typesafe consumer yet
+        # Overrides {Base#complete} only to refuse, naming the method to call instead.
+        #
+        # @param model [Models::Model] the model that was called
+        # @param system [String, nil] ignored
+        # @param user [String] ignored
+        # @param json [Boolean] ignored
+        # @return [void] never returns
+        # @raise [Error] always: this API answers typed questions; see {#ask}
         def complete(model, system:, user:, json: true) # rubocop:disable Lint/UnusedMethodArgument
-          raise Error,
-            "#{model.id}: no typesafe consumer yet; the typed request shape is defined by the first suite " \
-            "that declares request_types: [typesafe]"
+          raise Error, "#{model.id}: TypeSafe answers typed questions, not prompts; call Model#ask"
         end
+
+        # One request of typed questions over one state. They run in parallel
+        # and cannot see each other's answers.
+        #
+        # @param model [Models::Model] the model to ask
+        # @param state [Object] what the questions are about; any JSON-encodable value
+        # @param questions [Hash{String => Hash}] by name: type, instructions, criteria
+        # @return [Results::ProviderResponse] `raw` holds the answers by name; `text` is them as JSON
+        # @raise [HTTPError] on any status but 2xx
+        # @raise [Error] on a missing key, or a 2xx that is not a usable answer
+        def ask(model, state:, questions:)
+          body = client.ask({ model: model.name, state: state, questions: questions })
+          answers = body["answers"]
+          tokens = body.fetch("usage", {})
+          Results::ProviderResponse.new(text: JSON.generate(answers),
+            usage: usage(input_tokens: tokens["input_tokens"].to_i, output_tokens: tokens["output_tokens"].to_i),
+            raw: answers)
+        end
+
+        # Overrides {Base#retryable?}.
+        #
+        # @param error [StandardError] what a call raised
+        # @return [Boolean] 429, 529 and other server errors, and dropped connections
+        def retryable?(error)
+          case error
+          when HTTPError then error.status == 429 || error.status >= 500
+          when Net::OpenTimeout, Net::ReadTimeout, SocketError, SystemCallError, IOError, OpenSSL::SSL::SSLError then true
+          else false
+          end
+        end
+
+        # Overrides {Base#retry_after}.
+        #
+        # @param error [StandardError] what a call raised
+        # @return [Numeric, nil] the header's seconds from an {HTTPError}; otherwise nil
+        def retry_after(error) = error.is_a?(HTTPError) ? error.retry_after : nil
+
+        private
+
+        # Built on first use and reused.
+        #
+        # @return [Client]
+        def client = @client ||= Client.new
       end
     end
   end

data/lib/llm/evals/providers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Who answers a model's calls: one class per API, and the registry that picks among them.
+    module Providers
+    end
+  end
+end

data/lib/llm/evals/render/reports/call_stats_table.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      # The saved snapshot's presentations: JSON, HTML, terminal, and the pipeline joining them.
+      module Reports
+        # {Stats::CallStats} as the cells of a table, which the caller draws.
+        class CallStatsTable
+          # The table's column titles, in the order {#rows} fills them.
+          HEADER = ["calls by provider, vendor, model", "requests", "errors", "req/s", "avg", "p95"].freeze
+          # What a cell shows for a value that is missing.
+          EMPTY = "-"
+
+          # Formats only; the numbers stay {Stats::CallStats}'s to work out.
+          #
+          # @param stats [Stats::CallStats]
+          def initialize(stats)
+            @stats = stats
+          end
+
+          # The name and counts stay as they are; the rate and timings become Strings.
+          #
+          # @return [Array<Array>] one per {Stats::CallStats::Row}, in {HEADER}'s order
+          def rows
+            @stats.rows.map do |row|
+              [row.name, row.requests, row.errors, rate(row.per_second), seconds(row.latency), seconds(row.p95)]
+            end
+          end
+
+          private
+
+          # @param value [Float, nil] requests per second
+          # @return [String] two decimals, or {EMPTY}
+          def rate(value) = value ? format("%.2f", value) : EMPTY
+
+          # @param value [Float, nil] seconds
+          # @return [String] two decimals with an "s", or {EMPTY}
+          def seconds(value) = value ? format("%.2fs", value) : EMPTY
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html/dataset_section.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class HTML
+          # One dataset's section of the page: one entry per experiment, and its cases.
+          class DatasetSection
+            # Does nothing until {#to_h}.
+            #
+            # @param name [String]
+            # @param summaries [Array<Actions::Experiment::Summary>] the dataset's
+            # @param ranking [Ranking]
+            def initialize(name, summaries, ranking)
+              @name = name
+              @summaries = summaries
+              @ranking = ranking
+            end
+
+            # Ranks only the experiments that covered every case of this dataset.
+            #
+            # @return [Hash{Symbol => Object}] data for the section
+            def to_h
+              case_keys = results.map { it.case.key }.uniq
+              entries = @summaries.map { @ranking.entry([it], case_keys) }
+              complete = entries.select { it[:complete] }
+              @ranking.rank(complete)
+              { name: @name, summaries: @summaries, case_keys:,
+                winner: complete.max_by { it[:ranking] || -1 },
+                entries: entries.sort_by { it[:ranking] || -1 }.reverse,
+                cases: first_run_of_each_case }
+            end
+
+            private
+
+            # @return [Array<Results::RunResult>] every experiment's runs in this dataset
+            def results = @summaries.flat_map(&:results)
+
+            # Each case appears once on the page, so its input and expected answer come from its first run.
+            #
+            # @return [Array<Results::RunResult>] ordered by variant, number and id
+            def first_run_of_each_case
+              results.group_by { it.case.key }.values.map(&:first).sort_by { [it.case.variant, it.case.number || 0, it.case.id] }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html/format.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class HTML
+          # How the report's page prints numbers. Not {CLI::Formatting}: a page shows
+          # a missing value as an em dash, and rounds a cost up to the cent.
+          module Format
+            private
+
+            # Scales a share of 1 to 100%, and shows a missing value as an em dash.
+            #
+            # @param value [Numeric, nil]
+            # @return [String]
+            def percent(value) = value.nil? ? "—" : format("%.1f%%", value * 100)
+
+            # Goes through Rational and rounds up, so float noise cannot change a cost and a cost is never understated.
+            #
+            # @param value [Numeric]
+            # @return [String]
+            def money(value)
+              cents = (value.to_s.to_r * 100).ceil
+              format("$%<dollars>d.%<cents>02d", dollars: cents / 100, cents: cents % 100)
+            end
+
+            # Groups the digits with commas; fractions are dropped.
+            #
+            # @param value [Numeric]
+            # @return [String]
+            def tokens(value) = value.to_i.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html/global_standings.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class HTML
+          # The page's overall ranking: one entry per model at one setting, over
+          # every dataset. Only complete contenders are ranked.
+          class GlobalStandings
+            # Does nothing until {#to_h}.
+            #
+            # @param summaries [Array<Actions::Experiment::Summary>]
+            # @param ranking [Ranking]
+            def initialize(summaries, ranking)
+              @summaries = summaries
+              @ranking = ranking
+            end
+
+            # Coverage is judged against every case key seen in the report, across all datasets.
+            #
+            # @return [Hash{Symbol => Object}] report entries and ranking after coverage checks
+            def to_h
+              expected_keys = @summaries.flat_map(&:results).map { it.case.key }.uniq
+              entries = groups.map { @ranking.entry(it, expected_keys) }
+              complete = entries.select { it[:complete] }
+              @ranking.rank(complete)
+              ranked = complete.sort_by { it[:ranking] || -1 }
+              { entries:, expected_count: expected_keys.length, ranked:, leader: ranked.last, best_coverage: entries.max_by { it[:successful] } }
+            end
+
+            private
+
+            # @return [Array<Array<Actions::Experiment::Summary>>] one group per model at one setting
+            def groups = @summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html/ranking.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class HTML
+          # How the page measures a contender and ranks a group of them. Standings
+          # and dataset sections differ in what they group, never in this.
+          class Ranking
+            # Without a score, or after any failure, a result has nothing to contribute.
+            #
+            # @param result [Results::RunResult]
+            # @return [Boolean, nil] whether the result counts for nothing; nil, which is falsy, for a scored run with no reply
+            def self.failed?(result) = !result.score || result.failed? || result.reply&.error?
+
+            # Holds no per-group state, so standings and sections can share one.
+            #
+            # @param metrics [Array<Scoring::Metric>]
+            # @param weighted_score [Scoring::WeightedScore, nil]
+            def initialize(metrics:, weighted_score:)
+              @metrics = metrics
+              @weighted_score = weighted_score
+            end
+
+            # The metric marked primary, or "score" when none is; what ranks without a weighted score.
+            #
+            # @return [String]
+            def primary_metric = @metrics.find(&:primary)&.name || "score"
+
+            # Measures one contender. Complete means every expected case succeeded with no failures,
+            # and only a complete contender may be ranked. Repeat runs of a case count once.
+            #
+            # @param group [Array<Actions::Experiment::Summary>]
+            # @param expected_keys [Array<String>]
+            # @return [Hash{Symbol => Object}] model coverage and score inputs
+            def entry(group, expected_keys)
+              results = group.flat_map(&:results).uniq { it.case.key }
+              failures = results.count { self.class.failed?(it) }
+              successful_keys = results.reject { self.class.failed?(it) }.map { it.case.key }.uniq
+              {
+                model:         group.first.model,
+                successful:    successful_keys.length,
+                total:         expected_keys.length,
+                errors:        failures,
+                cost:          group.sum { it.usage.cost_usd },
+                input_tokens:  group.sum { it.usage.input_tokens },
+                output_tokens: group.sum { it.usage.output_tokens },
+                row:           row(group, results, failures),
+                complete:      expected_keys.length.positive? && successful_keys.sort == expected_keys.uniq.sort && failures.zero?
+              }
+            end
+
+            # Records each entry's :ranking, compared within the entries given.
+            #
+            # @param entries [Array<Hash{Symbol => Object}>]
+            # @return [void]
+            def rank(entries)
+              rows = entries.map { it[:row] }
+              scores = @weighted_score ? @weighted_score.call(rows) : rows.map { it[primary_metric] }
+              entries.zip(scores).each { |entry, score| entry[:ranking] = score }
+            end
+
+            private
+
+            # Means per metric, plus the columns {Scoring::WeightedScore} reads besides.
+            #
+            # @param group [Array<Actions::Experiment::Summary>] the contender's experiments
+            # @param results [Array<Results::RunResult>] its runs, one per case
+            # @param failures [Integer] how many of those failed
+            # @return [Hash{String => Numeric, nil}] by metric name, plus "score", "errors", "time" and "cost"
+            def row(group, results, failures)
+              metric_means = @metrics.to_h { |metric| [metric.name, mean(results.filter_map { it.score&.metrics&.fetch(metric.name, nil) })] }
+              scores = results.filter_map { it.score&.value unless it.failed? || it.reply&.error? }
+              metric_means.merge("score" => mean(scores), "errors" => failures, "time" => group.sum(&:duration), "cost" => group.sum { it.usage.cost_usd })
+            end
+
+            # @param values [Array<Numeric>]
+            # @return [Float, nil] nil with no values
+            def mean(values) = values.empty? ? nil : values.sum.fdiv(values.size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/render/reports/html/view_model.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Render
+      module Reports
+        class HTML
+          # What the report's page shows, worked out from a decoded snapshot; the
+          # template and its markup stay with {HTML}.
+          class ViewModel
+            include Format
+
+            # @return [JSON::Loaded] the decoded snapshot behind every answer here
+            attr_reader :report
+
+            # Shares one {Ranking} between the standings and every dataset section.
+            #
+            # @param report [JSON::Loaded]
+            def initialize(report)
+              @report = report
+              @ranking = Ranking.new(metrics: report.metrics, weighted_score: report.weighted_score)
+            end
+
+            # The page's heading.
+            #
+            # @return [String]
+            def report_title = "#{configuration_name} evaluation report"
+
+            # The suite's name, as the snapshot recorded it.
+            #
+            # @return [String]
+            def configuration_name = report.suite
+
+            # @return [Array<Actions::Experiment::Summary>] every experiment in the snapshot
+            def summaries = report.summaries
+
+            # @return [Scoring::WeightedScore, nil] nil when the lineup set no weights
+            def weighted_score = report.weighted_score
+
+            # Across every result in the report, with tokens and cost already formatted for display.
+            #
+            # @return [Hash{Symbol => Object}] health counts and cost for the template
+            def health_summary
+              usage = summaries.map(&:usage).reduce(Stats::Usage.zero, :+)
+              failed = results.count { Ranking.failed?(it) }
+              { successful:    results.size - failed,
+                total:         results.size,
+                failed:,
+                input_tokens:  tokens(usage.input_tokens),
+                output_tokens: tokens(usage.output_tokens),
+                cost:          money(usage.cost_usd) }
+            end
+
+            # The whole page's ranking, built fresh on each call.
+            #
+            # @return [Hash{Symbol => Object}] see {GlobalStandings#to_h}
+            def global_standings = GlobalStandings.new(summaries, @ranking).to_h
+
+            # One section per dataset, in name order.
+            #
+            # @return [Array<Hash{Symbol => Object}>] see {DatasetSection#to_h}
+            def dataset_reports
+              summaries.group_by(&:dataset).sort.map { |name, group| DatasetSection.new(name, group, @ranking).to_h }
+            end
+
+            # Details prefer the error of a run that raised over its score's diagnostics.
+            #
+            # @param kase [Datasets::Case]
+            # @param group [Array<Actions::Experiment::Summary>]
+            # @return [Array<Hash{Symbol => Object}>] runs and display values for one case
+            def case_results(kase, group)
+              group.flat_map(&:results).select { it.case.key == kase.key }.map do |run|
+                details = run.failed? ? { "error" => run.error } : run.score&.diagnostics
+                { run:, status: status(run), details: details || {} }
+              end
+            end
+
+            # Hides the entry's shape from the template.
+            #
+            # @param entry [Hash{Symbol => Object}]
+            # @return [Float, nil]
+            def ranking_value(entry) = entry[:ranking]
+
+            # The name of what ranks when no weighted score does.
+            #
+            # @return [String]
+            def primary_metric = @ranking.primary_metric
+
+            # The sentence under the standings explaining how they were ordered.
+            #
+            # @return [String]
+            def weight_description
+              weighted_score ? "Weighted score from the lineup; chart order runs worst to best." : "Ranked by #{primary_metric}; chart order runs worst to best."
+            end
+
+            private
+
+            # @return [Array<Results::RunResult>] every run in the report
+            def results = summaries.flat_map(&:results)
+
+            # The run's error, the model's failure, its percentage, or "No score".
+            #
+            # @param run [Results::RunResult]
+            # @return [String]
+            def status(run)
+              return run.error if run.failed?
+              return run.reply.error if run.reply&.error?
+
+              run.score ? percent(run.score.value) : "No score"
+            end
+          end
+        end
+      end
+    end
+  end
+end