llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/cli.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module LLM
   module Evals
     # Every command the executable answers to. The help's title and
@@ -8,17 +6,50 @@ module LLM
     module CLI
       extend Dry::CLI::Registry
 
+      class << self
+        # @return [String] the help's title
+        def header
+          p = Pastel.new
+          p.green.inverse("\n #{Runtime::Configuration::DEFAULT_NAME} ") +
+            p.yellow.inverse(" v#{VERSION} ") +
+            p.cyan.inverse.bold(" © 2026 Konstantin Gredeskoul, MIT License     ") +
+            p.cyan("") + p.reset("\n")
+        end
+
+        # Confirms which evals are running: a green box naming the suite and its
+        # file, or a yellow one saying there is no file.
+        #
+        # @param configuration [LLM::Evals::Runtime::Configuration]
+        # @param color [Boolean]
+        # @param width [Integer] the box's whole width; a console reporting no size, as CI does, gets the narrowest
+        # @return [String]
+        def dataset_conf_label(configuration:, color: false, width: IO.console&.winsize&.last.to_i.clamp(60..))
+          box = StatusBox.new(width:, color:)
+          return box.render(:warning, configuration.config_missing_message) unless configuration.config_found?
+
+          box.render(:found, "Evals configuration: #{configuration.name}\nFile: #{configuration.relative_config}")
+        end
+      end
+
       # What users type, so what the help and shell completion are written for.
       # exe/llm-evals exists only for a system where `evals` is taken.
-      PROGRAM = "evals"
+      PROGRAM = "evals".freeze
+
+      Dry::CLI::Help::Screens::Command.prepend(OptionGroups)
 
       Dry::CLI::Help.configure do
         configuration = LLM::Evals.configuration
-        title configuration.name == Configuration::DEFAULT_NAME ? PROGRAM : "#{PROGRAM} (#{configuration.name})"
-        types = configuration.providers.uniq(&:request_type).map { "#{it.request_type} (#{it.request_type_description})" }
+
+        title ::LLM::Evals::CLI.header
+
+        types = configuration.providers.uniq(&:request_type).map do |provider|
+          "#{provider.request_type} (#{provider.request_type_description})"
+        end
+
         description "#{configuration.description} Request types: #{types.join("; ")}."
 
-        columns = IO.console&.winsize&.last || 80
+        columns = (IO.console&.winsize&.last || 80).clamp(60, 90)
+
         width columns.clamp(60, 120)
         exit_code_without_arguments 0
 
@@ -36,6 +67,8 @@ module LLM
       register "render", Render
       register "cases", Cases
       register "verify", Verify
+      register "upgrade-config", UpgradeConfig
+      register "weights", Weights
       register "version", Version, aliases: ["-V", "--version"]
       register "completion", Dry::CLI::Autocomplete::Command[self, program_name: PROGRAM]
     end

data/lib/llm/evals/config/contender_builder.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    class Config
+      # One contender per effort and temperature a model's settings list.
+      class ContenderBuilder
+        # @param defaults [Hash{String => Object}] under the entry's own settings
+        # @param overrides [Hash{String => Object}] over them; nil entries are ignored
+        def initialize(defaults:, overrides:)
+          @defaults = defaults
+          @overrides = overrides.compact
+        end
+
+        # Expands one model's lists of efforts and temperatures into their product.
+        #
+        # @param model [String] the model id
+        # @param entry [Hash{String => Object}] the model's settings in the file
+        # @param provider [Symbol, nil] a direct provider's name; nil when the id picks the provider
+        # @return [Array<Models::Contender>]
+        def call(model, entry = {}, provider: nil)
+          settings = @defaults.merge(entry).merge(@overrides)
+          Array(settings["effort"]).product(Array(settings["temperature"])).map do |effort, temperature|
+            Models::Contender.new(model:, effort: effort.to_s, temperature:, max_tokens: Integer(settings["max_tokens"]), provider:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/config/error_limits.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    class Config
+      # The `errors:` section: when to give up on a run. Both limits must be met,
+      # and `percent` needs `total`, since one failed first call is already 100%.
+      #
+      # @!attribute [r] total
+      #   @return [Integer] errors before the run may abort
+      # @!attribute [r] percent
+      #   @return [Numeric, nil] the share of finished calls that must have failed
+      ErrorLimits = Data.define(:total, :percent) do
+        # Checks the section as the file wrote it.
+        #
+        # @param section [Object] the section as the file has it, not nil
+        # @param path [String] names the file in an error
+        # @return [ErrorLimits]
+        # @raise [Error] on a limit that is missing or out of range
+        def self.from_hash(section, path:)
+          limits = section["abort_after"] if section.is_a?(Hash)
+          raise Error, "#{path}: errors needs an abort_after: mapping of total and percent" unless limits.is_a?(Hash)
+
+          total = limits["total"]
+          percent = limits["percent"]
+          raise Error, "#{path}: errors.abort_after.total must be a whole number above 0" unless total.is_a?(Integer) && total.positive?
+          unless percent.nil? || (percent.is_a?(Numeric) && percent.positive? && percent <= 100)
+            raise Error, "#{path}: errors.abort_after.percent must be above 0 and at most 100"
+          end
+
+          new(total:, percent:)
+        end
+
+        # A new one each time, since a budget counts one run's calls.
+        #
+        # @return [Actions::ErrorBudget]
+        def budget = Actions::ErrorBudget.new(total:, percent:)
+      end
+    end
+  end
+end

data/lib/llm/evals/config/listing.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    class Config
+      # What a lineup's provider sections list: the models and the `rate_limit`s,
+      # checked and read once, so that a file wrong in either way is refused
+      # whichever of the two a command asks for first.
+      class Listing
+        # The keys a provider section may have.
+        SECTION_KEYS = %w[rate_limit vendors models].freeze
+        # The keys an entry under `vendors:` may have.
+        VENDOR_KEYS = %w[rate_limit models].freeze
+        # The keys one line under `models:` may have.
+        ENTRY_KEYS = %w[model effort temperature max_tokens rate_limit].freeze
+
+        # What marks a mapping as meant for a provider, under any name. Other
+        # mappings belong to trackers and suites, so they are left alone.
+        MARKERS = %w[vendors models model rate_limit].freeze
+
+        # One line under `models:`.
+        #
+        # @!attribute [r] provider
+        #   @return [Providers::Base] the section it was listed under
+        # @!attribute [r] id
+        #   @return [String] the id a provider recognises
+        # @!attribute [r] settings
+        #   @return [Hash{String => Object}] the line as the file has it, with `model` as a key
+        Entry = Data.define(:provider, :id, :settings)
+
+        # @return [Array<Entry>] in the file's order
+        attr_reader :entries
+
+        # @return [Hash{String => Models::Throttle::Limit}] by scope
+        attr_reader :limits
+
+        # @param data [Hash] the whole file
+        # @param path [String] named in every error
+        # @param providers [Providers::Registry] whose names the sections may use
+        # @raise [Error] on a section, entry or limit the file format does not have
+        def initialize(data, path:, providers:)
+          @path = path
+          @providers = providers
+          @entries = []
+          @limits = {}
+          sections(data).each { |provider, section| read(provider, section) }
+        end
+
+        private
+
+        attr_reader :path, :providers
+
+        # @!attribute [r] path
+        #   The lineup file, named in every error.
+        #   @return [String]
+
+        # @!attribute [r] providers
+        #   Whose names the sections may use.
+        #   @return [Providers::Registry]
+
+        # @param data [Hash] the whole file
+        # @return [Array<Array(Providers::Base, Hash)>] the file's provider sections
+        # @raise [Error] on a retired key, an unknown provider or an unknown key
+        def sections(data)
+          retired = %w[vendors direct] & data.keys
+          raise Error, "#{path}: #{retired.join(" and ")} no longer exist; run `evals upgrade-config` to move them under provider sections" if retired.any?
+
+          names = providers.map(&:name)
+          data.filter_map do |name, section|
+            next unless section.is_a?(Hash) && (names.include?(name.to_s) || section.keys.intersect?(MARKERS))
+            raise Error, "#{path}: unknown provider #{name.inspect}; known: #{names.join(", ")}" unless names.include?(name.to_s)
+
+            known!(name, section, SECTION_KEYS)
+            [providers.find { it.name == name.to_s }, section]
+          end
+        end
+
+        # Collects one provider section's limits and models, those under its vendors first.
+        #
+        # @param provider [Providers::Base] the section's provider
+        # @param section [Hash] the section as the file has it
+        # @return [void]
+        # @raise [Error] on a section the file format does not have
+        def read(provider, section)
+          limit(provider.name, section)
+          vendors = section["vendors"] || {}
+          raise Error, "#{path}: vendors must be a mapping of vendor to models" unless vendors.is_a?(Hash)
+
+          vendors.each do |vendor, listing|
+            raise Error, "#{path}: vendor #{vendor} needs a models: list" unless listing.is_a?(Hash)
+
+            known!(vendor, listing, VENDOR_KEYS)
+            limit("#{provider.name}/#{vendor}", listing)
+            list(provider, vendor, listing, "#{provider.name}/#{vendor}/", "#{provider.prefix}#{vendor}/")
+          end
+          list(provider, provider.name, section, "#{provider.name}/", provider.prefix.to_s)
+        end
+
+        # Adds each line of a `models:` list to {#entries}, and any limit it sets to {#limits}.
+        #
+        # @param provider [Providers::Base] the section's provider
+        # @param owner [String] the vendor or provider that lists them, for errors
+        # @param listing [Hash] the mapping holding `models:`
+        # @param scope [String] prepended to the model name to key its limit
+        # @param prefix [String] prepended to the model name to make its id
+        # @return [void]
+        # @raise [Error] on an entry without a model, or with an unknown key
+        def list(provider, owner, listing, scope, prefix)
+          Array(listing["models"]).each do |entry|
+            entry = { "model" => entry } if entry.is_a?(String)
+            raise Error, "#{path}: every entry under #{owner} needs a model" unless entry.is_a?(Hash) && entry["model"]
+
+            known!(entry["model"], entry, ENTRY_KEYS)
+            limit("#{scope}#{entry["model"]}", entry)
+            @entries << Entry.new(provider:, id: "#{prefix}#{entry["model"]}", settings: entry)
+          end
+        end
+
+        # Refuses keys the format does not have, so a typo is not ignored.
+        #
+        # @param owner [String] names the mapping in the error
+        # @param mapping [Hash] the mapping to check
+        # @param keys [Array<String>] the keys it may have
+        # @return [void]
+        # @raise [Error] on any other key
+        def known!(owner, mapping, keys)
+          unknown = mapping.keys - keys
+          raise Error, "#{path}: unknown keys under #{owner}: #{unknown.join(", ")}; known: #{keys.join(", ")}" if unknown.any?
+        end
+
+        # A scope has one limit, so a model listed twice, say at two efforts, may
+        # set it once: the second would otherwise replace the first in silence.
+        #
+        # @param scope [String] the key {Models::Throttle} looks the limit up by
+        # @param owner [Hash] the mapping that may hold `rate_limit:`
+        # @return [void]
+        # @raise [Error] on a limit that is malformed or set twice
+        def limit(scope, owner)
+          given = owner["rate_limit"]
+          return if given.nil?
+
+          burst, per_minute = given.values_at("burst", "per_minute") if given.is_a?(Hash)
+          unless [burst, per_minute].all? { it.is_a?(Integer) && it.positive? } && given.size == 2
+            raise Error, "#{path}: rate_limit of #{scope} needs burst and per_minute, whole numbers above 0"
+          end
+          raise Error, "#{path}: rate_limit of #{scope} is set twice; a model listed twice shares one limit, so set it once" if @limits.key?(scope)
+
+          @limits[scope] = Models::Throttle::Limit.new(burst:, per_minute:)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/config/results.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    class Config
+      # The `results:` section: where completed runs are saved and how they are
+      # presented. Paths are rooted at the caller's working directory, so the same
+      # config works from a project root or when invoked by a task runner.
+      #
+      # @!attribute [r] autosave
+      #   @return [Boolean] whether completed runs are saved under {#destination}
+      # @!attribute [r] destination
+      #   @return [String] absolute directory for saved runs
+      # @!attribute [r] temp_dir
+      #   @return [String] absolute directory for the temporary JSON used when autosave is off
+      # @!attribute [r] display_mode
+      #   @return [String] one of {RESULT_DISPLAYS}
+      Results = Data.define(:autosave, :destination, :temp_dir, :display_mode) do
+        # Applies the defaults, checks the types, and roots the paths at the working directory.
+        #
+        # @param section [Object] the section as the file has it
+        # @param path [String] names the file in an error
+        # @return [Results]
+        # @raise [Error] when a setting has the wrong type or an unknown display
+        def self.from_hash(section, path:)
+          raise Error, "#{path}: results must be a mapping" unless section.is_a?(Hash)
+
+          autosave = section.fetch("autosave", true)
+          raise Error, "#{path}: results.autosave must be true or false" unless [true, false].include?(autosave)
+
+          display = section.fetch("display", DEFAULT_DISPLAY).to_s
+          raise Error, "#{path}: results.display must be one of #{RESULT_DISPLAYS.join(", ")}" unless RESULT_DISPLAYS.include?(display)
+
+          new(autosave:,
+            destination:  File.expand_path(section.fetch("destination", "./evals/results").to_s, Dir.pwd),
+            temp_dir:     File.expand_path(section.fetch("temp_dir", "./tmp").to_s, Dir.pwd),
+            display_mode: display)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/config/score_weights.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    class Config
+      # The `weighted_score:` section; see {Scoring::WeightedScore}.
+      module ScoreWeights
+        # Only checks the shape; {Scoring::WeightedScore} judges the columns and numbers.
+        #
+        # @param section [Object, nil] the section as the file has it
+        # @param path [String] names the file in an error
+        # @return [Hash{String => Object}] by column; empty when the file sets none
+        # @raise [Error] when either level is not a mapping
+        def self.from_hash(section, path:)
+          return {} if section.nil?
+
+          weights = section["weights"] if section.is_a?(Hash)
+          raise Error, "#{path}: weighted_score needs a weights: mapping of column to number" unless weights.is_a?(Hash)
+
+          weights.transform_keys(&:to_s)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/config/upgrade.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module LLM
+  module Evals
+    class Config
+      # Rewrites a lineup from top-level `vendors:` and `direct:` to provider
+      # sections. It works on the text, since loading YAML would drop the comments
+      # that explain a lineup, then proves the result means the same before
+      # anything is written.
+      class Upgrade
+        # A line that opens a new top-level key, which ends any section being moved.
+        TOP_KEY = /\A[\w-]+:/
+        # The old `vendors:` line, keeping whatever follows the colon.
+        VENDORS = /\Avendors:(?<rest>.*)/m
+        # The old `direct:` line, with at most a comment after it.
+        DIRECT = /\Adirect:\s*(?<comment>#.*)?\n?\z/
+        # A key indented by two spaces: a vendor, or a direct provider.
+        GROUP = /\A  ([\w-]+):([^\n]*)\n?\z/
+
+        # @param text [String] the lineup as it is
+        # @param path [String] named in every error
+        # @raise [Error] when the text is not YAML, or not a mapping
+        def initialize(text, path:)
+          @text = text
+          @path = path
+          @data = YAML.safe_load(text) || {}
+          raise Error, "#{path}: a lineup must be a mapping, not a #{@data.class.name.downcase}" unless @data.is_a?(Hash)
+        rescue Psych::SyntaxError => e
+          raise Error, "#{path}: not valid YAML: #{e.message}"
+        end
+
+        # @return [Boolean] whether the lineup still has the old keys
+        def needed? = @data.key?("vendors") || @data.key?("direct")
+
+        # @return [String] the lineup in the new shape
+        # @raise [Error] when the rewrite would change what the file means
+        def call
+          taken = [("openrouter" if @data.key?("vendors")), *(@data["direct"] || {}).keys].compact & @data.keys
+          raise Error, "#{@path}: cannot upgrade: the file has both the old keys and a section named #{taken.join(", ")}" if taken.any?
+
+          rewritten = rewrite
+          return rewritten if same_meaning?(rewritten)
+
+          raise Error, "#{@path}: cannot upgrade: vendors: or direct: is not in the plain list form; move them by hand"
+        end
+
+        private
+
+        # The proof that a textual rewrite is safe to write.
+        #
+        # @param rewritten [String] the lineup in the new shape
+        # @return [Boolean] whether it loads to what {#expected} says; false when it is not YAML
+        def same_meaning?(rewritten)
+          YAML.safe_load(rewritten) == expected
+        rescue Psych::SyntaxError
+          false
+        end
+
+        # Rewrites the text line by line, tracking which old section the line falls in.
+        #
+        # @return [String] the lineup in the new shape
+        def rewrite
+          section = nil
+          @text.lines.flat_map do |line|
+            section = nil if line.match?(TOP_KEY)
+            if (vendors = VENDORS.match(line))
+              section = :vendors
+              ["openrouter:#{vendors[:rest]}", "  vendors:\n"]
+            elsif (direct = DIRECT.match(line))
+              section = :direct
+              direct[:comment] ? ["#{direct[:comment]}\n"] : []
+            else
+              moved(section, line)
+            end
+          end.join
+        end
+
+        # Reindents one line of an old section; comments and blank lines stay as they are.
+        #
+        # @param section [Symbol, nil] `:vendors`, `:direct`, or nil outside both
+        # @param line [String] the line, with its newline
+        # @return [Array<String>] the line or lines that replace it
+        def moved(section, line)
+          return [line] if section.nil? || line.strip.empty? || line.start_with?("#")
+
+          group = GROUP.match(line)
+          case section
+          when :vendors then group ? ["    #{group[1]}:#{group[2]}\n", "      models:\n"] : ["    #{line}"]
+          else group ? ["#{group[1]}:#{group[2]}\n", "  models:\n"] : [line]
+          end
+        end
+
+        # What the file means in the new shape, built from the parsed data rather than the text.
+        #
+        # @return [Hash{String => Object}] the lineup as {#same_meaning?} expects it to load
+        def expected
+          rest = @data.except("vendors", "direct")
+          vendors = @data["vendors"]&.transform_values { { "models" => it } }
+          direct = (@data["direct"] || {}).transform_values { { "models" => it } }
+          rest.merge(vendors ? { "openrouter" => { "vendors" => vendors } } : {}).merge(direct)
+        end
+      end
+    end
+  end
+end