llm-evals 0.4.0

data/lib/llm/evals/trackers/logfire.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Trackers
+      # Pydantic Logfire, which has no Ruby gem of its own: it is an OTLP
+      # backend for the standard OpenTelemetry Ruby SDK (pydantic.dev/logfire/ruby,
+      # read 2026-10-15). Traces only; Logfire's metrics and logs SDKs are
+      # pre-1.0. One span per experiment, one child span per case.
+      #
+      # The write token comes from `LOGFIRE_TOKEN`, our name for what their
+      # page passes as a raw `Authorization` header. When the standard
+      # `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_HEADERS` is
+      # already set, the SDK's own reading of it wins and {#describe} says so.
+      # The raw reply is not sent unless the config file opts in with
+      # `logfire: { include_replies: true }`.
+      class Logfire < Tracker
+        # Logfire's US region, per their docs on 2026-10-15; other regions
+        # exist, so LOGFIRE_BASE_URL overrides it.
+        DEFAULT_BASE_URL = "https://logfire-us.pydantic.dev"
+        TOKEN = "LOGFIRE_TOKEN"
+        BASE_URL = "LOGFIRE_BASE_URL"
+        OTEL_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"
+        OTEL_HEADERS = "OTEL_EXPORTER_OTLP_HEADERS"
+
+        # Bytes of the diagnostics JSON kept on a case span; the rest is cut,
+        # since an attribute is not the place for a whole transcript.
+        DIAGNOSTICS_LIMIT = 4096
+
+        def self.env_key = TOKEN
+        def self.gems = %w[opentelemetry-sdk opentelemetry-exporter-otlp]
+
+        attr_reader :endpoint, :source
+
+        # @param exporter [#export, nil] an exporter to use instead of OTLP, for tests
+        # @param env [Hash]
+        # @param context [Hash] see {Tracker#initialize}
+        def initialize(exporter: nil, env: ENV, **context)
+          super(**context)
+          @env = env
+          @exporter = exporter
+          @endpoint, @headers, @source = settings(env)
+          @cases = []
+        end
+
+        def describe = "logfire: endpoint #{endpoint || "(SDK default)"}, credentials from #{source}"
+
+        # @return [Boolean] whether raw replies ride along on the case spans
+        def include_replies? = options["include_replies"] == true || options[:include_replies] == true
+
+        def start(experiment)
+          @provider = provider
+          @tracer = @provider.tracer("llm-evals", LLM::Evals::VERSION)
+          @span = @tracer.start_span("experiment #{experiment.name}", attributes: experiment_attributes(experiment))
+        end
+
+        def record(result)
+          now = Time.now
+          span = @tracer.start_span("case #{result.case.key}",
+            with_parent:     OpenTelemetry::Trace.context_with_span(@span),
+            start_timestamp: now - result.duration,
+            attributes:      case_attributes(result))
+          if result.failed?
+            span.record_exception(Error.new(result.error))
+            span.status = OpenTelemetry::Trace::Status.error(result.error)
+          end
+          span.finish(end_timestamp: now)
+        end
+
+        # Closes the experiment span with the summary and flushes. No URL:
+        # Logfire's trace pages are not something the token lets us build.
+        #
+        # @return [Hash] `url:` nil
+        def finish(_experiment, summary)
+          summary.means.each { |metric, mean| @span.set_attribute("evals.mean.#{metric}", mean) unless mean.nil? }
+          @span.set_attribute("evals.errors", summary.errors.size)
+          @span.set_attribute("evals.duration_seconds", summary.duration)
+          @span.set_attribute("evals.usage.cost_usd", summary.usage.cost_usd)
+          @span.finish
+          @provider.force_flush
+          @provider.shutdown unless @exporter # an injected exporter is the caller's to read afterwards
+          { url: nil }
+        end
+
+        private
+
+        # Which credentials win: the standard OTEL_* variables when either is
+        # set, else our LOGFIRE_* pair. Returns the endpoint and headers to
+        # hand the exporter (nil leaves the SDK's own reading in place).
+        def settings(env)
+          if [OTEL_ENDPOINT, OTEL_HEADERS].any? { !env[it].to_s.empty? }
+            [env[OTEL_ENDPOINT], nil, "#{OTEL_ENDPOINT}/#{OTEL_HEADERS} (set in the environment; #{TOKEN} ignored)"]
+          else
+            base = env[BASE_URL].to_s.empty? ? DEFAULT_BASE_URL : env[BASE_URL]
+            ["#{base.chomp("/")}/v1/traces", { "Authorization" => env[TOKEN].to_s }, "#{TOKEN}#{" and #{BASE_URL}" unless env[BASE_URL].to_s.empty?}"]
+          end
+        end
+
+        # A tracer provider of our own, so the global one, which a consumer
+        # may have configured for its application, is left alone.
+        def provider
+          exporter = @exporter || OpenTelemetry::Exporter::OTLP::Exporter.new(**{ endpoint: endpoint, headers: @headers }.compact)
+          processor = if @exporter
+                        OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter)
+                      else
+                        OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(exporter)
+                      end
+          resource = OpenTelemetry::SDK::Resources::Resource.create("service.name" => context[:suite] || "llm-evals")
+          OpenTelemetry::SDK::Trace::TracerProvider.new(resource: resource).tap { it.add_span_processor(processor) }
+        end
+
+        def experiment_attributes(experiment)
+          model = experiment.model
+          { "evals.suite" => context[:suite].to_s, "evals.project" => context[:project].to_s, "evals.experiment" => experiment.name,
+            "evals.model" => model.id, "evals.provider" => model.provider.name, "evals.dataset" => experiment.dataset,
+            "evals.effort" => model.effort.to_s, "evals.temperature" => model.temperature.to_f, "evals.max_tokens" => model.max_tokens.to_i,
+            "evals.cases" => experiment.cases.size }
+        end
+
+        def case_attributes(result)
+          kase = result.case
+          attributes = { "evals.case" => kase.id, "evals.dataset" => kase.dataset, "evals.variant" => kase.variant,
+                         "evals.duration_seconds" => result.duration }
+          return attributes.merge("evals.error" => result.error) if result.failed?
+
+          score = result.score
+          attributes["evals.score"] = score.value
+          score.metrics.each { |metric, value| attributes["evals.metric.#{metric}"] = value unless value.nil? }
+          score.outcomes.group_by { |_, v| v.to_s }.each { |outcome, pairs| attributes["evals.outcome.#{outcome}"] = pairs.size }
+          attributes["evals.diagnostics"] = truncate(JSON.generate(score.diagnostics))
+          attributes["evals.usage.input_tokens"] = result.reply.usage.input_tokens
+          attributes["evals.usage.output_tokens"] = result.reply.usage.output_tokens
+          attributes["evals.usage.cost_usd"] = result.reply.usage.cost_usd
+          attributes["evals.reply_error"] = result.reply.error if result.reply.error?
+          attributes["evals.reply"] = truncate(result.reply.raw.to_s) if include_replies?
+          attributes
+        end
+
+        def truncate(text) = text.bytesize > DIAGNOSTICS_LIMIT ? "#{text.byteslice(0, DIAGNOSTICS_LIMIT)}..." : text
+      end
+    end
+  end
+end

data/lib/llm/evals/trackers/registry.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Trackers
+      # The tracker classes a run may choose from, by name, and the choice
+      # itself: exactly one remote tracker, or none. A consumer or a plugin
+      # gem registers its own with `c.tracker`.
+      class Registry
+        include Enumerable
+
+        # Spellings of "no remote tracker" on the command line and in the config.
+        NONE = %w[none local off].freeze
+
+        # @return [Registry] Braintrust and Logfire
+        def self.default = new([Braintrust, Logfire])
+
+        # @param classes [Array<Class<Tracker>>]
+        def initialize(classes = [])
+          @classes = classes.dup
+        end
+
+        # @param klass [Class<Tracker>] replaces one registered under the same name
+        # @return [Class<Tracker>]
+        def register(klass)
+          @classes.reject! { it.tracker_name == klass.tracker_name }
+          @classes << klass
+          klass
+        end
+
+        def each(&) = @classes.each(&)
+
+        # @return [Array<String>]
+        def names = map(&:tracker_name)
+
+        # @param name [String, Symbol]
+        # @return [Class<Tracker>]
+        # @raise [Error] naming the known trackers
+        def named(name)
+          find { it.tracker_name == name.to_s } || raise(Error, "no tracker named #{name.to_s.inspect}; known: #{names.join(", ")}")
+        end
+
+        # The one remote tracker to run, in this precedence: `--tracker`,
+        # then the config's `tracker:`, then the environment, where a single
+        # tracker's key being set selects it and two or more is an ambiguity
+        # to be settled by naming one. `--local` is none.
+        #
+        # @param flag [String, nil] what `--tracker` was given
+        # @param config [String, nil] the config file's `tracker:`
+        # @param env [Hash]
+        # @param local [Boolean] `--local`
+        # @return [Array(Class<Tracker>, String), nil] the class and why it was chosen; nil for none
+        # @raise [Error] on an unknown name or an ambiguous environment
+        def choose(flag: nil, config: nil, env: ENV, local: false)
+          return nil if local
+          return choice(flag, "--tracker") if present?(flag)
+          return choice(config, "tracker: in the config file") if present?(config)
+
+          detected = @classes.select { it.detect?(env) }
+          case detected.size
+          when 0 then nil
+          when 1 then [detected.first, "#{detected.first.env_key} is set (pass --tracker or set tracker: to choose explicitly)"]
+          else
+            raise Error,
+              "more than one tracker is configured in the environment (#{detected.map(&:env_key).join(", ")}); " \
+              "choose one with --tracker #{detected.map(&:tracker_name).join("|")} or tracker: in the config file"
+          end
+        end
+
+        private
+
+        def present?(value) = !value.nil? && !value.to_s.strip.empty?
+
+        def choice(name, from)
+          return nil if NONE.include?(name.to_s)
+
+          [named(name), from]
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/usage.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What one call cost, as the provider counted it. Summed over a run.
+    Usage = Data.define(:input_tokens, :output_tokens, :cost_usd) do
+      # @return [Usage] zero of everything
+      def self.zero = new(input_tokens: 0, output_tokens: 0, cost_usd: 0.0)
+
+      # Reads a Hash with Symbol or String keys, missing keys as zero.
+      #
+      # @param hash [Hash, Usage, nil]
+      # @return [Usage]
+      def self.from(hash)
+        return hash if hash.is_a?(Usage)
+        return zero if hash.nil?
+
+        h = hash.transform_keys(&:to_sym)
+        new(input_tokens: h.fetch(:input_tokens, 0).to_i,
+          output_tokens: h.fetch(:output_tokens, 0).to_i,
+          cost_usd: h.fetch(:cost_usd, 0.0).to_f)
+      end
+
+      # @param other [Usage]
+      # @return [Usage]
+      def +(other)
+        with(input_tokens: input_tokens + other.input_tokens,
+          output_tokens: output_tokens + other.output_tokens,
+          cost_usd: cost_usd + other.cost_usd)
+      end
+    end
+  end
+end

data/lib/llm/evals/verify.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Offline consistency checks over every case, so a typo in the suite is
+    # caught before it is billed as a model's mistake. The gem checks what
+    # every suite has: an input and a parsable expected answer. The adapter
+    # checks each dataset's shared files, and the scorer each case against
+    # what it will demand of it.
+    class Verify
+      # @param configuration [Configuration]
+      def initialize(configuration = LLM::Evals.configuration)
+        @configuration = configuration
+      end
+
+      # @return [Hash{String => Array<String>}] problems by "dataset/id"; empty when all is sound
+      def problems
+        discovery = @configuration.discovery
+        failures = {}
+        discovery.datasets.each do |dataset|
+          Array(adapter.verify(dataset)).each { (failures[dataset.to_s] ||= []) << it }
+          discovery.cases(dataset: dataset).each do |kase|
+            list = case_problems(kase)
+            failures[kase.key] = list unless list.empty?
+          end
+        end
+        failures
+      end
+
+      # @param kase [Case]
+      # @return [Array<String>]
+      def case_problems(kase)
+        problems = []
+        problems << "input is empty" if kase.input.empty?
+        problems << kase.metadata[:expected_error] if kase.metadata[:expected_error]
+        return problems unless kase.expected?
+
+        problems.concat(Array(scorer.verify(kase, context: adapter.context(kase))))
+      rescue StandardError => e
+        problems << "#{e.class}: #{e.message}"
+      end
+
+      private
+
+      def adapter = @configuration.adapter
+      def scorer = @configuration.scorer
+    end
+  end
+end

data/lib/llm/evals/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    VERSION = "0.4.0"
+  end
+end

data/lib/llm/evals/warnings.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module LLM
+  module Evals
+    # Everything a run would otherwise print as a warning while its progress
+    # bars redraw: cases that failed, and what Braintrust, RubyLLM, OpenTelemetry
+    # (whose exporters report failures on $stdout) and Ruby itself log. Printed among the bars, each one breaks the redraw; collected
+    # here, they are saved to one file, and the run prints only how many.
+    #
+    # It quacks as an IO, so a Logger, and $stderr, can write to it. Threads
+    # share it.
+    class Warnings
+      def initialize
+        @lines = []
+        @lock = Mutex.new
+      end
+
+      # @param text [#to_s]
+      # @return [self]
+      def record(text)
+        text = text.to_s.chomp
+        @lock.synchronize { @lines << text } unless text.strip.empty?
+        self
+      end
+
+      # @return [Integer] the bytes "written", as IO#write does
+      def write(*texts)
+        texts.each { record(it) }
+        texts.sum { it.to_s.bytesize }
+      end
+
+      def puts(*texts) = texts.each { record(it) } && nil
+      def print(*texts) = write(*texts) && nil
+      def <<(text) = record(text)
+      def flush = self
+      def sync = true
+      def sync=(_value); end
+      def close = nil
+      def tty? = false
+
+      # @return [Integer]
+      def count = @lock.synchronize { @lines.size }
+
+      # @return [Array<String>]
+      def to_a = @lock.synchronize { @lines.dup }
+
+      # @return [Boolean] whether nothing was recorded
+      def none? = count.zero?
+
+      # @param path [String]
+      # @return [String] the path
+      def save(path)
+        File.write(path, "#{to_a.join("\n\n")}\n")
+        path
+      end
+
+      # Runs the block with $stderr, Ruby's own warnings, and the loggers of
+      # whichever optional gems are loaded writing here, and puts them back after.
+      #
+      # @return [Object] what the block returns
+      def capture
+        logger = Logger.new(self, level: Logger::WARN)
+        stderr = $stderr
+        kernel_warn = Kernel.instance_method(:warn)
+        warnings = self
+        restore = redirect_loggers(logger)
+        $stderr = self
+        Kernel.define_method(:warn) { |*messages, **_options| warnings.write(messages.join("\n")) }
+        Kernel.send(:private, :warn)
+        yield
+      ensure
+        Kernel.define_method(:warn, kernel_warn) if kernel_warn
+        Kernel.send(:private, :warn)
+        restore&.call
+        $stderr = stderr
+      end
+
+      private
+
+      # @return [Proc] puts the loggers back
+      def redirect_loggers(logger)
+        braintrust = defined?(Braintrust::Log) ? Braintrust::Log.logger : nil
+        ruby_llm = defined?(RubyLLM) ? RubyLLM.instance_variable_get(:@logger) : nil
+        open_telemetry = defined?(OpenTelemetry) ? OpenTelemetry.logger : nil
+        Braintrust::Log.logger = logger if defined?(Braintrust::Log)
+        RubyLLM.instance_variable_set(:@logger, logger) if defined?(RubyLLM)
+        OpenTelemetry.logger = logger if defined?(OpenTelemetry)
+        lambda do
+          Braintrust::Log.logger = braintrust if defined?(Braintrust::Log)
+          RubyLLM.instance_variable_set(:@logger, ruby_llm) if defined?(RubyLLM)
+          OpenTelemetry.logger = open_telemetry if defined?(OpenTelemetry)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/weighted_score.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One number from the leaderboard's columns, weighted as the lineup file
+    # says, for a suite to state what matters most to it. A positive weight
+    # means higher is better; a negative one, lower is better:
+    #
+    #   weighted_score:
+    #     weights:
+    #       score: 0.4
+    #       hallucinated_rate: -0.2
+    #       cost: -0.2
+    #
+    #   weighted_score = Σ(|weight| × goodness) / Σ|weight|
+    #
+    # Goodness runs from 0 to 1. Metrics are used as they are, or as 1 - value
+    # under a negative weight. {MAGNITUDES}
+    # are placed on a log scale between the lowest and highest in the group, so
+    # that each doubling costs the same: on a linear scale, one model a hundred
+    # times the price would squash all the others together. Where the whole
+    # group is equal, each gets 1.
+    #
+    # Errors come off the total, when weighted: 20 points for one, 25 for two, 30
+    # for three, and so on, down to 0. A failed case has no scores, so no other
+    # column notices it, and a single failure must show. A clean contender gets
+    # the whole of the weight on errors.
+    #
+    # Magnitudes are relative, so a weighted score compares within one group
+    # only: another model in the lineup moves everyone's.
+    class WeightedScore
+      # Columns compared on a log scale.
+      MAGNITUDES = %w[time cost].freeze
+
+      # The gem's columns that can be weighted, beside the metrics. `temp` is a
+      # setting, not a result, and tokens are already in the cost.
+      COLUMNS = ["score", "errors", *MAGNITUDES].freeze
+
+      # What errors take off the total: {ERROR_BASE} + {ERROR_STEP} per error.
+      ERROR_BASE = 0.15
+      ERROR_STEP = 0.05
+
+      attr_reader :weights
+
+      # @param weights [Hash{String => Numeric}]
+      # @param metrics [Array<String>] the scorer's metric names
+      # @raise [Error] on a weight that is not a number, an unknown column, or no weight at all
+      def initialize(weights, metrics:)
+        known = (metrics + COLUMNS).uniq
+        unknown = weights.keys - known
+        raise Error, "weighted_score: unknown columns #{unknown.join(", ")}; known: #{known.join(", ")}" if unknown.any?
+
+        invalid = weights.reject { |_name, weight| weight.is_a?(Numeric) }.keys
+        raise Error, "weighted_score: weights must be numbers: #{invalid.join(", ")}" if invalid.any?
+
+        raise Error, "weighted_score: errors must have a negative weight, since fewer is better" if weights.fetch("errors", -1).positive?
+
+        @weights = weights.reject { |_name, weight| weight.zero? }
+        raise Error, "weighted_score: every weight is zero" if @weights.empty?
+      end
+
+      # A column no row has does not count. A row missing a column the others
+      # have gets nil: it was not measured on everything they were.
+      #
+      # @param rows [Array<Hash{String => Numeric, nil}>] one per contender, compared with each other
+      # @return [Array<Float, nil>] in the order given
+      def call(rows)
+        present = weights.select { |name, _weight| rows.any? { it[name] } }
+        total = present.values.sum(&:abs)
+        rows.map do |row|
+          next if present.empty? || present.keys.any? { row[it].nil? }
+
+          score = present.sum { |name, weight| weight.abs * goodness(name, row[name], weight, rows) } / total
+          [score - penalty(row["errors"], present), 0.0].max
+        end
+      end
+
+      # @return [String] the formula, with this lineup's weights, for the glossary
+      def description
+        "Σ(|weight| × goodness) / Σ|weight|, from 0% to 100%, and higher is better. A negative weight means that column is better low. " \
+          "Errors come off the total: 20 points for one, then 5 more for each further error. " \
+          "#{MAGNITUDES.join(", ")} are on a log scale within the group, so compare wscore within one table only. " \
+          "Weights: #{weights.map { |name, weight| "#{name} #{weight}" }.join(", ")}."
+      end
+
+      private
+
+      def penalty(errors, present)
+        return 0.0 unless present.key?("errors") && errors.positive?
+
+        ERROR_BASE + (ERROR_STEP * errors)
+      end
+
+      def goodness(name, value, weight, rows)
+        return 1.0 if name == "errors"
+
+        position = MAGNITUDES.include?(name) ? position(value, rows.filter_map { it[name] }) : value.to_f
+        return 1.0 if position.nil?
+
+        weight.negative? ? 1.0 - position : position
+      end
+
+      # A zero has no logarithm, so it stands at half the lowest above zero.
+      #
+      # @return [Float, nil] from 0 at the lowest to 1 at the highest; nil when all are equal
+      def position(value, values)
+        highest = values.max
+        return nil if values.min == highest
+
+        lowest = values.select(&:positive?).min
+        lowest /= 2.0 if values.any?(&:zero?)
+        (Math.log([value, lowest].max) - Math.log(lowest)) / (Math.log(highest) - Math.log(lowest))
+      end
+    end
+  end
+end

data/lib/llm/evals.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "etc"
+require "json"
+require "dry/cli"
+require "dry/cli/help"
+require "dry/cli/ui"
+require "dry/cli/autocomplete/command"
+require "zeitwerk"
+
+module LLM
+  # A generic evaluation runner with a one-way boundary. The consumer suite
+  # registers an {Adapter}, which runs one {Case} against a {Model} and
+  # returns a {Reply} without scoring it, and a {Scorer}, which turns a reply
+  # into a {Score} without calling anything. Everything else, from discovery
+  # to the leaderboard, is this gem's, and none of it knows what is being
+  # evaluated.
+  module Evals
+    class Error < StandardError; end
+
+    # Raised by a command that needs an optional gem the bundle does not hold.
+    class MissingDependency < Error; end
+
+    class << self
+      # The suite's settings, filled in by {.configure} from `evals/eval_helper.rb`.
+      #
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      # Builds the suite's configuration. Called once from the eval helper.
+      #
+      # @example evals/eval_helper.rb
+      #   LLM::Evals.configure do |c|
+      #     c.suite "extract", root: __dir__, env_files: ["../.env"]
+      #     c.cases "datasets/*/cases/**/input.txt"
+      #     c.adapter ExtractAdapter.new
+      #     c.scorer ExtractScorer.new
+      #     c.before_run { MyGem.reset_configuration! }
+      #   end
+      #
+      # @yieldparam configuration [Configuration]
+      # @return [Configuration]
+      def configure
+        yield configuration
+        configuration
+      end
+
+      # Forgets the configuration, so a spec that loads a second helper starts clean.
+      #
+      # @return [void]
+      def reset!
+        @configuration = nil
+      end
+
+      # @return [Boolean] whether a helper has registered an adapter and a scorer
+      def configured? = configuration.complete?
+
+      # Requires an optional gem the moment a command needs it, and names it
+      # when it is missing, instead of letting a bare LoadError surface three
+      # frames deep in a provider.
+      #
+      # @param feature [String] what to require, e.g. "ruby_llm"
+      # @param gem_name [String] the gem to add to the Gemfile; the feature when they agree
+      # @param needed_for [String] what the user was doing
+      # @return [void]
+      # @raise [MissingDependency]
+      def optional!(feature, gem_name = feature, needed_for:)
+        require feature
+      rescue LoadError
+        raise MissingDependency, "#{needed_for} needs the #{gem_name} gem: add `gem \"#{gem_name}\"` to your Gemfile"
+      end
+    end
+  end
+end
+
+require_relative "evals/version"
+
+loader = Zeitwerk::Loader.for_gem_extension(LLM)
+loader.inflector.inflect("cli" => "CLI", "html" => "HTML", "json" => "JSON")
+loader.ignore(File.expand_path("evals/version.rb", __dir__))
+loader.setup