llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/verify.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Offline consistency checks over every case, so a typo in the suite is
-    # caught before it is billed as a model's mistake. The gem checks what
-    # every suite has: an input and a parsable expected answer. The adapter
-    # checks each dataset's shared files, and the scorer each case against
-    # what it will demand of it.
-    class Verify
-      # @param configuration [Configuration]
-      def initialize(configuration = LLM::Evals.configuration)
-        @configuration = configuration
-      end
-
-      # @return [Hash{String => Array<String>}] problems by "dataset/id"; empty when all is sound
-      def problems
-        discovery = @configuration.discovery
-        failures = {}
-        discovery.datasets.each do |dataset|
-          Array(adapter.verify(dataset)).each { (failures[dataset.to_s] ||= []) << it }
-          discovery.cases(dataset: dataset).each do |kase|
-            list = case_problems(kase)
-            failures[kase.key] = list unless list.empty?
-          end
-        end
-        failures
-      end
-
-      # @param kase [Case]
-      # @return [Array<String>]
-      def case_problems(kase)
-        problems = []
-        problems << "input is empty" if kase.input.empty?
-        problems << kase.metadata[:expected_error] if kase.metadata[:expected_error]
-        return problems unless kase.expected?
-
-        problems.concat(Array(scorer.verify(kase, context: adapter.context(kase))))
-      rescue StandardError => e
-        problems << "#{e.class}: #{e.message}"
-      end
-
-      private
-
-      def adapter = @configuration.adapter
-      def scorer = @configuration.scorer
-    end
-  end
-end

data/lib/llm/evals/warnings.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-require "logger"
-
-module LLM
-  module Evals
-    # Everything a run would otherwise print as a warning while its progress
-    # bars redraw: cases that failed, and what Braintrust, RubyLLM, OpenTelemetry
-    # (whose exporters report failures on $stdout) and Ruby itself log. Printed among the bars, each one breaks the redraw; collected
-    # here, they are saved to one file, and the run prints only how many.
-    #
-    # It quacks as an IO, so a Logger, and $stderr, can write to it. Threads
-    # share it.
-    class Warnings
-      def initialize
-        @lines = []
-        @lock = Mutex.new
-      end
-
-      # @param text [#to_s]
-      # @return [self]
-      def record(text)
-        text = text.to_s.chomp
-        @lock.synchronize { @lines << text } unless text.strip.empty?
-        self
-      end
-
-      # @return [Integer] the bytes "written", as IO#write does
-      def write(*texts)
-        texts.each { record(it) }
-        texts.sum { it.to_s.bytesize }
-      end
-
-      def puts(*texts) = texts.each { record(it) } && nil
-      def print(*texts) = write(*texts) && nil
-      def <<(text) = record(text)
-      def flush = self
-      def sync = true
-      def sync=(_value); end
-      def close = nil
-      def tty? = false
-
-      # @return [Integer]
-      def count = @lock.synchronize { @lines.size }
-
-      # @return [Array<String>]
-      def to_a = @lock.synchronize { @lines.dup }
-
-      # @return [Boolean] whether nothing was recorded
-      def none? = count.zero?
-
-      # @param path [String]
-      # @return [String] the path
-      def save(path)
-        File.write(path, "#{to_a.join("\n\n")}\n")
-        path
-      end
-
-      # Runs the block with $stderr, Ruby's own warnings, and the loggers of
-      # whichever optional gems are loaded writing here, and puts them back after.
-      #
-      # @return [Object] what the block returns
-      def capture
-        logger = Logger.new(self, level: Logger::WARN)
-        stderr = $stderr
-        kernel_warn = Kernel.instance_method(:warn)
-        warnings = self
-        restore = redirect_loggers(logger)
-        $stderr = self
-        Kernel.define_method(:warn) { |*messages, **_options| warnings.write(messages.join("\n")) }
-        Kernel.send(:private, :warn)
-        yield
-      ensure
-        Kernel.define_method(:warn, kernel_warn) if kernel_warn
-        Kernel.send(:private, :warn)
-        restore&.call
-        $stderr = stderr
-      end
-
-      private
-
-      # @return [Proc] puts the loggers back
-      def redirect_loggers(logger)
-        braintrust = defined?(Braintrust::Log) ? Braintrust::Log.logger : nil
-        ruby_llm = defined?(RubyLLM) ? RubyLLM.instance_variable_get(:@logger) : nil
-        open_telemetry = defined?(OpenTelemetry) ? OpenTelemetry.logger : nil
-        Braintrust::Log.logger = logger if defined?(Braintrust::Log)
-        RubyLLM.instance_variable_set(:@logger, logger) if defined?(RubyLLM)
-        OpenTelemetry.logger = logger if defined?(OpenTelemetry)
-        lambda do
-          Braintrust::Log.logger = braintrust if defined?(Braintrust::Log)
-          RubyLLM.instance_variable_set(:@logger, ruby_llm) if defined?(RubyLLM)
-          OpenTelemetry.logger = open_telemetry if defined?(OpenTelemetry)
-        end
-      end
-    end
-  end
-end

data/lib/llm/evals/weighted_score.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One number from the leaderboard's columns, weighted as the lineup file
-    # says, for a suite to state what matters most to it. A positive weight
-    # means higher is better; a negative one, lower is better:
-    #
-    #   weighted_score:
-    #     weights:
-    #       score: 0.4
-    #       hallucinated_rate: -0.2
-    #       cost: -0.2
-    #
-    #   weighted_score = Σ(|weight| × goodness) / Σ|weight|
-    #
-    # Goodness runs from 0 to 1. Metrics are used as they are, or as 1 - value
-    # under a negative weight. {MAGNITUDES}
-    # are placed on a log scale between the lowest and highest in the group, so
-    # that each doubling costs the same: on a linear scale, one model a hundred
-    # times the price would squash all the others together. Where the whole
-    # group is equal, each gets 1.
-    #
-    # Errors come off the total, when weighted: 20 points for one, 25 for two, 30
-    # for three, and so on, down to 0. A failed case has no scores, so no other
-    # column notices it, and a single failure must show. A clean contender gets
-    # the whole of the weight on errors.
-    #
-    # Magnitudes are relative, so a weighted score compares within one group
-    # only: another model in the lineup moves everyone's.
-    class WeightedScore
-      # Columns compared on a log scale.
-      MAGNITUDES = %w[time cost].freeze
-
-      # The gem's columns that can be weighted, beside the metrics. `temp` is a
-      # setting, not a result, and tokens are already in the cost.
-      COLUMNS = ["score", "errors", *MAGNITUDES].freeze
-
-      # What errors take off the total: {ERROR_BASE} + {ERROR_STEP} per error.
-      ERROR_BASE = 0.15
-      ERROR_STEP = 0.05
-
-      attr_reader :weights
-
-      # @param weights [Hash{String => Numeric}]
-      # @param metrics [Array<String>] the scorer's metric names
-      # @raise [Error] on a weight that is not a number, an unknown column, or no weight at all
-      def initialize(weights, metrics:)
-        known = (metrics + COLUMNS).uniq
-        unknown = weights.keys - known
-        raise Error, "weighted_score: unknown columns #{unknown.join(", ")}; known: #{known.join(", ")}" if unknown.any?
-
-        invalid = weights.reject { |_name, weight| weight.is_a?(Numeric) }.keys
-        raise Error, "weighted_score: weights must be numbers: #{invalid.join(", ")}" if invalid.any?
-
-        raise Error, "weighted_score: errors must have a negative weight, since fewer is better" if weights.fetch("errors", -1).positive?
-
-        @weights = weights.reject { |_name, weight| weight.zero? }
-        raise Error, "weighted_score: every weight is zero" if @weights.empty?
-      end
-
-      # A column no row has does not count. A row missing a column the others
-      # have gets nil: it was not measured on everything they were.
-      #
-      # @param rows [Array<Hash{String => Numeric, nil}>] one per contender, compared with each other
-      # @return [Array<Float, nil>] in the order given
-      def call(rows)
-        present = weights.select { |name, _weight| rows.any? { it[name] } }
-        total = present.values.sum(&:abs)
-        rows.map do |row|
-          next if present.empty? || present.keys.any? { row[it].nil? }
-
-          score = present.sum { |name, weight| weight.abs * goodness(name, row[name], weight, rows) } / total
-          [score - penalty(row["errors"], present), 0.0].max
-        end
-      end
-
-      # @return [String] the formula, with this lineup's weights, for the glossary
-      def description
-        "Σ(|weight| × goodness) / Σ|weight|, from 0% to 100%, and higher is better. A negative weight means that column is better low. " \
-          "Errors come off the total: 20 points for one, then 5 more for each further error. " \
-          "#{MAGNITUDES.join(", ")} are on a log scale within the group, so compare wscore within one table only. " \
-          "Weights: #{weights.map { |name, weight| "#{name} #{weight}" }.join(", ")}."
-      end
-
-      private
-
-      def penalty(errors, present)
-        return 0.0 unless present.key?("errors") && errors.positive?
-
-        ERROR_BASE + (ERROR_STEP * errors)
-      end
-
-      def goodness(name, value, weight, rows)
-        return 1.0 if name == "errors"
-
-        position = MAGNITUDES.include?(name) ? position(value, rows.filter_map { it[name] }) : value.to_f
-        return 1.0 if position.nil?
-
-        weight.negative? ? 1.0 - position : position
-      end
-
-      # A zero has no logarithm, so it stands at half the lowest above zero.
-      #
-      # @return [Float, nil] from 0 at the lowest to 1 at the highest; nil when all are equal
-      def position(value, values)
-        highest = values.max
-        return nil if values.min == highest
-
-        lowest = values.select(&:positive?).min
-        lowest /= 2.0 if values.any?(&:zero?)
-        (Math.log([value, lowest].max) - Math.log(lowest)) / (Math.log(highest) - Math.log(lowest))
-      end
-    end
-  end
-end