agent-evals 0.1.1

data/lib/agent/evals/usage.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What one call cost, as the provider counted it. Summed over a run.
+    Usage = Data.define(:input_tokens, :output_tokens, :cost_usd) do
+      # @return [Usage] zero of everything
+      def self.zero = new(input_tokens: 0, output_tokens: 0, cost_usd: 0.0)
+
+      # Reads a Hash with Symbol or String keys, missing keys as zero.
+      #
+      # @param hash [Hash, Usage, nil]
+      # @return [Usage]
+      def self.from(hash)
+        return hash if hash.is_a?(Usage)
+        return zero if hash.nil?
+
+        h = hash.transform_keys(&:to_sym)
+        new(input_tokens: h.fetch(:input_tokens, 0).to_i,
+          output_tokens: h.fetch(:output_tokens, 0).to_i,
+          cost_usd: h.fetch(:cost_usd, 0.0).to_f)
+      end
+
+      # @param other [Usage]
+      # @return [Usage]
+      def +(other)
+        with(input_tokens: input_tokens + other.input_tokens,
+          output_tokens: output_tokens + other.output_tokens,
+          cost_usd: cost_usd + other.cost_usd)
+      end
+    end
+  end
+end

data/lib/agent/evals/verify.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Offline consistency checks over every case, so a typo in the suite is
+    # caught before it is billed as a model's mistake. The gem checks what
+    # every suite has: an input and a parsable expected answer. The adapter
+    # checks each dataset's shared files, and the scorer each case against
+    # what it will demand of it.
+    class Verify
+      # @param configuration [Configuration]
+      def initialize(configuration = Agent::Evals.configuration)
+        @configuration = configuration
+      end
+
+      # @return [Hash{String => Array<String>}] problems by "dataset/id"; empty when all is sound
+      def problems
+        discovery = @configuration.discovery
+        failures = {}
+        discovery.datasets.each do |dataset|
+          Array(adapter.verify(dataset)).each { (failures[dataset.to_s] ||= []) << it }
+          discovery.cases(dataset: dataset).each do |kase|
+            list = case_problems(kase)
+            failures[kase.key] = list unless list.empty?
+          end
+        end
+        failures
+      end
+
+      # @param kase [Case]
+      # @return [Array<String>]
+      def case_problems(kase)
+        problems = []
+        problems << "input is empty" if kase.input.empty?
+        problems << kase.metadata[:expected_error] if kase.metadata[:expected_error]
+        return problems unless kase.expected?
+
+        problems.concat(Array(scorer.verify(kase, context: adapter.context(kase))))
+      rescue StandardError => e
+        problems << "#{e.class}: #{e.message}"
+      end
+
+      private
+
+      def adapter = @configuration.adapter
+      def scorer = @configuration.scorer
+    end
+  end
+end

data/lib/agent/evals/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    VERSION = "0.1.1"
+  end
+end

data/lib/agent/evals/warnings.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module Agent
+  module Evals
+    # Everything a run would otherwise print as a warning while its progress
+    # bars redraw: cases that failed, and what Braintrust, RubyLLM and Ruby
+    # itself log. Printed among the bars, each one breaks the redraw; collected
+    # here, they are saved to one file, and the run prints only how many.
+    #
+    # It quacks as an IO, so a Logger, and $stderr, can write to it. Threads
+    # share it.
+    class Warnings
+      def initialize
+        @lines = []
+        @lock = Mutex.new
+      end
+
+      # @param text [#to_s]
+      # @return [self]
+      def record(text)
+        text = text.to_s.chomp
+        @lock.synchronize { @lines << text } unless text.strip.empty?
+        self
+      end
+
+      # @return [Integer] the bytes "written", as IO#write does
+      def write(*texts)
+        texts.each { record(it) }
+        texts.sum { it.to_s.bytesize }
+      end
+
+      def puts(*texts) = texts.each { record(it) } && nil
+      def print(*texts) = write(*texts) && nil
+      def <<(text) = record(text)
+      def flush = self
+      def sync = true
+      def sync=(_value); end
+      def close = nil
+      def tty? = false
+
+      # @return [Integer]
+      def count = @lock.synchronize { @lines.size }
+
+      # @return [Array<String>]
+      def to_a = @lock.synchronize { @lines.dup }
+
+      # @return [Boolean] whether nothing was recorded
+      def none? = count.zero?
+
+      # @param path [String]
+      # @return [String] the path
+      def save(path)
+        File.write(path, "#{to_a.join("\n\n")}\n")
+        path
+      end
+
+      # Runs the block with $stderr, Ruby's own warnings, and the loggers of
+      # whichever optional gems are loaded writing here, and puts them back after.
+      #
+      # @return [Object] what the block returns
+      def capture
+        logger = Logger.new(self, level: Logger::WARN)
+        stderr = $stderr
+        restore = redirect_loggers(logger)
+        $stderr = self
+        yield
+      ensure
+        restore&.call
+        $stderr = stderr
+      end
+
+      private
+
+      # @return [Proc] puts the loggers back
+      def redirect_loggers(logger)
+        braintrust = defined?(Braintrust::Log) ? Braintrust::Log.logger : nil
+        ruby_llm = defined?(RubyLLM) ? RubyLLM.instance_variable_get(:@logger) : nil
+        Braintrust::Log.logger = logger if defined?(Braintrust::Log)
+        RubyLLM.instance_variable_set(:@logger, logger) if defined?(RubyLLM)
+        lambda do
+          Braintrust::Log.logger = braintrust if defined?(Braintrust::Log)
+          RubyLLM.instance_variable_set(:@logger, ruby_llm) if defined?(RubyLLM)
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "json"
+require "dry/cli"
+require "dry/cli/help"
+require "dry/cli/ui"
+require "dry/cli/autocomplete/command"
+require "zeitwerk"
+
+module Agent
+  # A generic evaluation runner with a one-way boundary. The consumer suite
+  # registers an {Adapter}, which runs one {Case} against a {Model} and
+  # returns a {Reply} without scoring it, and a {Scorer}, which turns a reply
+  # into a {Score} without calling anything. Everything else, from discovery
+  # to the leaderboard, is this gem's, and none of it knows what is being
+  # evaluated.
+  module Evals
+    class Error < StandardError; end
+
+    # Raised by a command that needs an optional gem the bundle does not hold.
+    class MissingDependency < Error; end
+
+    class << self
+      # The suite's settings, filled in by {.configure} from `evals/eval_helper.rb`.
+      #
+      # @return [Configuration]
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      # Builds the suite's configuration. Called once from the eval helper.
+      #
+      # @example evals/eval_helper.rb
+      #   Agent::Evals.configure do |c|
+      #     c.suite "extract", root: __dir__, env_files: ["../.env"]
+      #     c.cases "datasets/*/cases/**/input.txt"
+      #     c.adapter ExtractAdapter.new
+      #     c.scorer ExtractScorer.new
+      #     c.before_run { MyGem.reset_configuration! }
+      #   end
+      #
+      # @yieldparam configuration [Configuration]
+      # @return [Configuration]
+      def configure
+        yield configuration
+        configuration
+      end
+
+      # Forgets the configuration, so a spec that loads a second helper starts clean.
+      #
+      # @return [void]
+      def reset!
+        @configuration = nil
+      end
+
+      # @return [Boolean] whether a helper has registered an adapter and a scorer
+      def configured? = configuration.complete?
+
+      # Requires an optional gem the moment a command needs it, and names it
+      # when it is missing, instead of letting a bare LoadError surface three
+      # frames deep in a provider.
+      #
+      # @param feature [String] what to require, e.g. "ruby_llm"
+      # @param gem_name [String] the gem to add to the Gemfile; the feature when they agree
+      # @param needed_for [String] what the user was doing
+      # @return [void]
+      # @raise [MissingDependency]
+      def optional!(feature, gem_name = feature, needed_for:)
+        require feature
+      rescue LoadError
+        raise MissingDependency, "#{needed_for} needs the #{gem_name} gem: add `gem \"#{gem_name}\"` to your Gemfile"
+      end
+    end
+  end
+end
+
+require_relative "evals/version"
+
+loader = Zeitwerk::Loader.for_gem_extension(Agent)
+loader.inflector.inflect("cli" => "CLI")
+loader.ignore(File.expand_path("evals/version.rb", __dir__))
+loader.setup

metadata

@@ -0,0 +1,257 @@
+--- !ruby/object:Gem::Specification
+name: agent-evals
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+  - Konstantin Gredeskoul
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+  - !ruby/object:Gem::Dependency
+    name: dotenv
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "3.2"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "3.2"
+  - !ruby/object:Gem::Dependency
+    name: dry-cli
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "1.4"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "1.4"
+  - !ruby/object:Gem::Dependency
+    name: dry-cli-autocomplete
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.6"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.6"
+  - !ruby/object:Gem::Dependency
+    name: dry-cli-help
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.6"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.6"
+  - !ruby/object:Gem::Dependency
+    name: dry-cli-ui
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.7"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.7"
+  - !ruby/object:Gem::Dependency
+    name: zeitwerk
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "2.8"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "2.8"
+  - !ruby/object:Gem::Dependency
+    name: pastel
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.8"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.8"
+  - !ruby/object:Gem::Dependency
+    name: tty-box
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.7"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.7"
+  - !ruby/object:Gem::Dependency
+    name: tty-table
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.12"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "0.12"
+  - !ruby/object:Gem::Dependency
+    name: logger
+    requirement: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "1.6"
+    type: :runtime
+    prerelease: false
+    version_requirements: !ruby/object:Gem::Requirement
+      requirements:
+        -
+          - ~>
+          - !ruby/object:Gem::Version
+            version: "1.6"
+description: "Runs a suite of cases against one or more models through an adapter the consumer writes, scores each reply with a scorer the consumer writes, and prints leaderboards, per-case breakdowns and Braintrust experiments. The gem knows nothing about what is being evaluated: it owns discovery, the model lineup, providers, parallel execution, reporting and the command line."
+email:
+  - kigster@gmail.com
+executables:
+  - agent-evals
+extensions: []
+extra_rdoc_files: []
+files:
+  - CHANGELOG.md
+  - LICENSE.txt
+  - README.md
+  - exe/agent-evals
+  - lib/agent/evals.rb
+  - lib/agent/evals/adapter.rb
+  - lib/agent/evals/case.rb
+  - lib/agent/evals/cli.rb
+  - lib/agent/evals/cli/cases.rb
+  - lib/agent/evals/cli/command.rb
+  - lib/agent/evals/cli/compare.rb
+  - lib/agent/evals/cli/model_command.rb
+  - lib/agent/evals/cli/progress_bars.rb
+  - lib/agent/evals/cli/run.rb
+  - lib/agent/evals/cli/verify.rb
+  - lib/agent/evals/cli/version.rb
+  - lib/agent/evals/completion.rb
+  - lib/agent/evals/config.rb
+  - lib/agent/evals/configuration.rb
+  - lib/agent/evals/contender.rb
+  - lib/agent/evals/discovery.rb
+  - lib/agent/evals/experiment.rb
+  - lib/agent/evals/helper.rb
+  - lib/agent/evals/launcher.rb
+  - lib/agent/evals/metric.rb
+  - lib/agent/evals/model.rb
+  - lib/agent/evals/providers/base.rb
+  - lib/agent/evals/providers/catalog.rb
+  - lib/agent/evals/providers/open_router.rb
+  - lib/agent/evals/providers/registry.rb
+  - lib/agent/evals/providers/type_safe.rb
+  - lib/agent/evals/reply.rb
+  - lib/agent/evals/run_result.rb
+  - lib/agent/evals/runtime.rb
+  - lib/agent/evals/schema_validator.rb
+  - lib/agent/evals/score.rb
+  - lib/agent/evals/scorer.rb
+  - lib/agent/evals/testing/echo_adapter.rb
+  - lib/agent/evals/testing/fake_model.rb
+  - lib/agent/evals/testing/fake_provider.rb
+  - lib/agent/evals/tracker.rb
+  - lib/agent/evals/trackers/braintrust.rb
+  - lib/agent/evals/trackers/local.rb
+  - lib/agent/evals/trackers/logfire.rb
+  - lib/agent/evals/trackers/registry.rb
+  - lib/agent/evals/usage.rb
+  - lib/agent/evals/verify.rb
+  - lib/agent/evals/version.rb
+  - lib/agent/evals/warnings.rb
+homepage: "https://github.com/kigster/agent-evals"
+licenses:
+  - MIT
+metadata:
+  allowed_push_host: "https://rubygems.org"
+  homepage_uri: "https://github.com/kigster/agent-evals"
+  source_code_uri: "https://github.com/kigster/agent-evals"
+  changelog_uri: "https://github.com/kigster/agent-evals/blob/main/CHANGELOG.md"
+  rubygems_mfa_required: "true"
+rdoc_options: []
+require_paths:
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+    -
+      - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+    -
+      - ">="
+      - !ruby/object:Gem::Version
+        version: "0"
+requirements: []
+rubygems_version: 4.1.0.beta2
+specification_version: 4
+summary: "A generic evaluation runner for LLM-backed code: adapters run, scorers score, the CLI reports"
+test_files: []