llm-evals 0.4.0 → 0.5.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-evals
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Konstantin Gredeskoul
@@ -163,6 +163,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: ruby-limiter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 description: |2-
 
       Run multi-case evaluation suites against datasets using any model available
@@ -186,64 +200,117 @@ files:
 - README.md
 - exe/evals
 - exe/llm-evals
+- lib/.DS_Store
+- lib/llm/.DS_Store
 - lib/llm/evals.rb
-- lib/llm/evals/adapter.rb
-- lib/llm/evals/case.rb
+- lib/llm/evals/actions.rb
+- lib/llm/evals/actions/adapter.rb
+- lib/llm/evals/actions/error_budget.rb
+- lib/llm/evals/actions/error_diagnosis.rb
+- lib/llm/evals/actions/error_streak.rb
+- lib/llm/evals/actions/experiment.rb
+- lib/llm/evals/actions/verify.rb
 - lib/llm/evals/cli.rb
 - lib/llm/evals/cli/cases.rb
 - lib/llm/evals/cli/command.rb
 - lib/llm/evals/cli/compare.rb
+- lib/llm/evals/cli/contender_selector.rb
+- lib/llm/evals/cli/error_pause.rb
+- lib/llm/evals/cli/experiment_pool.rb
 - lib/llm/evals/cli/formatting.rb
 - lib/llm/evals/cli/leaderboard.rb
 - lib/llm/evals/cli/model_colors.rb
 - lib/llm/evals/cli/model_command.rb
+- lib/llm/evals/cli/model_options.rb
+- lib/llm/evals/cli/option_groups.rb
 - lib/llm/evals/cli/progress_bars.rb
 - lib/llm/evals/cli/render.rb
+- lib/llm/evals/cli/report_dispatcher.rb
+- lib/llm/evals/cli/report_dispatcher/console.rb
+- lib/llm/evals/cli/report_dispatcher/html_output.rb
+- lib/llm/evals/cli/report_dispatcher/json_output.rb
+- lib/llm/evals/cli/report_dispatcher/output.rb
+- lib/llm/evals/cli/report_dispatcher/pipeline_output.rb
 - lib/llm/evals/cli/run.rb
+- lib/llm/evals/cli/run_planner.rb
+- lib/llm/evals/cli/run_record.rb
 - lib/llm/evals/cli/standings.rb
+- lib/llm/evals/cli/status_box.rb
+- lib/llm/evals/cli/table.rb
+- lib/llm/evals/cli/text.rb
+- lib/llm/evals/cli/tracker_resolver.rb
+- lib/llm/evals/cli/upgrade_config.rb
 - lib/llm/evals/cli/verify.rb
 - lib/llm/evals/cli/version.rb
-- lib/llm/evals/completion.rb
+- lib/llm/evals/cli/weights.rb
 - lib/llm/evals/config.rb
-- lib/llm/evals/configuration.rb
-- lib/llm/evals/contender.rb
-- lib/llm/evals/discovery.rb
-- lib/llm/evals/experiment.rb
-- lib/llm/evals/helper.rb
+- lib/llm/evals/config/contender_builder.rb
+- lib/llm/evals/config/error_limits.rb
+- lib/llm/evals/config/listing.rb
+- lib/llm/evals/config/results.rb
+- lib/llm/evals/config/score_weights.rb
+- lib/llm/evals/config/upgrade.rb
+- lib/llm/evals/datasets.rb
+- lib/llm/evals/datasets/case.rb
+- lib/llm/evals/datasets/discovery.rb
 - lib/llm/evals/launcher.rb
-- lib/llm/evals/metric.rb
-- lib/llm/evals/model.rb
+- lib/llm/evals/models.rb
+- lib/llm/evals/models/contender.rb
+- lib/llm/evals/models/model.rb
+- lib/llm/evals/models/throttle.rb
+- lib/llm/evals/providers.rb
 - lib/llm/evals/providers/base.rb
 - lib/llm/evals/providers/catalog.rb
 - lib/llm/evals/providers/open_router.rb
 - lib/llm/evals/providers/registry.rb
 - lib/llm/evals/providers/type_safe.rb
+- lib/llm/evals/providers/type_safe/client.rb
+- lib/llm/evals/render.rb
+- lib/llm/evals/render/reports/call_stats_table.rb
 - lib/llm/evals/render/reports/html.rb
+- lib/llm/evals/render/reports/html/dataset_section.rb
+- lib/llm/evals/render/reports/html/format.rb
+- lib/llm/evals/render/reports/html/global_standings.rb
+- lib/llm/evals/render/reports/html/ranking.rb
+- lib/llm/evals/render/reports/html/view_model.rb
 - lib/llm/evals/render/reports/json.rb
+- lib/llm/evals/render/reports/json/decoder.rb
+- lib/llm/evals/render/reports/json/encoder.rb
 - lib/llm/evals/render/reports/pipeline.rb
 - lib/llm/evals/render/reports/terminal.rb
-- lib/llm/evals/reply.rb
-- lib/llm/evals/run_result.rb
+- lib/llm/evals/render/templates/html_report.css
+- lib/llm/evals/render/templates/html_report.html.erb
+- lib/llm/evals/render/templates/html_report.js
+- lib/llm/evals/results.rb
+- lib/llm/evals/results/provider_response.rb
+- lib/llm/evals/results/reply.rb
+- lib/llm/evals/results/run_outcome.rb
+- lib/llm/evals/results/run_result.rb
+- lib/llm/evals/results/score.rb
+- lib/llm/evals/results/warnings.rb
 - lib/llm/evals/runtime.rb
-- lib/llm/evals/schema_validator.rb
-- lib/llm/evals/score.rb
-- lib/llm/evals/scorer.rb
-- lib/llm/evals/templates/html_report.css
-- lib/llm/evals/templates/html_report.html.erb
-- lib/llm/evals/templates/html_report.js
+- lib/llm/evals/runtime/configuration.rb
+- lib/llm/evals/runtime/environment.rb
+- lib/llm/evals/runtime/helper.rb
+- lib/llm/evals/scoring.rb
+- lib/llm/evals/scoring/metric.rb
+- lib/llm/evals/scoring/schema_validator.rb
+- lib/llm/evals/scoring/scorer.rb
+- lib/llm/evals/scoring/weighted_score.rb
+- lib/llm/evals/stats.rb
+- lib/llm/evals/stats/call_stats.rb
+- lib/llm/evals/stats/usage.rb
+- lib/llm/evals/testing.rb
 - lib/llm/evals/testing/echo_adapter.rb
 - lib/llm/evals/testing/fake_model.rb
 - lib/llm/evals/testing/fake_provider.rb
-- lib/llm/evals/tracker.rb
+- lib/llm/evals/trackers.rb
+- lib/llm/evals/trackers/base.rb
 - lib/llm/evals/trackers/braintrust.rb
 - lib/llm/evals/trackers/local.rb
 - lib/llm/evals/trackers/logfire.rb
 - lib/llm/evals/trackers/registry.rb
-- lib/llm/evals/usage.rb
-- lib/llm/evals/verify.rb
 - lib/llm/evals/version.rb
-- lib/llm/evals/warnings.rb
-- lib/llm/evals/weighted_score.rb
 homepage: https://github.com/kigster/llm-evals
 licenses:
 - MIT

data/lib/llm/evals/adapter.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What a suite implements to run one case: everything the model sees, and
-    # everything done to its answer before it is scored. An adapter never
-    # scores; that is the {Scorer}'s, and keeping them apart is what lets one
-    # suite compare prompts and another compare models with the same runner.
-    #
-    # @abstract Subclass and override {#call}.
-    class Adapter
-      # Called once per dataset before its cases run; load what the dataset
-      # shares (a schema, a flow definition) here, outside the threads.
-      #
-      # @param dataset [String]
-      # @return [void]
-      def prepare(_dataset) = nil
-
-      # Runs one case against one model.
-      #
-      # @param kase [Case]
-      # @param model [Model]
-      # @return [Reply] with `error:` set when the model failed to answer; raise for a transport failure
-      def call(kase, model:)
-        raise NotImplementedError, "#{self.class}#call(kase, model:) must return an LLM::Evals::Reply"
-      end
-
-      # What the scorer needs to know about a case beyond the case itself: a
-      # schema, field types, the prompt. Passed to {Scorer#call} as `context:`.
-      #
-      # @param kase [Case]
-      # @return [Hash]
-      def context(_kase) = {}
-
-      # Offline checks of a dataset's shared files. `llm-evals verify` prints them.
-      #
-      # @param dataset [String]
-      # @return [Array<String>] problems; empty when the dataset is sound
-      def verify(_dataset) = []
-
-      # Recorded with each Braintrust experiment, e.g. which prompt version ran.
-      #
-      # @return [Hash]
-      def metadata = {}
-    end
-  end
-end

data/lib/llm/evals/case.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One eval case, as {Discovery} found it on disk: the text the model is
-    # given, the answer a perfect reply would produce, and where it came from.
-    #
-    # @!attribute [r] id
-    #   @return [String] unique within its dataset, e.g. "clear/003"
-    # @!attribute [r] dataset
-    #   @return [String] the folder under datasets/, e.g. "colors"
-    # @!attribute [r] variant
-    #   @return [String] the folder under cases/, e.g. "clear" or "vague"
-    # @!attribute [r] input
-    #   @return [String] input.txt, stripped
-    # @!attribute [r] expected
-    #   @return [Object, nil] expected.json parsed; nil when missing or unparsable
-    # @!attribute [r] path
-    #   @return [String] the case's directory
-    # @!attribute [r] metadata
-    #   @return [Hash{Symbol => Object}] number, difficulty, and any `expected_error`
-    Case = Data.define(:id, :dataset, :variant, :input, :expected, :path, :metadata) do
-      # @return [Integer, nil] the NNN/ level as a number, when the case has one
-      def number = metadata[:number]
-
-      # @return [Integer] the number, else 0; what the tables sort and show difficulty by
-      def difficulty = metadata.fetch(:difficulty, number || 0)
-
-      # @return [String] "dataset/id", unique across the suite
-      def key = "#{dataset}/#{id}"
-
-      # @return [Boolean] whether expected.json was there and parsed
-      def expected? = !expected.nil?
-    end
-  end
-end

data/lib/llm/evals/completion.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What a {Model} returns from one call: the text, what it cost, and the
-    # provider's raw response for an adapter that needs more than the text.
-    Completion = Data.define(:text, :usage, :raw) do
-      def initialize(text:, usage: Usage.zero, raw: nil) = super(text: text.to_s, usage: Usage.from(usage), raw: raw)
-    end
-  end
-end

data/lib/llm/evals/configuration.rb

@@ -1,173 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Everything a suite tells the gem, through {LLM::Evals.configure}.
-    # Paths are relative to `root`, the directory the eval helper lives in.
-    class Configuration
-      DEFAULT_GLOB = "datasets/*/cases/**/input.txt"
-      DEFAULT_NAME = "llm-evals"
-      DEFAULT_CONFIG = ".llm-evals.config.yml"
-
-      attr_writer :adapter, :scorer
-      attr_reader :name, :root, :env_files, :cases_glob, :hooks, :providers, :trackers, :description
-
-      def initialize
-        @name = DEFAULT_NAME
-        @root = Dir.pwd
-        @env_files = []
-        @cases_glob = DEFAULT_GLOB
-        @config_path = DEFAULT_CONFIG
-        @hooks = []
-        @providers = Providers::Registry.default
-        @trackers = Trackers::Registry.default
-        @description = "Run a suite of cases against one or more models and score the replies."
-        @project = nil
-      end
-
-      # Names the suite. The name titles the help and tags the experiments.
-      #
-      # @param name [String]
-      # @param root [String] where .llm-evals.config.yml and datasets/ live; the helper's directory
-      # @param env_files [Array<String>] dotenv files to load, relative to root; the first that exists wins nothing,
-      #   all that exist load, and an exported variable beats them all
-      # @param description [String, nil] one line for the help's header
-      # @return [void]
-      def suite(name, root:, env_files: [], description: nil)
-        @name = name
-        @root = File.expand_path(root)
-        @env_files = env_files.map { File.expand_path(it, @root) }
-        @description = description if description
-      end
-
-      # Where the cases are: a glob relative to root ending in the input file.
-      # Each match is a case; the expected answer is `expected.json` beside it.
-      #
-      # @param glob [String]
-      # @return [void]
-      def cases(glob)
-        @cases_glob = glob
-      end
-
-      # @param object [Adapter]
-      # @return [void]
-
-      # @param object [Adapter, nil] setting; reading when nil
-      # @return [Adapter]
-      def adapter(object = nil)
-        @adapter = object unless object.nil?
-        @adapter
-      end
-
-      # @param object [Scorer, nil] setting; reading when nil
-      # @return [Scorer]
-      def scorer(object = nil)
-        @scorer = object unless object.nil?
-        @scorer
-      end
-
-      # @param object [Scorer]
-      # @return [void]
-
-      # Runs once, after the env files are loaded and before any model is
-      # called. The place to reset a library that read ENV at require time.
-      #
-      # @yield
-      # @return [void]
-      def before_run(&block)
-        @hooks << block
-      end
-
-      # Adds a provider, ahead of the defaults, so a suite's own transport
-      # wins the ids it claims.
-      #
-      # @param provider [Providers::Base]
-      # @return [void]
-      def provider(provider)
-        @providers.register(provider)
-      end
-
-      # Adds a tracker class, so a run can choose it by name with `--tracker`
-      # or `tracker:`, or have its env key select it.
-      #
-      # @param klass [Class<Tracker>]
-      # @return [void]
-      def tracker(klass)
-        @trackers.register(klass)
-      end
-
-      # @param path [String, nil] setting, relative to root; reading when nil
-      # @return [String] the absolute path of the lineup file, `.llm-evals.config.yml` at the root by default
-      def config(path = nil)
-        @config_path = path unless path.nil?
-        File.expand_path(@config_path, root)
-      end
-
-      # @param name [String, nil] setting; reading when nil
-      # @return [String, nil] the Braintrust project, when .llm-evals.config.yml does not name one
-      def project(name = nil)
-        @project = name unless name.nil?
-        @project
-      end
-
-      # @return [Discovery] the cases, as the glob finds them
-      def discovery = Discovery.new(root: root, glob: cases_glob)
-
-      # @return [Config] the lineup file
-      def lineup = Config.new(config, direct_providers: providers.direct_names)
-
-      # @return [Runtime]
-      def runtime = Runtime.new(env_files: env_files, hooks: hooks, app_name: name)
-
-      # @return [Boolean] whether an adapter and a scorer are registered
-      def complete? = !adapter.nil? && !scorer.nil?
-
-      # Fails a command that needs the suite when no helper was loaded, with
-      # the fix in the message, instead of a NoMethodError on nil.
-      #
-      # @return [self]
-      # @raise [Error]
-      def require_suite!
-        resolve_from_config!
-        return self if complete?
-
-        raise Error,
-          "no suite is configured: register an adapter and a scorer in evals/eval_helper.rb, or set " \
-          "adapter: and scorer: (with require:) in #{config}; --helper PATH or LLM_EVALS_HELPER names another helper"
-      end
-
-      # Fills whatever the helper left unset from the config file's `require:`,
-      # `adapter:` and `scorer:` keys, so a suite can be config-only: its Ruby
-      # lives in its own gem and `evals/` holds the file and the datasets. The
-      # helper, when there is one, was loaded first and wins.
-      #
-      # @return [void]
-      # @raise [Error] naming the key and the value when a require fails or a constant is unknown
-      def resolve_from_config!
-        return if complete? || !File.exist?(config)
-
-        file = lineup
-        file.requires.each do |feature|
-          require feature
-        rescue LoadError => e
-          raise Error, "#{config}: require: #{feature} could not be loaded (#{e.message}); is its gem in the bundle?"
-        end
-        @adapter ||= instantiate("adapter", file.adapter_name)
-        @scorer ||= instantiate("scorer", file.scorer_name) # rubocop:disable Naming/MemoizedInstanceVariableName
-      end
-
-      private
-
-      # @return [Object, nil] a new instance of the named constant; nil when no name is set
-      def instantiate(key, name)
-        return nil if name.nil? || name.empty?
-
-        Object.const_get(name).new
-      rescue NameError, ArgumentError => e
-        raise Error,
-          "#{config}: #{key}: #{name} could not be instantiated (#{e.class}: #{e.message}); " \
-          "name a constant the require: key loads, taking no arguments"
-      end
-    end
-  end
-end

data/lib/llm/evals/contender.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One model at one setting: what a single experiment runs. Built by
-    # {Config} from the lineup file, or from `-m` on the command line.
-    #
-    # `provider` is nil for a `vendors:` entry, which OpenRouter serves, and
-    # the provider's name for a `direct:` entry. It travels with the
-    # contender rather than being read back out of the id, so `typesafe/jev-router`
-    # (Jev on OpenRouter) and a direct `typesafe/jev-latest` can never be confused.
-    Contender = Data.define(:model, :effort, :temperature, :max_tokens, :provider) do
-      def initialize(model:, effort: "default", temperature: "default", max_tokens: nil, provider: nil) = super
-
-      # @return [Boolean] whether the entry came from `direct:`
-      def direct? = !provider.nil?
-
-      # @return [String] unique across the lineup: "typesafe:typesafe/jev-latest" for a direct entry
-      def key = direct? ? "#{provider}:#{model}" : model
-
-      # The settings as a {Model} takes them: "default" means nil, send nothing.
-      #
-      # @return [Hash{Symbol => Object}]
-      def settings
-        { effort:      effort.to_s == "default" || effort.nil? ? nil : effort.to_s,
-          temperature: temperature.to_s == "default" || temperature.nil? ? nil : Float(temperature),
-          max_tokens:  max_tokens.nil? ? nil : Integer(max_tokens) }
-      end
-    end
-  end
-end

data/lib/llm/evals/discovery.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Finds the cases under a root with a glob and decodes each path into a
-    # {Case}. The default layout is
-    #
-    #   datasets/<dataset>/cases/<variant>/[NNN/]input.txt
-    #   datasets/<dataset>/cases/<variant>/[NNN/]expected.json
-    #
-    # The folder after `datasets/` is the dataset, the folder holding the
-    # input is the variant, unless it is a number, in which case it is the
-    # case number and its parent is the variant.
-    class Discovery
-      EXPECTED = "expected.json"
-      METADATA = "metadata.json"
-
-      attr_reader :root, :glob
-
-      # @param root [String] an absolute directory
-      # @param glob [String] relative to root
-      def initialize(root:, glob: Configuration::DEFAULT_GLOB)
-        @root = root
-        @glob = glob
-      end
-
-      # @return [Array<String>] every dataset that has at least one case, sorted
-      def datasets = all.map(&:dataset).uniq.sort
-
-      # @param dataset [String, nil] one dataset; nil for all of them
-      # @param only [Array<String>] id prefixes to keep ("clear", "clear/00"); empty keeps all
-      # @return [Array<Case>] sorted by dataset, variant and number
-      # @raise [Error] when the dataset does not exist
-      def cases(dataset: nil, only: [])
-        found = all
-        if dataset
-          raise Error, "no dataset #{dataset.inspect}; have #{datasets.join(", ")}" unless datasets.include?(dataset)
-
-          found = found.select { it.dataset == dataset }
-        end
-        only = only.flat_map { it.split(",") }.map(&:strip).reject(&:empty?)
-        found = found.select { |kase| only.any? { kase.id.start_with?(it) || kase.key.start_with?(it) } } unless only.empty?
-        found
-      end
-
-      # @return [Array<Case>]
-      def all
-        @all ||= Dir[File.join(root, glob)].map { decode(it) }
-      end
-
-      # Builds the case at one input file.
-      #
-      # @param input_path [String]
-      # @return [Case]
-      def decode(input_path)
-        dir = File.dirname(input_path)
-        parts = Pathname(dir).relative_path_from(Pathname(root)).each_filename.to_a
-        number = parts.last.match?(/\A\d+\z/) ? parts.last : nil
-        variant = number ? parts[-2] : parts.last
-        expected, problem = read_expected(File.join(dir, EXPECTED))
-        Case.new(
-          id:       [variant, number].compact.join("/"),
-          dataset:  dataset_of(parts),
-          variant:  variant,
-          input:    File.read(input_path).strip,
-          expected: expected,
-          path:     dir,
-          metadata: metadata(dir, number, problem)
-        )
-      end
-
-      private
-
-      # The folder after `datasets/`, else the first folder, so a glob that
-      # does not say `datasets` still groups by its top level.
-      def dataset_of(parts)
-        index = parts.index("datasets")
-        index && parts[index + 1] ? parts[index + 1] : parts.first
-      end
-
-      def read_expected(path)
-        return [nil, "#{EXPECTED} is missing"] unless File.exist?(path)
-
-        [JSON.parse(File.read(path)), nil]
-      rescue JSON::ParserError => e
-        [nil, "#{EXPECTED} is not valid JSON: #{e.message.lines.first&.strip}"]
-      end
-
-      def metadata(dir, number, problem)
-        extra = File.exist?(File.join(dir, METADATA)) ? JSON.parse(File.read(File.join(dir, METADATA))).transform_keys(&:to_sym) : {}
-        { number: number&.to_i, difficulty: number.to_i, expected_error: problem }.compact.merge(extra)
-      end
-    end
-  end
-end