llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/experiment.rb

@@ -1,158 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One model at one setting over every case of one dataset, run on a
-    # thread pool the gem owns, plus the per-case detail the terminal report
-    # needs. Trackers only watch: {Trackers::Local} always, and at most one
-    # remote tracker, each told when the experiment starts, when a case
-    # finishes, and what the summary was.
-    class Experiment
-      # What one finished experiment reports back. `means` holds one entry
-      # per metric plus "score", the mean of each case's overall value;
-      # `usage` is summed over the cases; `permalink` is the remote tracker's
-      # page, when it has one.
-      Summary = Data.define(:model, :dataset, :name, :means, :results, :errors, :duration, :permalink, :usage) do
-        # @return [Float, nil] the mean overall score
-        def score = means["score"]
-
-        # @param metric [Metric, String]
-        # @return [Float, nil]
-        def mean(metric) = means[metric.respond_to?(:name) ? metric.name : metric.to_s]
-
-        # The same experiment over one variant's cases. Its duration is the sum
-        # of those cases' own, since they overlapped with the others.
-        #
-        # @param variant [String]
-        # @param metrics [Array<Metric>] the scorer's
-        # @return [Summary]
-        def for_variant(variant, metrics)
-          chosen = results.select { it.case.variant == variant }
-          with(means:    Experiment.means(metrics, chosen),
-            results:  chosen,
-            errors:   chosen.select(&:failed?).map { "#{it.case.key}: #{it.error}" },
-            duration: chosen.sum(&:duration),
-            usage:    Experiment.usage(chosen))
-        end
-      end
-
-      # @param metrics [Array<Metric>]
-      # @param results [Array<RunResult>]
-      # @return [Hash{String => Float, nil}] one mean per metric, plus "score"
-      def self.means(metrics, results)
-        scored = results.filter_map(&:score)
-        means = metrics.to_h { |metric| [metric.name, metric.combine(scored.map { it.metrics[metric.name] })] }
-        means.merge("score" => scored.empty? ? nil : scored.sum(&:value).fdiv(scored.size))
-      end
-
-      # @param results [Array<RunResult>]
-      # @return [Usage] summed over the results
-      def self.usage(results) = results.filter_map(&:reply).map(&:usage).reduce(Usage.zero, :+)
-
-      attr_reader :model, :adapter, :scorer, :cases, :dataset, :parallelism, :trackers
-
-      # @param model [Model]
-      # @param cases [Array<Case>] all of one dataset
-      # @param adapter [Adapter]
-      # @param scorer [Scorer]
-      # @param parallelism [Integer] cases in flight at once
-      # @param trackers [Array<Tracker>] told about the run, in this order
-      def initialize(model:, cases:, adapter:, scorer:, parallelism: 4, trackers: [])
-        raise ArgumentError, "an experiment needs at least one case" if cases.empty?
-
-        @model = model
-        @cases = cases.sort_by { [it.variant, it.number || 0, it.id] }
-        @dataset = cases.first.dataset
-        @adapter = adapter
-        @scorer = scorer
-        @parallelism = [parallelism.to_i, 1].max
-        @trackers = trackers.select(&:enabled?)
-        @results = {}
-        @lock = Mutex.new
-      end
-
-      # @return [String] e.g. "anthropic/claude-haiku-5.5 effort=low t=0.0 [clear]"
-      def name = "#{model} [#{dataset}]"
-
-      # @param on_case [#call, nil] called once per finished case
-      # @return [Summary]
-      def run(on_case: nil)
-        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        adapter.prepare(dataset)
-        trackers.each { it.start(self) }
-        errors = run_locally(on_case)
-        summary = summarize(errors, nil, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
-        urls = trackers.map { it.finish(self, summary)&.fetch(:url, nil) }
-        summary.with(permalink: urls.compact.first)
-      end
-
-      # Runs one case: the adapter, then the scorer. A reply the model failed
-      # (`Reply#error`) scores 0 without troubling the scorer; an exception is
-      # the transport's and is left to the caller, which counts it as an error.
-      #
-      # @param kase [Case]
-      # @return [RunResult]
-      def evaluate(kase)
-        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        reply = adapter.call(kase, model: model)
-        raise Error, "#{adapter.class}#call returned #{reply.class}, not an LLM::Evals::Reply" unless reply.is_a?(Reply)
-
-        score = reply.error? ? Score.failed(scorer.metrics, reply.error) : scorer.call(kase, reply, context: adapter.context(kase))
-        RunResult.new(experiment: self,
-          case: kase,
-          reply: reply,
-          score: score,
-          duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
-      end
-
-      private
-
-      # Remembers a result and tells the trackers, under one lock, so a
-      # tracker sees results one at a time and in the order they are kept.
-      def remember(result)
-        @lock.synchronize do
-          @results[result.case.key] = result
-          trackers.each { it.record(result) }
-        end
-        result
-      end
-
-      # A fixed pool of threads over a queue: `parallelism` cases in flight.
-      def run_locally(on_case)
-        queue = Queue.new
-        cases.each { queue << it }
-        queue.close
-        errors = Queue.new
-        Array.new([parallelism, cases.size].min) do
-          Thread.new do
-            while (kase = queue.pop)
-              begin
-                remember evaluate(kase)
-              rescue StandardError => e
-                remember RunResult.new(experiment: self, case: kase, error: "#{e.class}: #{e.message}")
-                errors << "#{kase.key}: #{e.class}: #{e.message}"
-              end
-              on_case&.call
-            end
-          end
-        end.each(&:join)
-        Array.new(errors.size) { errors.pop }
-      end
-
-      def summarize(errors, permalink, duration)
-        results = @lock.synchronize { @results.values }.sort_by { cases.index(it.case) }
-        Summary.new(
-          model:     model,
-          dataset:   dataset,
-          name:      name,
-          means:     self.class.means(scorer.metrics, results),
-          results:   results,
-          errors:    Array(errors),
-          duration:  duration,
-          permalink: permalink,
-          usage:     self.class.usage(results)
-        )
-      end
-    end
-  end
-end

data/lib/llm/evals/helper.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Finds and loads the suite: `evals/eval_helper.rb`, the file that calls
-    # {LLM::Evals.configure}, or, for a config-only suite, the directory
-    # holding `evals/.llm-evals.config.yml`. The executable looks before
-    # dispatching, the way rspec finds spec_helper, so the suite's own
-    # directory needs no wrapper script.
-    module Helper
-      FILE = File.join("evals", "eval_helper.rb")
-      ENV_VAR = "LLM_EVALS_HELPER"
-      FLAG = "--helper"
-
-      # What {.locate} found: a helper to load, a config file to read, or neither.
-      Found = Data.define(:helper, :config) do
-        def initialize(helper: nil, config: nil) = super
-
-        # @return [Boolean] whether there is anything to load
-        def any? = !helper.nil? || !config.nil?
-
-        # @return [Boolean]
-        def none? = !any?
-      end
-
-      module_function
-
-      # Pulls `--helper PATH` (or `--helper=PATH`) out of argv, so dry-cli
-      # never sees a flag no command declares.
-      #
-      # @param argv [Array<String>]
-      # @return [Array(Array<String>, String, nil)] the remaining arguments and the path, if given
-      def extract(argv)
-        rest = []
-        path = nil
-        skip = false
-        argv.each_with_index do |arg, index|
-          next skip = false if skip
-
-          if arg == FLAG
-            path = argv[index + 1]
-            skip = true
-          elsif arg.start_with?("#{FLAG}=")
-            path = arg.delete_prefix("#{FLAG}=")
-          else
-            rest << arg
-          end
-        end
-        [rest, path]
-      end
-
-      # The suite to load: the flag, then the variable, then, from the
-      # directory upward, the nearest `evals/eval_helper.rb` or, failing
-      # that, the nearest `evals/.llm-evals.config.yml` (also found from
-      # inside `evals/` itself).
-      #
-      # @param explicit [String, nil] what `--helper` was given
-      # @param from [String] the directory to search up from
-      # @param env [Hash] the environment
-      # @return [Found] absolute paths; empty when there is none
-      def locate(explicit = nil, from: Dir.pwd, env: ENV)
-        candidate = explicit || env[ENV_VAR]
-        return Found.new(helper: File.expand_path(candidate, from)) if candidate && !candidate.empty?
-
-        Pathname(from).expand_path.ascend do |dir|
-          helper = dir.join(FILE)
-          config = [dir.join("evals", Configuration::DEFAULT_CONFIG), dir.join(Configuration::DEFAULT_CONFIG)].find(&:file?)
-          return Found.new(helper: helper.to_s, config: config&.to_s) if helper.file?
-          return Found.new(config: config.to_s) if config
-        end
-        Found.new
-      end
-
-      # Loads the helper, once, or roots the configuration at a config-only
-      # suite's directory so {Configuration#resolve_from_config!} can read it.
-      # A helper is loaded first and wins; the config fills what it left unset.
-      #
-      # @param found [Found, String, nil] nothing is loaded for nil or an empty Found
-      # @param configuration [Configuration]
-      # @return [Boolean] whether anything was loaded
-      # @raise [Error] when the helper does not exist
-      def load!(found, configuration: LLM::Evals.configuration)
-        found = Found.new(helper: found) if found.is_a?(String)
-        return false if found.nil? || found.none?
-
-        if found.helper
-          raise Error, "no eval helper at #{found.helper}" unless File.file?(found.helper)
-
-          require File.expand_path(found.helper)
-        else
-          root = File.dirname(found.config)
-          configuration.suite(File.basename(File.dirname(root)), root: root) unless configuration.complete?
-        end
-        true
-      end
-    end
-  end
-end

data/lib/llm/evals/metric.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One number a {Scorer} reports per case. `aggregate` is how the run
-    # combines them (only :mean today), skipping nil, so a metric a case does
-    # not apply to never drags its mean down with a free 0 or up with a free 1.
-    # The `primary` metric ranks the leaderboard and picks the best value.
-    class Metric < Data.define(:name, :description, :aggregate, :primary)
-      AGGREGATES = %i[mean].freeze
-
-      def initialize(name:, description: "", aggregate: :mean, primary: false)
-        raise ArgumentError, "aggregate must be one of #{AGGREGATES.join(", ")}" unless AGGREGATES.include?(aggregate)
-
-        super(name: name.to_s, description: description, aggregate: aggregate, primary: primary)
-      end
-
-      # @param values [Array<Float, nil>] one per case
-      # @return [Float, nil] nil when no case reported the metric
-      def combine(values)
-        present = values.compact
-        present.sum.fdiv(present.size) unless present.empty?
-      end
-    end
-  end
-end

data/lib/llm/evals/model.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One model at one setting, reached through its provider. Adapters call
-    # {#complete}; the settings an experiment is named by are read here.
-    class Model
-      # The settings every provider understands; a provider may accept more.
-      SETTINGS = %i[effort temperature max_tokens].freeze
-
-      attr_reader :id, :provider, :settings
-
-      # @param id [String] the model id as the lineup names it, e.g. "anthropic/claude-haiku-5.5"
-      # @param provider [Providers::Base] what answers {#complete}
-      # @param settings [Hash{Symbol => Object}] effort, temperature, max_tokens; nil leaves the model's default
-      def initialize(id:, provider:, settings: {})
-        @id = id
-        @provider = provider
-        @settings = settings.transform_keys(&:to_sym).freeze
-      end
-
-      # @return [String, nil] reasoning effort; nil sends none
-      def effort = settings[:effort]
-
-      # @return [Float, nil] sampling temperature; nil sends none
-      def temperature = settings[:temperature]
-
-      # @return [Integer, nil] output token cap
-      def max_tokens = settings[:max_tokens]
-
-      # @return [String] the id without the provider's prefix, as the provider's API knows it
-      def name = provider.model_name(id)
-
-      # @return [String] "provider:id", unique where two providers render the same id
-      def key = "#{provider.name}:#{id}"
-
-      # One call. Transport failures raise; they are the run's errors, not the model's.
-      #
-      # @param system [String, nil]
-      # @param user [String]
-      # @param json [Boolean] ask the provider for a JSON object reply where it supports that
-      # @return [Completion]
-      def complete(user:, system: nil, json: true)
-        provider.complete(self, system: system, user: user, json: json)
-      end
-
-      # @return [String] e.g. "effort=low t=0.2"
-      def label
-        "effort=#{effort || "default"} t=#{temperature.nil? ? "default" : temperature}"
-      end
-
-      def to_s = "#{id} #{label}"
-    end
-  end
-end

data/lib/llm/evals/reply.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What an {Adapter} returns for one case. `error` is the model's failure
-    # (it did not answer the question: no JSON, a refusal) and scores 0; a
-    # transport failure is not the model's and is raised instead, so it is
-    # counted as an error of the run rather than a wrong answer.
-    Reply = Data.define(:output, :raw, :usage, :error, :metadata) do
-      def initialize(output: nil, raw: nil, usage: Usage.zero, error: nil, metadata: {})
-        super(output: output, raw: raw, usage: Usage.from(usage), error: error, metadata: metadata)
-      end
-
-      # @return [Boolean] whether the model failed to answer
-      def error? = !error.nil?
-    end
-  end
-end

data/lib/llm/evals/run_result.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # One case, run once: what came back and what it scored. `reply` is nil
-    # when the adapter raised, and `error` then holds the message.
-    RunResult = Data.define(:experiment, :case, :reply, :score, :duration, :error) do
-      def initialize(experiment:, case:, reply: nil, score: nil, duration: 0.0, error: nil) = super
-
-      # @return [Boolean] whether the adapter raised rather than answered
-      def failed? = !error.nil?
-    end
-  end
-end

data/lib/llm/evals/schema_validator.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Checks a Hash of answers against a field map, for scorers and verifiers
-    # whose expected values are typed and closed. Nothing here is specific to
-    # what the fields mean.
-    #
-    # The map is `{ "name" => { type:, enum:, nullable: } }`; `type` is one of
-    # {TYPES}, `enum` lists the allowed values for enum and multi_enum, and a
-    # nullable field accepts nil (the text never answered it).
-    class SchemaValidator
-      TYPES = %i[boolean integer number string enum multi_enum].freeze
-
-      attr_reader :fields
-
-      # @param fields [Hash{String, Symbol => Hash}]
-      # @raise [ArgumentError] on a type outside {TYPES}, since a typo there would pass everything
-      def initialize(fields)
-        @fields = fields.to_h { |name, spec| [name.to_s, normalize(name, spec)] }.freeze
-      end
-
-      # @return [Array<String>] the field names
-      def names = fields.keys
-
-      # @param values [Hash] String or Symbol keys
-      # @return [Array<String>] every problem found; empty when the values fit the map
-      def problems(values)
-        return ["#{values.inspect} is not a Hash"] unless values.is_a?(Hash)
-
-        actual = values.transform_keys(&:to_s)
-        missing = names - actual.keys
-        extra = actual.keys - names
-        problems = []
-        problems << "missing #{missing.join(", ")}" if missing.any?
-        problems << "unknown fields #{extra.join(", ")}" if extra.any?
-        fields.each do |name, spec|
-          next unless actual.key?(name)
-
-          problem = value_problem(spec, actual[name])
-          problems << "#{name}: #{problem}" if problem
-        end
-        problems
-      end
-
-      # @param values [Hash]
-      # @return [Boolean]
-      def valid?(values) = problems(values).empty?
-
-      # @param spec [Hash] one field's normalized spec
-      # @param value [Object]
-      # @return [String, nil] what is wrong with the value, or nil
-      def value_problem(spec, value)
-        return (spec[:nullable] ? nil : "is null") if value.nil?
-
-        case spec[:type]
-        when :boolean then "#{value.inspect} is not a boolean" unless [true, false].include?(value)
-        when :integer then "#{value.inspect} is not an integer" unless value.is_a?(Integer)
-        when :number then "#{value.inspect} is not a number" unless value.is_a?(Numeric)
-        when :string then "#{value.inspect} is not a string" unless value.is_a?(String)
-        when :enum then "#{value.inspect} is not one of #{spec[:enum].join("|")}" unless spec[:enum].include?(value)
-        when :multi_enum then multi_enum_problem(spec, value)
-        end
-      end
-
-      private
-
-      def multi_enum_problem(spec, value)
-        return "#{value.inspect} is not an array" unless value.is_a?(Array)
-
-        stray = value - spec[:enum]
-        "#{stray.join(", ")} not in #{spec[:enum].join("|")}" if stray.any?
-      end
-
-      def normalize(name, spec)
-        spec = spec.transform_keys(&:to_sym)
-        type = spec.fetch(:type).to_sym
-        raise ArgumentError, "#{name}: unknown type #{type.inspect}, expected one of #{TYPES.join(", ")}" unless TYPES.include?(type)
-
-        { type: type, enum: Array(spec[:enum] || spec[:values]).map(&:to_s), nullable: spec.fetch(:nullable, false) }
-      end
-    end
-  end
-end

data/lib/llm/evals/score.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What a {Scorer} says about one reply. `value` is the case's overall
-    # score in 0..1; `metrics` holds one entry per {Metric} the scorer
-    # declares, nil where the case does not apply; `outcomes` and
-    # `diagnostics` are free-form, keyed by field, and only ever displayed.
-    Score = Data.define(:value, :metrics, :outcomes, :diagnostics) do
-      def initialize(value:, metrics: {}, outcomes: {}, diagnostics: {})
-        raise ArgumentError, "score value #{value.inspect} is not within 0..1" unless value.is_a?(Numeric) && (0..1).cover?(value)
-
-        super(value: value.to_f, metrics: metrics.transform_keys(&:to_s), outcomes: outcomes, diagnostics: diagnostics)
-      end
-
-      # The score of a reply the model failed: 0 everywhere, the failure as a diagnostic.
-      #
-      # @param metrics [Array<Metric>]
-      # @param error [String]
-      # @return [Score]
-      def self.failed(metrics, error)
-        new(value: 0.0, metrics: metrics.to_h { [it.name, 0.0] }, diagnostics: { "error" => error })
-      end
-    end
-  end
-end

data/lib/llm/evals/scorer.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What a suite implements to score one reply. Pure and deterministic: it
-    # reads the case, the reply and the adapter's context, and calls nothing.
-    #
-    # @abstract Subclass and override {#metrics} and {#call}.
-    class Scorer
-      # The metrics every {Score} this scorer returns carries. The CLI builds
-      # its columns, its glossary and its Braintrust scorers from this list,
-      # so a metric missing here is a metric nobody sees.
-      #
-      # @return [Array<Metric>]
-      def metrics
-        raise NotImplementedError, "#{self.class}#metrics must return an Array of LLM::Evals::Metric"
-      end
-
-      # @return [Metric] the one flagged primary, else the first
-      def primary = metrics.find(&:primary) || metrics.first
-
-      # @param kase [Case]
-      # @param reply [Reply] never one with `error` set; the runner scores those 0 itself
-      # @param context [Hash] what {Adapter#context} returned for the case
-      # @return [Score]
-      def call(kase, reply, context:)
-        raise NotImplementedError, "#{self.class}#call(kase, reply, context:) must return an LLM::Evals::Score"
-      end
-
-      # Offline checks of one case against what the scorer will demand of it,
-      # so a typo in expected.json is caught before it is billed as a model's mistake.
-      #
-      # @param kase [Case]
-      # @param context [Hash] what {Adapter#context} returned for the case
-      # @return [Array<String>] problems; empty when the case is sound
-      def verify(kase, context:) = [] # rubocop:disable Lint/UnusedMethodArgument -- the signature is the contract
-
-      # One short cell describing the case in `llm-evals cases`, e.g. "5/8 answered".
-      #
-      # @param kase [Case]
-      # @return [String, nil] nil for the generic fallback
-      def summary(_kase) = nil
-    end
-  end
-end

data/lib/llm/evals/tracker.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # Where a run's results go besides the terminal. The gem always owns the
-    # run loop; a tracker only receives callbacks: {#start} once per
-    # experiment, {#record} once per finished case, {#finish} once with the
-    # summary. The local JSON log ({Trackers::Local}) always runs; at most
-    # one remote tracker runs beside it, chosen by `--tracker`, the
-    # `tracker:` key, or the environment (see {Trackers::Registry#choose}).
-    #
-    # A tracker for another framework (W&B Weave, an OTLP backend) subclasses
-    # this in a consumer or a plugin gem and is registered with `c.tracker`.
-    #
-    # @abstract Override the three callbacks; declare {.env_key} and {.gems}.
-    class Tracker
-      class << self
-        # @return [String] the name `--tracker` and `tracker:` use; the class name, dasherized, by default
-        def tracker_name = (name || "Tracker").split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
-
-        # @return [String, nil] the environment variable whose presence selects this tracker when nothing else does
-        def env_key = nil
-
-        # @return [Array<String>] the optional gems this tracker requires when chosen
-        def gems = []
-
-        # @param env [Hash] the environment
-        # @return [Boolean] whether the environment names this tracker
-        def detect?(env = ENV) = !env_key.nil? && !env[env_key].to_s.strip.empty?
-
-        # Loads the gems and checks the key, with an error that names both, so
-        # a chosen tracker that cannot run says so before any model is called.
-        #
-        # @param env [Hash]
-        # @return [void]
-        # @raise [MissingDependency, Error]
-        def requirements!(env = ENV)
-          gems.each do |gem_name|
-            LLM::Evals.optional!(gem_name.tr("-", "/"), gem_name, needed_for: "the #{tracker_name} tracker")
-          end
-          return if env_key.nil? || detect?(env)
-
-          raise Error, "the #{tracker_name} tracker needs #{env_key} in the environment (gems: #{gems.join(", ")})"
-        end
-      end
-
-      attr_reader :context
-
-      # @param context [Hash] `suite:`, `project:`, `project_id:` and `options:` (the tracker's own key in the config file)
-      def initialize(**context)
-        @context = context
-      end
-
-      # @return [String]
-      def name = self.class.tracker_name
-
-      # @return [Boolean] false turns every callback into a no-op
-      def enabled? = true
-
-      # @return [String] one line for `--verbose`: where this tracker sends things, and why
-      def describe = name
-
-      # @param experiment [Experiment]
-      # @return [void]
-      def start(_experiment) = nil
-
-      # @param result [RunResult]
-      # @return [void]
-      def record(_result) = nil
-
-      # @param experiment [Experiment]
-      # @param summary [Experiment::Summary]
-      # @return [Hash] `url:` to the experiment when one is known, else nil
-      def finish(_experiment, _summary) = { url: nil }
-
-      # @return [Hash] the tracker's own section of the config file
-      def options = context.fetch(:options, nil) || {}
-    end
-  end
-end

data/lib/llm/evals/usage.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module LLM
-  module Evals
-    # What one call cost, as the provider counted it. Summed over a run.
-    Usage = Data.define(:input_tokens, :output_tokens, :cost_usd) do
-      # @return [Usage] zero of everything
-      def self.zero = new(input_tokens: 0, output_tokens: 0, cost_usd: 0.0)
-
-      # Reads a Hash with Symbol or String keys, missing keys as zero.
-      #
-      # @param hash [Hash, Usage, nil]
-      # @return [Usage]
-      def self.from(hash)
-        return hash if hash.is_a?(Usage)
-        return zero if hash.nil?
-
-        h = hash.transform_keys(&:to_sym)
-        new(input_tokens: h.fetch(:input_tokens, 0).to_i,
-          output_tokens: h.fetch(:output_tokens, 0).to_i,
-          cost_usd: h.fetch(:cost_usd, 0.0).to_f)
-      end
-
-      # @param other [Usage]
-      # @return [Usage]
-      def +(other)
-        with(input_tokens: input_tokens + other.input_tokens,
-          output_tokens: output_tokens + other.output_tokens,
-          cost_usd: cost_usd + other.cost_usd)
-      end
-    end
-  end
-end