llm-evals 0.4.0

data/lib/llm/evals/discovery.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Finds the cases under a root with a glob and decodes each path into a
+    # {Case}. The default layout is
+    #
+    #   datasets/<dataset>/cases/<variant>/[NNN/]input.txt
+    #   datasets/<dataset>/cases/<variant>/[NNN/]expected.json
+    #
+    # The folder after `datasets/` is the dataset, the folder holding the
+    # input is the variant, unless it is a number, in which case it is the
+    # case number and its parent is the variant.
+    class Discovery
+      EXPECTED = "expected.json"
+      METADATA = "metadata.json"
+
+      attr_reader :root, :glob
+
+      # @param root [String] an absolute directory
+      # @param glob [String] relative to root
+      def initialize(root:, glob: Configuration::DEFAULT_GLOB)
+        @root = root
+        @glob = glob
+      end
+
+      # @return [Array<String>] every dataset that has at least one case, sorted
+      def datasets = all.map(&:dataset).uniq.sort
+
+      # @param dataset [String, nil] one dataset; nil for all of them
+      # @param only [Array<String>] id prefixes to keep ("clear", "clear/00"); empty keeps all
+      # @return [Array<Case>] sorted by dataset, variant and number
+      # @raise [Error] when the dataset does not exist
+      def cases(dataset: nil, only: [])
+        found = all
+        if dataset
+          raise Error, "no dataset #{dataset.inspect}; have #{datasets.join(", ")}" unless datasets.include?(dataset)
+
+          found = found.select { it.dataset == dataset }
+        end
+        only = only.flat_map { it.split(",") }.map(&:strip).reject(&:empty?)
+        found = found.select { |kase| only.any? { kase.id.start_with?(it) || kase.key.start_with?(it) } } unless only.empty?
+        found
+      end
+
+      # @return [Array<Case>]
+      def all
+        @all ||= Dir[File.join(root, glob)].map { decode(it) }
+      end
+
+      # Builds the case at one input file.
+      #
+      # @param input_path [String]
+      # @return [Case]
+      def decode(input_path)
+        dir = File.dirname(input_path)
+        parts = Pathname(dir).relative_path_from(Pathname(root)).each_filename.to_a
+        number = parts.last.match?(/\A\d+\z/) ? parts.last : nil
+        variant = number ? parts[-2] : parts.last
+        expected, problem = read_expected(File.join(dir, EXPECTED))
+        Case.new(
+          id:       [variant, number].compact.join("/"),
+          dataset:  dataset_of(parts),
+          variant:  variant,
+          input:    File.read(input_path).strip,
+          expected: expected,
+          path:     dir,
+          metadata: metadata(dir, number, problem)
+        )
+      end
+
+      private
+
+      # The folder after `datasets/`, else the first folder, so a glob that
+      # does not say `datasets` still groups by its top level.
+      def dataset_of(parts)
+        index = parts.index("datasets")
+        index && parts[index + 1] ? parts[index + 1] : parts.first
+      end
+
+      def read_expected(path)
+        return [nil, "#{EXPECTED} is missing"] unless File.exist?(path)
+
+        [JSON.parse(File.read(path)), nil]
+      rescue JSON::ParserError => e
+        [nil, "#{EXPECTED} is not valid JSON: #{e.message.lines.first&.strip}"]
+      end
+
+      def metadata(dir, number, problem)
+        extra = File.exist?(File.join(dir, METADATA)) ? JSON.parse(File.read(File.join(dir, METADATA))).transform_keys(&:to_sym) : {}
+        { number: number&.to_i, difficulty: number.to_i, expected_error: problem }.compact.merge(extra)
+      end
+    end
+  end
+end

data/lib/llm/evals/experiment.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One model at one setting over every case of one dataset, run on a
+    # thread pool the gem owns, plus the per-case detail the terminal report
+    # needs. Trackers only watch: {Trackers::Local} always, and at most one
+    # remote tracker, each told when the experiment starts, when a case
+    # finishes, and what the summary was.
+    class Experiment
+      # What one finished experiment reports back. `means` holds one entry
+      # per metric plus "score", the mean of each case's overall value;
+      # `usage` is summed over the cases; `permalink` is the remote tracker's
+      # page, when it has one.
+      Summary = Data.define(:model, :dataset, :name, :means, :results, :errors, :duration, :permalink, :usage) do
+        # @return [Float, nil] the mean overall score
+        def score = means["score"]
+
+        # @param metric [Metric, String]
+        # @return [Float, nil]
+        def mean(metric) = means[metric.respond_to?(:name) ? metric.name : metric.to_s]
+
+        # The same experiment over one variant's cases. Its duration is the sum
+        # of those cases' own, since they overlapped with the others.
+        #
+        # @param variant [String]
+        # @param metrics [Array<Metric>] the scorer's
+        # @return [Summary]
+        def for_variant(variant, metrics)
+          chosen = results.select { it.case.variant == variant }
+          with(means:    Experiment.means(metrics, chosen),
+            results:  chosen,
+            errors:   chosen.select(&:failed?).map { "#{it.case.key}: #{it.error}" },
+            duration: chosen.sum(&:duration),
+            usage:    Experiment.usage(chosen))
+        end
+      end
+
+      # @param metrics [Array<Metric>]
+      # @param results [Array<RunResult>]
+      # @return [Hash{String => Float, nil}] one mean per metric, plus "score"
+      def self.means(metrics, results)
+        scored = results.filter_map(&:score)
+        means = metrics.to_h { |metric| [metric.name, metric.combine(scored.map { it.metrics[metric.name] })] }
+        means.merge("score" => scored.empty? ? nil : scored.sum(&:value).fdiv(scored.size))
+      end
+
+      # @param results [Array<RunResult>]
+      # @return [Usage] summed over the results
+      def self.usage(results) = results.filter_map(&:reply).map(&:usage).reduce(Usage.zero, :+)
+
+      attr_reader :model, :adapter, :scorer, :cases, :dataset, :parallelism, :trackers
+
+      # @param model [Model]
+      # @param cases [Array<Case>] all of one dataset
+      # @param adapter [Adapter]
+      # @param scorer [Scorer]
+      # @param parallelism [Integer] cases in flight at once
+      # @param trackers [Array<Tracker>] told about the run, in this order
+      def initialize(model:, cases:, adapter:, scorer:, parallelism: 4, trackers: [])
+        raise ArgumentError, "an experiment needs at least one case" if cases.empty?
+
+        @model = model
+        @cases = cases.sort_by { [it.variant, it.number || 0, it.id] }
+        @dataset = cases.first.dataset
+        @adapter = adapter
+        @scorer = scorer
+        @parallelism = [parallelism.to_i, 1].max
+        @trackers = trackers.select(&:enabled?)
+        @results = {}
+        @lock = Mutex.new
+      end
+
+      # @return [String] e.g. "anthropic/claude-haiku-5.5 effort=low t=0.0 [clear]"
+      def name = "#{model} [#{dataset}]"
+
+      # @param on_case [#call, nil] called once per finished case
+      # @return [Summary]
+      def run(on_case: nil)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        adapter.prepare(dataset)
+        trackers.each { it.start(self) }
+        errors = run_locally(on_case)
+        summary = summarize(errors, nil, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+        urls = trackers.map { it.finish(self, summary)&.fetch(:url, nil) }
+        summary.with(permalink: urls.compact.first)
+      end
+
+      # Runs one case: the adapter, then the scorer. A reply the model failed
+      # (`Reply#error`) scores 0 without troubling the scorer; an exception is
+      # the transport's and is left to the caller, which counts it as an error.
+      #
+      # @param kase [Case]
+      # @return [RunResult]
+      def evaluate(kase)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        reply = adapter.call(kase, model: model)
+        raise Error, "#{adapter.class}#call returned #{reply.class}, not an LLM::Evals::Reply" unless reply.is_a?(Reply)
+
+        score = reply.error? ? Score.failed(scorer.metrics, reply.error) : scorer.call(kase, reply, context: adapter.context(kase))
+        RunResult.new(experiment: self,
+          case: kase,
+          reply: reply,
+          score: score,
+          duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      end
+
+      private
+
+      # Remembers a result and tells the trackers, under one lock, so a
+      # tracker sees results one at a time and in the order they are kept.
+      def remember(result)
+        @lock.synchronize do
+          @results[result.case.key] = result
+          trackers.each { it.record(result) }
+        end
+        result
+      end
+
+      # A fixed pool of threads over a queue: `parallelism` cases in flight.
+      def run_locally(on_case)
+        queue = Queue.new
+        cases.each { queue << it }
+        queue.close
+        errors = Queue.new
+        Array.new([parallelism, cases.size].min) do
+          Thread.new do
+            while (kase = queue.pop)
+              begin
+                remember evaluate(kase)
+              rescue StandardError => e
+                remember RunResult.new(experiment: self, case: kase, error: "#{e.class}: #{e.message}")
+                errors << "#{kase.key}: #{e.class}: #{e.message}"
+              end
+              on_case&.call
+            end
+          end
+        end.each(&:join)
+        Array.new(errors.size) { errors.pop }
+      end
+
+      def summarize(errors, permalink, duration)
+        results = @lock.synchronize { @results.values }.sort_by { cases.index(it.case) }
+        Summary.new(
+          model:     model,
+          dataset:   dataset,
+          name:      name,
+          means:     self.class.means(scorer.metrics, results),
+          results:   results,
+          errors:    Array(errors),
+          duration:  duration,
+          permalink: permalink,
+          usage:     self.class.usage(results)
+        )
+      end
+    end
+  end
+end

data/lib/llm/evals/helper.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Finds and loads the suite: `evals/eval_helper.rb`, the file that calls
+    # {LLM::Evals.configure}, or, for a config-only suite, the directory
+    # holding `evals/.llm-evals.config.yml`. The executable looks before
+    # dispatching, the way rspec finds spec_helper, so the suite's own
+    # directory needs no wrapper script.
+    module Helper
+      FILE = File.join("evals", "eval_helper.rb")
+      ENV_VAR = "LLM_EVALS_HELPER"
+      FLAG = "--helper"
+
+      # What {.locate} found: a helper to load, a config file to read, or neither.
+      Found = Data.define(:helper, :config) do
+        def initialize(helper: nil, config: nil) = super
+
+        # @return [Boolean] whether there is anything to load
+        def any? = !helper.nil? || !config.nil?
+
+        # @return [Boolean]
+        def none? = !any?
+      end
+
+      module_function
+
+      # Pulls `--helper PATH` (or `--helper=PATH`) out of argv, so dry-cli
+      # never sees a flag no command declares.
+      #
+      # @param argv [Array<String>]
+      # @return [Array(Array<String>, String, nil)] the remaining arguments and the path, if given
+      def extract(argv)
+        rest = []
+        path = nil
+        skip = false
+        argv.each_with_index do |arg, index|
+          next skip = false if skip
+
+          if arg == FLAG
+            path = argv[index + 1]
+            skip = true
+          elsif arg.start_with?("#{FLAG}=")
+            path = arg.delete_prefix("#{FLAG}=")
+          else
+            rest << arg
+          end
+        end
+        [rest, path]
+      end
+
+      # The suite to load: the flag, then the variable, then, from the
+      # directory upward, the nearest `evals/eval_helper.rb` or, failing
+      # that, the nearest `evals/.llm-evals.config.yml` (also found from
+      # inside `evals/` itself).
+      #
+      # @param explicit [String, nil] what `--helper` was given
+      # @param from [String] the directory to search up from
+      # @param env [Hash] the environment
+      # @return [Found] absolute paths; empty when there is none
+      def locate(explicit = nil, from: Dir.pwd, env: ENV)
+        candidate = explicit || env[ENV_VAR]
+        return Found.new(helper: File.expand_path(candidate, from)) if candidate && !candidate.empty?
+
+        Pathname(from).expand_path.ascend do |dir|
+          helper = dir.join(FILE)
+          config = [dir.join("evals", Configuration::DEFAULT_CONFIG), dir.join(Configuration::DEFAULT_CONFIG)].find(&:file?)
+          return Found.new(helper: helper.to_s, config: config&.to_s) if helper.file?
+          return Found.new(config: config.to_s) if config
+        end
+        Found.new
+      end
+
+      # Loads the helper, once, or roots the configuration at a config-only
+      # suite's directory so {Configuration#resolve_from_config!} can read it.
+      # A helper is loaded first and wins; the config fills what it left unset.
+      #
+      # @param found [Found, String, nil] nothing is loaded for nil or an empty Found
+      # @param configuration [Configuration]
+      # @return [Boolean] whether anything was loaded
+      # @raise [Error] when the helper does not exist
+      def load!(found, configuration: LLM::Evals.configuration)
+        found = Found.new(helper: found) if found.is_a?(String)
+        return false if found.nil? || found.none?
+
+        if found.helper
+          raise Error, "no eval helper at #{found.helper}" unless File.file?(found.helper)
+
+          require File.expand_path(found.helper)
+        else
+          root = File.dirname(found.config)
+          configuration.suite(File.basename(File.dirname(root)), root: root) unless configuration.complete?
+        end
+        true
+      end
+    end
+  end
+end

data/lib/llm/evals/launcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Starts the CLI. The executable and the specs both build one, each with
+    # its own argv and streams, and it exits only through kernel.exit.
+    class Launcher
+      def initialize(argv, stdin = $stdin, stdout = $stdout, stderr = $stderr, kernel = Kernel)
+        @argv = argv
+        @stdin = stdin
+        @stdout = stdout
+        @stderr = stderr
+        @kernel = kernel
+      end
+
+      def execute!
+        @kernel.exit(run)
+      end
+
+      private
+
+      # dry-cli exits by itself after help or a usage error; its status is kept.
+      def run
+        argv, helper = Helper.extract(@argv)
+        Helper.load!(Helper.locate(helper))
+        Dry::CLI.new(CLI).call(arguments: argv, out: @stdout, err: @stderr)
+        0
+      rescue SystemExit => e
+        e.status
+      rescue StandardError => e
+        @stderr.puts "error: #{e.message}"
+        @stderr.puts(e.backtrace) if @argv.intersect?(%w[-v --verbose])
+        1
+      end
+    end
+  end
+end

data/lib/llm/evals/metric.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One number a {Scorer} reports per case. `aggregate` is how the run
+    # combines them (only :mean today), skipping nil, so a metric a case does
+    # not apply to never drags its mean down with a free 0 or up with a free 1.
+    # The `primary` metric ranks the leaderboard and picks the best value.
+    class Metric < Data.define(:name, :description, :aggregate, :primary)
+      AGGREGATES = %i[mean].freeze
+
+      def initialize(name:, description: "", aggregate: :mean, primary: false)
+        raise ArgumentError, "aggregate must be one of #{AGGREGATES.join(", ")}" unless AGGREGATES.include?(aggregate)
+
+        super(name: name.to_s, description: description, aggregate: aggregate, primary: primary)
+      end
+
+      # @param values [Array<Float, nil>] one per case
+      # @return [Float, nil] nil when no case reported the metric
+      def combine(values)
+        present = values.compact
+        present.sum.fdiv(present.size) unless present.empty?
+      end
+    end
+  end
+end

data/lib/llm/evals/model.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One model at one setting, reached through its provider. Adapters call
+    # {#complete}; the settings an experiment is named by are read here.
+    class Model
+      # The settings every provider understands; a provider may accept more.
+      SETTINGS = %i[effort temperature max_tokens].freeze
+
+      attr_reader :id, :provider, :settings
+
+      # @param id [String] the model id as the lineup names it, e.g. "anthropic/claude-haiku-5.5"
+      # @param provider [Providers::Base] what answers {#complete}
+      # @param settings [Hash{Symbol => Object}] effort, temperature, max_tokens; nil leaves the model's default
+      def initialize(id:, provider:, settings: {})
+        @id = id
+        @provider = provider
+        @settings = settings.transform_keys(&:to_sym).freeze
+      end
+
+      # @return [String, nil] reasoning effort; nil sends none
+      def effort = settings[:effort]
+
+      # @return [Float, nil] sampling temperature; nil sends none
+      def temperature = settings[:temperature]
+
+      # @return [Integer, nil] output token cap
+      def max_tokens = settings[:max_tokens]
+
+      # @return [String] the id without the provider's prefix, as the provider's API knows it
+      def name = provider.model_name(id)
+
+      # @return [String] "provider:id", unique where two providers render the same id
+      def key = "#{provider.name}:#{id}"
+
+      # One call. Transport failures raise; they are the run's errors, not the model's.
+      #
+      # @param system [String, nil]
+      # @param user [String]
+      # @param json [Boolean] ask the provider for a JSON object reply where it supports that
+      # @return [Completion]
+      def complete(user:, system: nil, json: true)
+        provider.complete(self, system: system, user: user, json: json)
+      end
+
+      # @return [String] e.g. "effort=low t=0.2"
+      def label
+        "effort=#{effort || "default"} t=#{temperature.nil? ? "default" : temperature}"
+      end
+
+      def to_s = "#{id} #{label}"
+    end
+  end
+end

data/lib/llm/evals/providers/base.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Providers
+      # What answers a {Model}'s calls: one API, the ids it serves, the keys it
+      # needs and the models it lists. A suite with its own transport
+      # subclasses this and registers it through `c.provider`.
+      #
+      # @abstract Override {#serves?}, {#env_keys}, {#catalog} and {#complete}.
+      class Base
+        # The reasoning efforts this provider's API accepts.
+        EFFORTS = %w[none minimal low medium high xhigh max].freeze
+
+        # The kind of request this provider answers; see {Config::REQUEST_TYPES}.
+        # A suite admits a provider only when its config lists this type.
+        #
+        # @return [Symbol]
+        def request_type
+          raise NotImplementedError, "#{self.class}#request_type must be one of #{Config::REQUEST_TYPES.join(", ")}"
+        end
+
+        # One line on what the request type is, for the README and `--help`;
+        # a suite's config never says it, the provider owns it.
+        #
+        # @return [String]
+        def request_type_description = ""
+
+        # A direct provider serves a vendor's own API and is reached only
+        # through a `direct:` entry that names it; it never claims an id from
+        # `vendors:`, however the id is spelled.
+        #
+        # @return [Boolean]
+        def direct? = false
+
+        # @return [String] short name, for messages and the leaderboard
+        def name = self.class.name.split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+
+        # @param model_id [String] as the lineup names it
+        # @return [Boolean] whether this provider serves the model
+        def serves?(model_id) = model_id.start_with?(prefix)
+
+        # @return [String] the id prefix this provider claims; "" claims everything
+        def prefix = ""
+
+        # @param model_id [String]
+        # @return [String] the id as the provider's API knows it
+        def model_name(model_id) = model_id.delete_prefix(prefix)
+
+        # @return [Array<String>] environment variables a run through this provider needs
+        def env_keys = []
+
+        # @return [Array<String>] the efforts the `-e` flag accepts for this provider's models
+        def efforts = EFFORTS
+
+        # Lists the model ids the provider serves, from the network, so a
+        # misspelled id aborts the run before any experiment starts instead
+        # of failing every case and showing up as a row of errors.
+        #
+        # @return [Array<String>, nil] ids as the lineup names them; nil when the provider cannot list them
+        def catalog = nil
+
+        # Prepares the client once per process, after the env files loaded.
+        #
+        # @option options [String] :app_name what the client tells the API the caller is
+        # @return [void]
+        def setup!(**) = nil
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean]
+        # @return [Completion]
+        def complete(model, system:, user:, json: true)
+          raise NotImplementedError, "#{self.class}#complete must return an LLM::Evals::Completion"
+        end
+
+        # @param contender [Contender]
+        # @return [Model]
+        def model_for(contender) = Model.new(id: contender.model, provider: self, settings: contender.settings)
+
+        # Every id in `model_ids` the provider does not list.
+        #
+        # @param model_ids [Array<String>]
+        # @return [Array<String>]
+        def unknown(model_ids)
+          listed = catalog
+          listed.nil? ? [] : model_ids - listed
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/providers/catalog.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module LLM
+  module Evals
+    module Providers
+      # One GET of a provider's model list, parsed. Every failure becomes an
+      # {Error} that names the URL, so an offline run says so instead of
+      # printing a socket error from inside a provider.
+      module Catalog
+        module_function
+
+        # @param url [String]
+        # @param headers [Hash{String => String}]
+        # @return [Hash] the parsed JSON body
+        # @raise [Error] when the list cannot be read
+        def fetch(url, headers = {})
+          uri = URI(url)
+          response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 10, read_timeout: 20) do |http|
+            http.request(Net::HTTP::Get.new(uri, headers))
+          end
+          raise Error, "could not list models at #{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+          JSON.parse(response.body)
+        rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout, JSON::ParserError, KeyError => e
+          raise Error, "could not list models at #{url}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/providers/open_router.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Providers
+      # Every model OpenRouter serves, through RubyLLM, because Braintrust
+      # traces RubyLLM calls (tokens, latency, cost) and not raw HTTP. Claims
+      # every id no other provider does, so it is always asked last.
+      class OpenRouter < Base
+        URL = "https://openrouter.ai/api/v1/models"
+        KEY = "OPENROUTER_API_KEY"
+
+        # OpenRouter's reasoning.effort values.
+        EFFORTS = %w[none minimal low medium high xhigh max].freeze
+
+        def name = "openrouter"
+        def request_type = :prompt
+        def request_type_description = "a chat completion: system and user text in, text out"
+        def env_keys = [KEY]
+        def efforts = EFFORTS
+
+        # @return [Array<String>] every id OpenRouter lists
+        def catalog = Catalog.fetch(URL).fetch("data").map { it.fetch("id") }
+
+        # Points RubyLLM at OpenRouter. Requires it here, not at load, so a
+        # suite that never reaches OpenRouter never needs the gem.
+        #
+        # @param app_name [String]
+        # @return [void]
+        def setup!(app_name:)
+          LLM::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
+          RubyLLM.configure do |config|
+            config.openrouter_api_key = ENV.fetch(KEY, nil)
+            config.openrouter_app_name = app_name
+          end
+        end
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean] asks for `response_format: json_object`
+        # @return [Completion] text, plus the tokens and the cost OpenRouter billed
+        def complete(model, system:, user:, json: true)
+          LLM::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
+          chat = RubyLLM.chat(model: model.name, provider: :openrouter, assume_model_exists: true)
+          chat.with_instructions(system) if system
+          chat.with_temperature(model.temperature) unless model.temperature.nil?
+          chat.with_max_output_tokens(model.max_tokens) if model.max_tokens
+          chat.with_thinking(effort: model.effort.to_sym) if model.effort
+          chat.with_provider_options(response_format: { type: "json_object" }) if json
+
+          message = chat.ask(user)
+          Completion.new(
+            text:  message.content.to_s,
+            usage: Usage.new(input_tokens: message.tokens.input.to_i,
+              output_tokens: message.tokens.output.to_i,
+              cost_usd: message.tokens.reported_cost.to_f),
+            raw:   message
+          )
+        end
+      end
+    end
+  end
+end