ruby_llm-contract 0.4.5 → 0.5.0

data/lib/ruby_llm/contract/eval/report.rb

@@ -1,209 +1,37 @@
 # frozen_string_literal: true
 
-require "json"
-require "fileutils"
+require "forwardable"
 
 module RubyLLM
   module Contract
     module Eval
       class Report
-        attr_reader :dataset_name, :results
+        extend Forwardable
+
+        attr_reader :dataset_name, :results, :step_name
+
+        GENERIC_DETAILS = ["passed", "not passed"].freeze
+        HISTORY_DIR = ".eval_history"
+        BASELINE_DIR = ".eval_baselines"
+
+        def_delegators :@stats, :score, :passed, :failed, :skipped, :failures, :pass_rate, :total_cost, :avg_latency_ms,
+                       :passed?
+        def_delegators :@presenter, :summary, :to_s, :print_summary
+        def_delegators :@storage, :save_history!, :eval_history, :save_baseline!, :compare_with_baseline,
+                       :baseline_exists?
 
         def initialize(dataset_name:, results:, step_name: nil)
           @dataset_name = dataset_name
           @step_name = step_name
           @results = results.dup.freeze
+          @stats = ReportStats.new(results: @results)
+          @presenter = ReportPresenter.new(report: self, stats: @stats)
+          @storage = ReportStorage.new(report: self, stats: @stats)
           freeze
         end
 
-        def score
-          evaluated = evaluated_results
-          return 0.0 if evaluated.empty?
-
-          evaluated.sum(&:score) / evaluated.length
-        end
-
-        def passed
-          evaluated_results.count(&:passed?)
-        end
-
-        def failed
-          evaluated_results.count(&:failed?)
-        end
-
-        def skipped
-          results.count { |r| r.step_status == :skipped }
-        end
-
-        def failures
-          evaluated_results.select(&:failed?)
-        end
-
-        def pass_rate
-          "#{passed}/#{evaluated_results.length}"
-        end
-
-        def total_cost
-          results.sum { |r| r.cost || 0.0 }
-        end
-
-        def avg_latency_ms
-          latencies = results.filter_map(&:duration_ms)
-          return nil if latencies.empty?
-
-          latencies.sum.to_f / latencies.length
-        end
-
-        def passed?
-          evaluated = evaluated_results
-          return false if evaluated.empty?
-
-          evaluated.all?(&:passed?)
-        end
-
-        def each(&)
-          results.each(&)
-        end
-
-        def summary
-          parts = ["#{dataset_name}: #{pass_rate} checks passed"]
-          parts << "#{skipped} skipped" if skipped.positive?
-          parts << format_cost(total_cost) if total_cost.positive?
-          parts.join(", ")
-        end
-
-        GENERIC_DETAILS = ["passed", "not passed"].freeze
-
-        def to_s
-          lines = [summary]
-          failures.each do |result|
-            lines << format_failure(result)
-          end
-          lines.join("\n")
-        end
-
-        def save_history!(path: nil, model: nil)
-          file = path || default_history_path(model: model)
-          run_data = {
-            date: Time.now.strftime("%Y-%m-%d"),
-            score: score,
-            total_cost: total_cost,
-            pass_rate: pass_rate,
-            cases_count: evaluated_results.length
-          }
-          EvalHistory.append(file, run_data)
-          file
-        end
-
-        def eval_history(path: nil, model: nil)
-          file = path || default_history_path(model: model)
-          EvalHistory.load(file)
-        end
-
-        def save_baseline!(path: nil, model: nil)
-          file = path || default_baseline_path(model: model)
-          FileUtils.mkdir_p(File.dirname(file))
-          File.write(file, JSON.pretty_generate(serialize_for_baseline))
-          file
-        end
-
-        def compare_with_baseline(path: nil, model: nil)
-          file = path || default_baseline_path(model: model)
-          raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)
-
-          baseline_data = JSON.parse(File.read(file), symbolize_names: true)
-          validate_baseline!(baseline_data)
-          BaselineDiff.new(
-            baseline_cases: baseline_data[:cases],
-            current_cases: results.map { |r| serialize_case(r) }
-          )
-        end
-
-        def baseline_exists?(path: nil, model: nil)
-          File.exist?(path || default_baseline_path(model: model))
-        end
-
-        def print_summary(io = $stdout)
-          io.puts summary
-          io.puts
-          results.each do |result|
-            icon = result.label
-            cost_str = result.cost ? "  #{format_cost(result.cost)}" : ""
-            latency_str = result.duration_ms ? "  #{result.duration_ms}ms" : ""
-            io.puts "  #{icon}  #{result.name}#{cost_str}#{latency_str}"
-            io.puts "        #{result.details}" if result.failed? && useful_details?(result.details)
-          end
-        end
-
-        private
-
-        def format_failure(result)
-          line = "  FAIL  #{result.name}"
-          line += ": #{result.details}" if useful_details?(result.details)
-          line
-        end
-
-        def useful_details?(details)
-          details && !GENERIC_DETAILS.include?(details)
-        end
-
-        def evaluated_results
-          results.reject { |r| r.step_status == :skipped }
-        end
-
-        def default_history_path(model: nil)
-          parts = [".eval_history"]
-          parts << sanitize_name(@step_name) if @step_name
-          name = sanitize_name(dataset_name)
-          name = "#{name}_#{sanitize_name(model)}" if model
-          parts << "#{name}.jsonl"
-          File.join(*parts)
-        end
-
-        def default_baseline_path(model: nil)
-          parts = [".eval_baselines"]
-          parts << sanitize_name(@step_name) if @step_name
-          name = sanitize_name(dataset_name)
-          name = "#{name}_#{sanitize_name(model)}" if model
-          parts << "#{name}.json"
-          File.join(*parts)
-        end
-
-        def validate_baseline!(data)
-          if data[:dataset_name] && data[:dataset_name] != dataset_name
-            raise ArgumentError, "Baseline eval '#{data[:dataset_name]}' does not match '#{dataset_name}'"
-          end
-          if data[:step_name] && @step_name && data[:step_name] != @step_name
-            raise ArgumentError, "Baseline step '#{data[:step_name]}' does not match '#{@step_name}'"
-          end
-        end
-
-        def sanitize_name(name)
-          name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
-        end
-
-        def serialize_for_baseline
-          {
-            dataset_name: dataset_name,
-            step_name: @step_name,
-            score: score,
-            total_cost: total_cost,
-            cases: evaluated_results.map { |r| serialize_case(r) }
-          }
-        end
-
-        def serialize_case(result)
-          {
-            name: result.name,
-            passed: result.passed?,
-            score: result.score,
-            details: result.details,
-            cost: result.cost
-          }
-        end
-
-        def format_cost(cost)
-          "$#{format("%.6f", cost)}"
+        def each(&block)
+          results.each(&block)
         end
       end
     end

data/lib/ruby_llm/contract/eval/report_presenter.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Formats eval reports for console and string output.
+      class ReportPresenter
+        def initialize(report:, stats:)
+          @report = report
+          @stats = stats
+        end
+
+        def summary
+          summary_parts.join(", ")
+        end
+
+        def to_s
+          ([summary] + @stats.failures.map { |result| format_failure(result) }).join("\n")
+        end
+
+        def print_summary(io = $stdout)
+          io.puts summary
+          io.puts
+          @report.results.each { |result| print_result(io, result) }
+        end
+
+        private
+
+        def summary_parts
+          parts = ["#{@report.dataset_name}: #{@stats.pass_rate} checks passed"]
+          parts << "#{@stats.skipped} skipped" if @stats.skipped.positive?
+          parts << format_cost(@stats.total_cost) if @stats.total_cost.positive?
+          parts
+        end
+
+        def format_failure(result)
+          line = "  FAIL  #{result.name}"
+          line += ": #{result.details}" if useful_details?(result.details)
+          line
+        end
+
+        def print_result(io, result)
+          io.puts "  #{result.label}  #{result.name}#{result_cost(result)}#{result_latency(result)}"
+          io.puts "        #{result.details}" if result.failed? && useful_details?(result.details)
+        end
+
+        def useful_details?(details)
+          details && !Report::GENERIC_DETAILS.include?(details)
+        end
+
+        def result_cost(result)
+          result.cost ? "  #{format_cost(result.cost)}" : ""
+        end
+
+        def result_latency(result)
+          result.duration_ms ? "  #{result.duration_ms}ms" : ""
+        end
+
+        def format_cost(cost)
+          "$#{format("%.6f", cost)}"
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/report_stats.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Computes aggregate metrics for an eval report.
+      class ReportStats
+        def initialize(results:)
+          @results = results
+        end
+
+        def score
+          return 0.0 if evaluated_results.empty?
+
+          evaluated_results.sum(&:score) / evaluated_results.length
+        end
+
+        def passed
+          evaluated_results.count(&:passed?)
+        end
+
+        def failed
+          evaluated_results.count(&:failed?)
+        end
+
+        def skipped
+          @results.count { |result| result.step_status == :skipped }
+        end
+
+        def failures
+          evaluated_results.select(&:failed?)
+        end
+
+        def pass_rate
+          "#{passed}/#{evaluated_results.length}"
+        end
+
+        def total_cost
+          @results.sum { |result| result.cost || 0.0 }
+        end
+
+        def avg_latency_ms
+          latencies = @results.filter_map(&:duration_ms)
+          return nil if latencies.empty?
+
+          latencies.sum.to_f / latencies.length
+        end
+
+        def passed?
+          return false if evaluated_results.empty?
+
+          evaluated_results.all?(&:passed?)
+        end
+
+        def evaluated_results
+          @evaluated_results ||= @results.reject { |result| result.step_status == :skipped }
+        end
+
+        def evaluated_results_count
+          evaluated_results.length
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/report_storage.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Persists eval reports as history entries and regression baselines.
+      class ReportStorage
+        def initialize(report:, stats:)
+          @report = report
+          @stats = stats
+        end
+
+        def save_history!(path: nil, model: nil)
+          file = path || storage_path(Report::HISTORY_DIR, "jsonl", model: model)
+          EvalHistory.append(file, history_entry)
+          file
+        end
+
+        def eval_history(path: nil, model: nil)
+          EvalHistory.load(path || storage_path(Report::HISTORY_DIR, "jsonl", model: model))
+        end
+
+        def save_baseline!(path: nil, model: nil)
+          file = path || storage_path(Report::BASELINE_DIR, "json", model: model)
+          FileUtils.mkdir_p(File.dirname(file))
+          File.write(file, JSON.pretty_generate(serialize_for_baseline))
+          file
+        end
+
+        def compare_with_baseline(path: nil, model: nil)
+          file = path || storage_path(Report::BASELINE_DIR, "json", model: model)
+          raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)
+
+          baseline_data = JSON.parse(File.read(file), symbolize_names: true)
+          validate_baseline!(baseline_data)
+
+          BaselineDiff.new(
+            baseline_cases: baseline_data[:cases],
+            current_cases: @report.results.map { |result| serialize_case(result) }
+          )
+        end
+
+        def baseline_exists?(path: nil, model: nil)
+          File.exist?(path || storage_path(Report::BASELINE_DIR, "json", model: model))
+        end
+
+        private
+
+        def history_entry
+          {
+            date: Time.now.strftime("%Y-%m-%d"),
+            score: @stats.score,
+            total_cost: @stats.total_cost,
+            pass_rate: @stats.pass_rate,
+            cases_count: @stats.evaluated_results_count
+          }
+        end
+
+        def serialize_for_baseline
+          {
+            dataset_name: @report.dataset_name,
+            step_name: @report.step_name,
+            score: @stats.score,
+            total_cost: @stats.total_cost,
+            cases: @stats.evaluated_results.map { |result| serialize_case(result) }
+          }
+        end
+
+        def serialize_case(result)
+          {
+            name: result.name,
+            passed: result.passed?,
+            score: result.score,
+            details: result.details,
+            cost: result.cost
+          }
+        end
+
+        def storage_path(root_dir, extension, model:)
+          parts = [root_dir]
+          parts << sanitize_name(@report.step_name) if @report.step_name
+
+          dataset_name = sanitize_name(@report.dataset_name)
+          dataset_name = "#{dataset_name}_#{sanitize_name(model)}" if model
+
+          File.join(*parts, "#{dataset_name}.#{extension}")
+        end
+
+        def validate_baseline!(data)
+          if data[:dataset_name] && data[:dataset_name] != @report.dataset_name
+            raise ArgumentError, "Baseline eval '#{data[:dataset_name]}' does not match '#{@report.dataset_name}'"
+          end
+          if data[:step_name] && @report.step_name && data[:step_name] != @report.step_name
+            raise ArgumentError, "Baseline step '#{data[:step_name]}' does not match '#{@report.step_name}'"
+          end
+        end
+
+        def sanitize_name(name)
+          name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
+        end
+      end
+    end
+  end
+end