ruby_llm-contract 0.2.0

data/lib/ruby_llm/contract/eval/dataset.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class Dataset
+        attr_reader :name, :cases
+
+        def initialize(name = "unnamed", &block)
+          @name = name
+          @cases = []
+          instance_eval(&block) if block
+        end
+
+        def self.define(name = "unnamed", &)
+          new(name, &)
+        end
+
+        private
+
+        # DSL: define a test case
+        # dataset.case "name", input: {...}, expected: {...}
+        # dataset.case "name", input: {...}, expected_traits: {...}
+        # dataset.case "name", input: {...}, evaluator: proc
+        def add_case(name = nil, input:, expected: nil, expected_traits: nil, evaluator: nil)
+          @cases << Case.new(
+            name: name || "case_#{@cases.length + 1}",
+            input: input,
+            expected: expected,
+            expected_traits: expected_traits,
+            evaluator: evaluator
+          )
+        end
+
+        # Allow using `add_case` in DSL
+        public :add_case
+      end
+
+      class Case
+        attr_reader :name, :input, :expected, :expected_traits, :evaluator
+
+        def initialize(name:, input:, expected: nil, expected_traits: nil, evaluator: nil)
+          @name = name
+          @input = input
+          @expected = expected
+          @expected_traits = expected_traits
+          @evaluator = evaluator
+          freeze
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/eval_definition.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class EvalDefinition
+        attr_reader :name, :cases
+
+        def initialize(name, step_class: nil, &)
+          @name = name
+          @step_class = step_class
+          @default_input = nil
+          @sample_response = nil
+          @cases = []
+          instance_eval(&)
+        end
+
+        def default_input(input)
+          @default_input = input
+        end
+
+        def sample_response(response)
+          @sample_response = response
+          pre_validate_sample! if @step_class
+        end
+
+        def build_adapter
+          return nil unless @sample_response
+
+          Adapters::Test.new(response: @sample_response.is_a?(String) ? @sample_response : @sample_response.to_json)
+        end
+
+        def add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil)
+          case_input = input || @default_input
+          raise ArgumentError, "add_case requires input (set default_input or pass input:)" unless case_input
+
+          @cases << {
+            name: description,
+            input: case_input,
+            expected: expected,
+            expected_traits: expected_traits,
+            evaluator: evaluator
+          }
+        end
+
+        def verify(description, expected_or_proc = nil, input: nil, expect: nil)
+          if expected_or_proc && expect
+            raise ArgumentError, "verify accepts either a positional argument or expect: keyword, not both"
+          end
+
+          expected_or_proc = expect if expect
+          case_input = input || @default_input
+          validate_verify_args!(expected_or_proc, case_input)
+
+          evaluator = expected_or_proc.is_a?(::Proc) ? expected_or_proc : nil
+
+          @cases << {
+            name: description,
+            input: case_input,
+            expected: evaluator ? nil : expected_or_proc,
+            evaluator: evaluator
+          }
+        end
+
+        def build_dataset
+          eval_cases = effective_cases
+          eval_name = @name
+          Dataset.define(eval_name) do
+            eval_cases.each do |eval_case|
+              add_case(eval_case[:name], input: eval_case[:input], expected: eval_case[:expected],
+                                         expected_traits: eval_case[:expected_traits],
+                                         evaluator: eval_case[:evaluator])
+            end
+          end
+        end
+
+        private
+
+        def effective_cases
+          return @cases if @cases.any?
+          return [] unless @default_input
+
+          # Zero-verify: auto-add a contract check case
+          [{ name: "contract check", input: @default_input, expected: nil, evaluator: nil }]
+        end
+
+        def validate_verify_args!(expected_or_proc, case_input)
+          raise ArgumentError, "verify requires either a positional argument or expect: keyword" unless expected_or_proc
+          raise ArgumentError, "verify requires input (set default_input or pass input:)" unless case_input
+        end
+
+        def pre_validate_sample!
+          schema = @step_class.respond_to?(:output_schema) ? @step_class.output_schema : nil
+          return unless schema
+
+          errors = validate_sample_against_schema(schema)
+          return if errors.empty?
+
+          raise ArgumentError, "sample_response does not satisfy step schema: #{errors.join(", ")}"
+        rescue JSON::ParserError
+          # Not JSON -- skip pre-validation
+        end
+
+        def validate_sample_against_schema(schema)
+          response_hash = @sample_response.is_a?(Hash) ? @sample_response : JSON.parse(@sample_response.to_s)
+          symbolized = Parser.symbolize_keys(response_hash)
+          SchemaValidator.validate(symbolized, schema)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluation_result.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class EvaluationResult
+        attr_reader :score, :passed, :label, :details
+
+        def initialize(score:, passed:, label: nil, details: nil)
+          @score = score.to_f.clamp(0.0, 1.0)
+          @passed = passed
+          @label = label || (passed ? "PASS" : "FAIL")
+          @details = details
+          freeze
+        end
+
+        def failed?
+          !@passed
+        end
+
+        def to_s
+          "#{@label} (score: #{@score}#{" — #{@details}" if @details})"
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluator/exact.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      module Evaluator
+        class Exact
+          def call(output:, expected:, input: nil) # rubocop:disable Lint/UnusedMethodArgument
+            if output == expected
+              EvaluationResult.new(score: 1.0, passed: true, details: "exact match")
+            else
+              EvaluationResult.new(score: 0.0, passed: false,
+                                   details: "expected #{expected.inspect}, got #{output.inspect}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluator/json_includes.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      module Evaluator
+        class JsonIncludes
+          def call(output:, expected:, input: nil) # rubocop:disable Lint/UnusedMethodArgument
+            return type_error(output, expected) unless output.is_a?(Hash) && expected.is_a?(Hash)
+
+            errors = check_keys(output, expected)
+            build_result(errors, expected.length)
+          end
+
+          private
+
+          def check_keys(output, expected)
+            expected.each_with_object([]) do |(key, value), errors|
+              actual = output[key]
+              error = check_single_key(key, value, actual)
+              errors << error if error
+            end
+          end
+
+          def check_single_key(key, expected_value, actual)
+            if actual.nil?
+              "missing key: #{key}"
+            elsif expected_value.is_a?(::Regexp)
+              mismatch_message(key, expected_value, actual) unless actual.to_s.match?(expected_value)
+            elsif actual != expected_value
+              mismatch_message(key, expected_value, actual)
+            end
+          end
+
+          def mismatch_message(key, expected_value, actual)
+            "#{key}: expected #{expected_value.inspect}, got #{actual.inspect}"
+          end
+
+          def build_result(errors, total)
+            if errors.empty?
+              return EvaluationResult.new(score: 1.0, passed: true,
+                                          details: "all expected keys present and matching")
+            end
+
+            matched = total - errors.length
+            score = total.zero? ? 0.0 : matched.to_f / total
+            EvaluationResult.new(score: score, passed: false, details: errors.join("; "))
+          end
+
+          def type_error(output, expected)
+            EvaluationResult.new(score: 0.0, passed: false,
+                                 details: "expected Hash, got #{output.class} and #{expected.class}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      module Evaluator
+        class ProcEvaluator
+          def initialize(callable)
+            @callable = callable
+          end
+
+          def call(output:, expected: nil, input: nil) # rubocop:disable Lint/UnusedMethodArgument,Metrics
+            result = if @callable.arity == 2 || (@callable.arity.negative? && @callable.parameters.length >= 2)
+                       @callable.call(output, input)
+                     else
+                       @callable.call(output)
+                     end
+
+            if result.nil?
+              warn "[ruby_llm-contract] verify/evaluator proc returned nil. " \
+                   "This usually means a key mismatch (string vs symbol). " \
+                   "Output keys are always symbols."
+            end
+
+            case result
+            when true
+              EvaluationResult.new(score: 1.0, passed: true, details: "passed")
+            when false
+              EvaluationResult.new(score: 0.0, passed: false, details: "not passed")
+            when Numeric
+              EvaluationResult.new(score: result, passed: result >= 0.5, details: "custom score: #{result}")
+            else
+              EvaluationResult.new(score: result ? 1.0 : 0.0, passed: !!result, details: "custom: #{result}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluator/regex.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      module Evaluator
+        class Regex
+          def initialize(pattern)
+            @pattern = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern)
+          end
+
+          def call(output:, expected: nil, input: nil) # rubocop:disable Lint/UnusedMethodArgument
+            text = output.is_a?(Hash) ? output.values.join(" ") : output.to_s
+
+            if text.match?(@pattern)
+              EvaluationResult.new(score: 1.0, passed: true,
+                                   details: "matches #{@pattern.inspect}")
+            else
+              EvaluationResult.new(score: 0.0, passed: false,
+                                   details: "does not match #{@pattern.inspect}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/model_comparison.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class ModelComparison
+        attr_reader :eval_name, :reports
+
+        def initialize(eval_name:, reports:)
+          @eval_name = eval_name
+          @reports = reports.freeze # { "model_name" => Report }
+          freeze
+        end
+
+        def models
+          @reports.keys
+        end
+
+        def score_for(model)
+          @reports[model]&.score
+        end
+
+        def cost_for(model)
+          @reports[model]&.total_cost
+        end
+
+        def best_for(min_score: 0.0)
+          eligible = @reports.select { |_, report| report.score > 0.0 && report.score >= min_score }
+          return nil if eligible.empty?
+
+          eligible.min_by { |_, report| report.total_cost }&.first
+        end
+
+        def cost_per_point
+          @reports.transform_values do |report|
+            report.score.positive? ? report.total_cost / report.score : Float::INFINITY
+          end
+        end
+
+        def table
+          lines = ["  Model                      Score       Cost  Avg Latency"]
+          lines << "  #{"-" * 57}"
+
+          @reports.each do |model, report|
+            latency = report.avg_latency_ms ? "#{report.avg_latency_ms.round}ms" : "n/a"
+            cost = report.total_cost.positive? ? "$#{format("%.4f", report.total_cost)}" : "n/a"
+            lines << format("  %-25s %6.2f %10s %12s", model, report.score, cost, latency)
+          end
+
+          lines.join("\n")
+        end
+
+        def print_summary(io = $stdout)
+          io.puts "#{@eval_name} — model comparison"
+          io.puts
+          io.puts table
+          io.puts
+
+          best = best_for(min_score: 0.0)
+          io.puts "  Best overall: #{best}" if best
+
+          cheapest_passing = best_for(min_score: 1.0)
+          io.puts "  Cheapest at 100%: #{cheapest_passing}" if cheapest_passing
+        end
+
+        def to_h
+          @reports.transform_values do |report|
+            {
+              score: report.score,
+              total_cost: report.total_cost,
+              avg_latency_ms: report.avg_latency_ms,
+              pass_rate: report.pass_rate,
+              passed: report.passed?
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/pipeline_result_adapter.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Lightweight adapter that wraps a Pipeline::Result to look like a Step::Result.
+      # Replaces OpenStruct usage in Runner#normalize_pipeline_result.
+      PipelineResultAdapter = Struct.new(:status, :ok_flag, :parsed_output, :validation_errors, :trace) do
+        def ok?
+          ok_flag
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/report.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class Report
+        attr_reader :dataset_name, :results
+
+        def initialize(dataset_name:, results:)
+          @dataset_name = dataset_name
+          @results = results.freeze
+          freeze
+        end
+
+        def score
+          evaluated = evaluated_results
+          return 0.0 if evaluated.empty?
+
+          evaluated.sum(&:score) / evaluated.length
+        end
+
+        def passed
+          evaluated_results.count(&:passed?)
+        end
+
+        def failed
+          evaluated_results.count(&:failed?)
+        end
+
+        def skipped
+          results.count { |r| r.step_status == :skipped }
+        end
+
+        def failures
+          evaluated_results.select(&:failed?)
+        end
+
+        def pass_rate
+          "#{passed}/#{evaluated_results.length}"
+        end
+
+        def total_cost
+          results.sum { |r| r.cost || 0.0 }
+        end
+
+        def avg_latency_ms
+          latencies = results.filter_map(&:duration_ms)
+          return nil if latencies.empty?
+
+          latencies.sum.to_f / latencies.length
+        end
+
+        def passed?
+          evaluated = evaluated_results
+          return false if evaluated.empty?
+
+          evaluated.all?(&:passed?)
+        end
+
+        def each(&)
+          results.each(&)
+        end
+
+        def summary
+          parts = ["#{dataset_name}: #{pass_rate} checks passed"]
+          parts << "#{skipped} skipped" if skipped.positive?
+          parts << format_cost(total_cost) if total_cost.positive?
+          parts.join(", ")
+        end
+
+        GENERIC_DETAILS = ["passed", "not passed"].freeze
+
+        def to_s
+          lines = [summary]
+          failures.each do |result|
+            lines << format_failure(result)
+          end
+          lines.join("\n")
+        end
+
+        def print_summary(io = $stdout)
+          io.puts summary
+          io.puts
+          results.each do |result|
+            icon = result.label
+            cost_str = result.cost ? "  #{format_cost(result.cost)}" : ""
+            latency_str = result.duration_ms ? "  #{result.duration_ms}ms" : ""
+            io.puts "  #{icon}  #{result.name}#{cost_str}#{latency_str}"
+            io.puts "        #{result.details}" if result.failed? && useful_details?(result.details)
+          end
+        end
+
+        private
+
+        def format_failure(result)
+          line = "  FAIL  #{result.name}"
+          line += ": #{result.details}" if useful_details?(result.details)
+          line
+        end
+
+        def useful_details?(details)
+          details && !GENERIC_DETAILS.include?(details)
+        end
+
+        def evaluated_results
+          results.reject { |r| r.step_status == :skipped }
+        end
+
+        def format_cost(cost)
+          "$#{format("%.6f", cost)}"
+        end
+      end
+    end
+  end
+end