eval-ruby 0.1.0

data/lib/eval_ruby/metrics/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class Base
+      attr_reader :judge
+
+      def initialize(judge: nil)
+        @judge = judge
+      end
+
+      def call(**kwargs)
+        raise NotImplementedError, "#{self.class}#call must be implemented"
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/context_precision.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class ContextPrecision < Base
+      PROMPT_TEMPLATE = <<~PROMPT
+        Given the following question and a list of retrieved contexts, evaluate
+        whether each context is relevant to answering the question.
+
+        Question:
+        %{question}
+
+        Contexts:
+        %{contexts}
+
+        For each context, determine if it is RELEVANT or NOT RELEVANT to answering the question.
+
+        Respond in JSON: {"evaluations": [{"index": 0, "relevant": true}], "score": 0.0}
+        The score should be the proportion of relevant contexts (0.0 to 1.0).
+      PROMPT
+
+      def call(question:, context:, **_kwargs)
+        contexts = Array(context)
+        return {score: 0.0, details: {}} if contexts.empty?
+
+        contexts_text = contexts.each_with_index.map { |c, i| "[#{i}] #{c}" }.join("\n\n")
+        prompt = format(PROMPT_TEMPLATE, question: question, contexts: contexts_text)
+
+        result = judge.call(prompt)
+        raise Error, "Judge returned invalid response for context_precision" unless result&.key?("score")
+
+        {
+          score: result["score"].to_f.clamp(0.0, 1.0),
+          details: {evaluations: result["evaluations"] || []}
+        }
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/context_recall.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class ContextRecall < Base
+      PROMPT_TEMPLATE = <<~PROMPT
+        Given the following ground truth answer and retrieved contexts, evaluate
+        whether the contexts contain enough information to support the ground truth.
+
+        Ground Truth:
+        %{ground_truth}
+
+        Contexts:
+        %{contexts}
+
+        For each statement in the ground truth, determine if it can be attributed
+        to the retrieved contexts.
+
+        Respond in JSON: {"statements": [{"statement": "...", "attributed": true}], "score": 0.0}
+        The score should be the proportion of statements attributed to context (0.0 to 1.0).
+      PROMPT
+
+      def call(context:, ground_truth:, **_kwargs)
+        contexts = Array(context)
+        return {score: 0.0, details: {}} if contexts.empty?
+
+        contexts_text = contexts.each_with_index.map { |c, i| "[#{i}] #{c}" }.join("\n\n")
+        prompt = format(PROMPT_TEMPLATE, ground_truth: ground_truth, contexts: contexts_text)
+
+        result = judge.call(prompt)
+        raise Error, "Judge returned invalid response for context_recall" unless result&.key?("score")
+
+        {
+          score: result["score"].to_f.clamp(0.0, 1.0),
+          details: {statements: result["statements"] || []}
+        }
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/correctness.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class Correctness < Base
+      PROMPT_TEMPLATE = <<~PROMPT
+        Given the following answer and ground truth, evaluate whether the answer
+        is factually correct.
+
+        Answer:
+        %{answer}
+
+        Ground Truth:
+        %{ground_truth}
+
+        Evaluate correctness on a scale from 0.0 to 1.0 where:
+        - 1.0 = the answer is completely correct and matches the ground truth
+        - 0.5 = the answer is partially correct
+        - 0.0 = the answer is completely wrong
+
+        Consider both semantic meaning and factual accuracy, not just exact string matching.
+
+        Respond in JSON: {"reasoning": "...", "score": 0.0}
+      PROMPT
+
+      def call(answer:, ground_truth:, **_kwargs)
+        if judge
+          llm_score(answer, ground_truth)
+        else
+          string_similarity_score(answer, ground_truth)
+        end
+      end
+
+      private
+
+      def llm_score(answer, ground_truth)
+        prompt = format(PROMPT_TEMPLATE, answer: answer, ground_truth: ground_truth)
+
+        result = judge.call(prompt)
+        raise Error, "Judge returned invalid response for correctness" unless result&.key?("score")
+
+        {
+          score: result["score"].to_f.clamp(0.0, 1.0),
+          details: {reasoning: result["reasoning"]}
+        }
+      end
+
+      def string_similarity_score(answer, ground_truth)
+        answer_tokens = tokenize(answer)
+        truth_tokens = tokenize(ground_truth)
+
+        return {score: 1.0, details: {method: :exact_match}} if answer_tokens == truth_tokens
+        return {score: 0.0, details: {method: :token_overlap}} if answer_tokens.empty? || truth_tokens.empty?
+
+        overlap = (answer_tokens & truth_tokens).size
+        precision = overlap.to_f / answer_tokens.size
+        recall = overlap.to_f / truth_tokens.size
+        f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0.0
+
+        {score: f1.clamp(0.0, 1.0), details: {method: :token_overlap, precision: precision, recall: recall}}
+      end
+
+      def tokenize(text)
+        text.downcase.scan(/\w+/)
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/faithfulness.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class Faithfulness < Base
+      PROMPT_TEMPLATE = <<~PROMPT
+        Given the following context and answer, evaluate whether the answer
+        is faithful to (supported by) the context.
+
+        For each claim in the answer, determine if it is:
+        1. SUPPORTED - directly supported by the context
+        2. NOT SUPPORTED - contradicts or is not mentioned in the context
+
+        Context:
+        %{context}
+
+        Answer:
+        %{answer}
+
+        List each claim and whether it is SUPPORTED or NOT SUPPORTED.
+        Then give a faithfulness score from 0.0 to 1.0 where:
+        - 1.0 = all claims are supported
+        - 0.0 = no claims are supported
+
+        Respond in JSON: {"claims": [{"claim": "...", "supported": true}], "score": 0.0}
+      PROMPT
+
+      def call(answer:, context:, **_kwargs)
+        context_text = Array(context).join("\n\n")
+        prompt = format(PROMPT_TEMPLATE, context: context_text, answer: answer)
+
+        result = judge.call(prompt)
+        raise Error, "Judge returned invalid response for faithfulness" unless result&.key?("score")
+
+        {
+          score: result["score"].to_f.clamp(0.0, 1.0),
+          details: {claims: result["claims"] || []}
+        }
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/mrr.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class MRR < Base
+      def call(retrieved:, relevant:, **_kwargs)
+        retrieved.each_with_index do |doc, i|
+          return 1.0 / (i + 1) if relevant.include?(doc)
+        end
+        0.0
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/ndcg.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class NDCG < Base
+      def call(retrieved:, relevant:, k: nil, **_kwargs)
+        k ||= retrieved.length
+        top_k = retrieved.first(k)
+
+        dcg = top_k.each_with_index.sum do |doc, i|
+          rel = relevant.include?(doc) ? 1.0 : 0.0
+          rel / Math.log2(i + 2)
+        end
+
+        ideal_length = [relevant.length, k].min
+        idcg = ideal_length.times.sum { |i| 1.0 / Math.log2(i + 2) }
+
+        idcg.zero? ? 0.0 : dcg / idcg
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/precision_at_k.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class PrecisionAtK < Base
+      def call(retrieved:, relevant:, k: nil, **_kwargs)
+        k ||= retrieved.length
+        top_k = retrieved.first(k)
+        return 0.0 if top_k.empty?
+
+        hits = top_k.count { |doc| relevant.include?(doc) }
+        hits.to_f / top_k.size
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/recall_at_k.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class RecallAtK < Base
+      def call(retrieved:, relevant:, k: nil, **_kwargs)
+        return 0.0 if relevant.empty?
+
+        k ||= retrieved.length
+        top_k = retrieved.first(k)
+        hits = top_k.count { |doc| relevant.include?(doc) }
+        hits.to_f / relevant.size
+      end
+    end
+  end
+end

data/lib/eval_ruby/metrics/relevance.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Metrics
+    class Relevance < Base
+      PROMPT_TEMPLATE = <<~PROMPT
+        Given the following question and answer, evaluate whether the answer
+        is relevant to and addresses the question.
+
+        Question:
+        %{question}
+
+        Answer:
+        %{answer}
+
+        Evaluate relevance on a scale from 0.0 to 1.0 where:
+        - 1.0 = the answer fully and directly addresses the question
+        - 0.5 = the answer partially addresses the question
+        - 0.0 = the answer is completely irrelevant to the question
+
+        Respond in JSON: {"reasoning": "...", "score": 0.0}
+      PROMPT
+
+      def call(question:, answer:, **_kwargs)
+        prompt = format(PROMPT_TEMPLATE, question: question, answer: answer)
+
+        result = judge.call(prompt)
+        raise Error, "Judge returned invalid response for relevance" unless result&.key?("score")
+
+        {
+          score: result["score"].to_f.clamp(0.0, 1.0),
+          details: {reasoning: result["reasoning"]}
+        }
+      end
+    end
+  end
+end

data/lib/eval_ruby/minitest.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "eval_ruby"
+
+module EvalRuby
+  module Assertions
+    def assert_faithful(answer, context, threshold: 0.8, message: nil)
+      result = eval_metric(:faithfulness, answer: answer, context: Array(context))
+      msg = message || "Expected faithfulness >= #{threshold}, got #{result[:score].round(4)}"
+      assert result[:score] >= threshold, msg
+    end
+
+    def assert_relevant(question, answer, threshold: 0.8, message: nil)
+      result = eval_metric(:relevance, question: question, answer: answer)
+      msg = message || "Expected relevance >= #{threshold}, got #{result[:score].round(4)}"
+      assert result[:score] >= threshold, msg
+    end
+
+    def assert_correct(answer, ground_truth:, threshold: 0.7, message: nil)
+      result = eval_metric(:correctness, answer: answer, ground_truth: ground_truth)
+      msg = message || "Expected correctness >= #{threshold}, got #{result[:score].round(4)}"
+      assert result[:score] >= threshold, msg
+    end
+
+    def assert_precision_at_k(retrieved, relevant, k:, threshold: 0.5, message: nil)
+      score = Metrics::PrecisionAtK.new.call(retrieved: retrieved, relevant: relevant, k: k)
+      msg = message || "Expected precision@#{k} >= #{threshold}, got #{score.round(4)}"
+      assert score >= threshold, msg
+    end
+
+    def refute_hallucination(answer, context, threshold: 0.8, message: nil)
+      result = eval_metric(:faithfulness, answer: answer, context: Array(context))
+      msg = message || "Expected no hallucination (faithfulness >= #{threshold}), got #{result[:score].round(4)}"
+      assert result[:score] >= threshold, msg
+    end
+
+    private
+
+    def eval_metric(metric_name, **kwargs)
+      judge = EvalRuby.send(:build_judge)
+      metric_class = case metric_name
+      when :faithfulness then Metrics::Faithfulness
+      when :relevance then Metrics::Relevance
+      when :correctness then Metrics::Correctness
+      when :context_precision then Metrics::ContextPrecision
+      when :context_recall then Metrics::ContextRecall
+      end
+      metric_class.new(judge: judge).call(**kwargs)
+    end
+  end
+end

data/lib/eval_ruby/report.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "csv"
+require "json"
+
+module EvalRuby
+  class Report
+    attr_reader :results, :duration, :samples
+
+    def initialize(results:, samples: [], duration: nil)
+      @results = results
+      @samples = samples
+      @duration = duration
+    end
+
+    def summary
+      lines = []
+      metric_stats.each do |metric, stats|
+        lines << format("%-20s %.4f (+/- %.4f)", "#{metric}:", stats[:mean], stats[:std])
+      end
+      lines << ""
+      lines << "Total: #{@results.size} samples | Duration: #{format_duration}"
+      lines.join("\n")
+    end
+
+    def metric_stats
+      return {} if @results.empty?
+
+      all_metrics = @results.flat_map { |r| r.scores.keys }.uniq
+      all_metrics.each_with_object({}) do |metric, hash|
+        values = @results.filter_map { |r| r.scores[metric] }
+        next if values.empty?
+
+        mean = values.sum / values.size.to_f
+        variance = values.sum { |v| (v - mean)**2 } / values.size.to_f
+        std = Math.sqrt(variance)
+        hash[metric] = {mean: mean, std: std, min: values.min, max: values.max, count: values.size}
+      end
+    end
+
+    def worst(n = 5)
+      @results.sort_by { |r| r.overall || 0.0 }.first(n)
+    end
+
+    def failures(threshold: nil)
+      threshold ||= EvalRuby.configuration.default_threshold
+      @results.select { |r| (r.overall || 0.0) < threshold }
+    end
+
+    def to_csv(path)
+      return if @results.empty?
+
+      all_metrics = @results.flat_map { |r| r.scores.keys }.uniq
+      CSV.open(path, "w") do |csv|
+        csv << ["sample_index"] + all_metrics.map(&:to_s) + ["overall"]
+        @results.each_with_index do |result, i|
+          row = [i] + all_metrics.map { |m| result.scores[m]&.round(4) } + [result.overall&.round(4)]
+          csv << row
+        end
+      end
+    end
+
+    def to_json(path)
+      data = @results.each_with_index.map do |result, i|
+        {index: i, scores: result.scores, overall: result.overall, sample: @samples[i]}
+      end
+      File.write(path, JSON.pretty_generate({results: data, summary: metric_stats}))
+    end
+
+    private
+
+    def format_duration
+      return "N/A" unless @duration
+
+      if @duration < 60
+        "#{@duration.round(1)}s"
+      else
+        "#{(@duration / 60).floor}m #{(@duration % 60).round(1)}s"
+      end
+    end
+  end
+end

data/lib/eval_ruby/result.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  class Result
+    METRICS = %i[faithfulness relevance correctness context_precision context_recall].freeze
+
+    attr_reader :scores, :details
+
+    def initialize(scores: {}, details: {})
+      @scores = scores
+      @details = details
+    end
+
+    METRICS.each do |metric|
+      define_method(metric) { @scores[metric] }
+    end
+
+    def overall(weights: nil)
+      weights ||= METRICS.each_with_object({}) { |m, h| h[m] = 1.0 }
+      available = @scores.select { |k, v| weights.key?(k) && v }
+      return nil if available.empty?
+
+      total_weight = available.sum { |k, _| weights[k] }
+      available.sum { |k, v| v * weights[k] } / total_weight
+    end
+
+    def to_h
+      @scores.merge(overall: overall)
+    end
+
+    def to_s
+      lines = @scores.map { |k, v| "  #{k}: #{v&.round(4) || 'N/A'}" }
+      lines << "  overall: #{overall&.round(4) || 'N/A'}"
+      "EvalRuby::Result\n#{lines.join("\n")}"
+    end
+  end
+end

data/lib/eval_ruby/rspec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "eval_ruby"
+
+module EvalRuby
+  module RSpecMatchers
+    class BeFaithfulTo
+      def initialize(context)
+        @context = Array(context)
+        @threshold = 0.8
+      end
+
+      def with_threshold(threshold)
+        @threshold = threshold
+        self
+      end
+
+      def matches?(answer)
+        @answer = answer
+        judge = EvalRuby.send(:build_judge)
+        result = Metrics::Faithfulness.new(judge: judge).call(answer: answer, context: @context)
+        @score = result[:score]
+        @score >= @threshold
+      end
+
+      def failure_message
+        "expected answer to be faithful to context (threshold: #{@threshold}), but got score #{@score.round(4)}"
+      end
+
+      def failure_message_when_negated
+        "expected answer not to be faithful to context, but got score #{@score.round(4)}"
+      end
+    end
+
+    class HavePrecisionAtK
+      def initialize(k)
+        @k = k
+        @threshold = 0.5
+      end
+
+      def above(threshold)
+        @threshold = threshold
+        self
+      end
+
+      def matches?(results)
+        @results = results
+        # results should respond to retrieved and relevant, or be arrays
+        if results.is_a?(EvalRuby::RetrievalResult)
+          @score = results.precision_at_k(@k)
+        else
+          raise ArgumentError, "Expected EvalRuby::RetrievalResult or use assert_precision_at_k"
+        end
+        @score >= @threshold
+      end
+
+      def failure_message
+        "expected precision@#{@k} >= #{@threshold}, but got #{@score.round(4)}"
+      end
+    end
+
+    def be_faithful_to(context)
+      BeFaithfulTo.new(context)
+    end
+
+    def have_precision_at_k(k)
+      HavePrecisionAtK.new(k)
+    end
+  end
+end

data/lib/eval_ruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  VERSION = "0.1.0"
+end

data/lib/eval_ruby.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative "eval_ruby/version"
+require_relative "eval_ruby/configuration"
+require_relative "eval_ruby/judges/base"
+require_relative "eval_ruby/judges/openai"
+require_relative "eval_ruby/judges/anthropic"
+require_relative "eval_ruby/metrics/base"
+require_relative "eval_ruby/metrics/faithfulness"
+require_relative "eval_ruby/metrics/relevance"
+require_relative "eval_ruby/metrics/correctness"
+require_relative "eval_ruby/metrics/context_precision"
+require_relative "eval_ruby/metrics/context_recall"
+require_relative "eval_ruby/metrics/precision_at_k"
+require_relative "eval_ruby/metrics/recall_at_k"
+require_relative "eval_ruby/metrics/mrr"
+require_relative "eval_ruby/metrics/ndcg"
+require_relative "eval_ruby/result"
+require_relative "eval_ruby/evaluator"
+require_relative "eval_ruby/report"
+require_relative "eval_ruby/dataset"
+require_relative "eval_ruby/comparison"
+
+module EvalRuby
+  class Error < StandardError; end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    def evaluate(question:, answer:, context: [], ground_truth: nil)
+      Evaluator.new.evaluate(
+        question: question,
+        answer: answer,
+        context: context,
+        ground_truth: ground_truth
+      )
+    end
+
+    def evaluate_retrieval(question:, retrieved:, relevant:)
+      Evaluator.new.evaluate_retrieval(
+        question: question,
+        retrieved: retrieved,
+        relevant: relevant
+      )
+    end
+
+    def evaluate_batch(dataset, pipeline: nil)
+      samples = dataset.is_a?(Dataset) ? dataset.samples : dataset
+      evaluator = Evaluator.new
+      start_time = Time.now
+
+      results = samples.map do |sample|
+        if pipeline
+          response = pipeline.query(sample[:question])
+          evaluator.evaluate(
+            question: sample[:question],
+            answer: response.respond_to?(:text) ? response.text : response.to_s,
+            context: response.respond_to?(:context) ? response.context : sample[:context],
+            ground_truth: sample[:ground_truth]
+          )
+        else
+          evaluator.evaluate(**sample.slice(:question, :answer, :context, :ground_truth))
+        end
+      end
+
+      Report.new(results: results, samples: samples, duration: Time.now - start_time)
+    end
+
+    def compare(report_a, report_b)
+      Comparison.new(report_a, report_b)
+    end
+
+    private
+
+    def build_judge
+      config = configuration
+      case config.judge_llm
+      when :openai then Judges::OpenAI.new(config)
+      when :anthropic then Judges::Anthropic.new(config)
+      else raise Error, "Unknown judge LLM: #{config.judge_llm}"
+      end
+    end
+  end
+end