eval-ruby 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 07e82b10ad871e882d8a5da7a3d85ef7436c3b1695840c2974b22a0df70ae0fc
+  data.tar.gz: f483be5c375db41ff367162257186dde52627aa4b9bba4d493dfa1455363d310
+SHA512:
+  metadata.gz: b4938e44301b2440500d6506057588fbafa5ca91f6cf574f288690132748ba81f2010c61ba76ad2f536a5d3cb7610442f1db854b08f0a8bf65d4bef7cf3b607c
+  data.tar.gz: f76fbe015937d962fb9747ff90a0fc245ae32fc47834644a894d4b69595dd31c30cd621922dca94386e8e737c6be82f726e4b76d039ebd38b8f73d6f548b5ce7

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    eval-ruby (0.1.0)
+      csv
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.9)
+      public_suffix (>= 2.0.2, < 8.0)
+    bigdecimal (4.0.1)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    csv (3.3.5)
+    hashdiff (1.2.1)
+    minitest (5.27.0)
+    public_suffix (7.0.5)
+    rake (13.3.1)
+    rexml (3.4.4)
+    webmock (3.26.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  arm64-darwin-24
+  ruby
+
+DEPENDENCIES
+  eval-ruby!
+  minitest (~> 5.0)
+  rake (~> 13.0)
+  webmock (~> 3.0)
+
+CHECKSUMS
+  addressable (2.8.9) sha256=cc154fcbe689711808a43601dee7b980238ce54368d23e127421753e46895485
+  bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
+  crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e
+  csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
+  eval-ruby (0.1.0)
+  hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1
+  minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5
+  public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623
+  rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
+  rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142
+  webmock (3.26.1) sha256=4f696fb57c90a827c20aadb2d4f9058bbff10f7f043bd0d4c3f58791143b1cd7
+
+BUNDLED WITH
+  4.0.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Johannes Dwi Cahyo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# eval-ruby
+
+Evaluation framework for LLM and RAG applications in Ruby. Measures quality metrics like faithfulness, relevance, context precision, and answer correctness.
+
+Think [Ragas](https://github.com/explodinggradients/ragas) or [DeepEval](https://github.com/confident-ai/deepeval) for Ruby.
+
+## Installation
+
+```ruby
+gem "eval-ruby"
+```
+
+## Quick Start
+
+```ruby
+require "eval_ruby"
+
+EvalRuby.configure do |config|
+  config.judge_llm = :openai  # or :anthropic
+  config.judge_model = "gpt-4o"
+  config.api_key = ENV["OPENAI_API_KEY"]
+end
+
+result = EvalRuby.evaluate(
+  question: "What is the capital of France?",
+  answer: "The capital of France is Paris.",
+  context: ["Paris is the capital of France."],
+  ground_truth: "Paris"
+)
+
+result.faithfulness      # => 0.95
+result.relevance         # => 0.92
+result.context_precision # => 0.85
+result.correctness       # => 0.98
+result.overall           # => 0.94
+```
+
+## Metrics
+
+### LLM-as-Judge
+- **Faithfulness** — Is the answer supported by the context?
+- **Relevance** — Does the answer address the question?
+- **Correctness** — Does the answer match the ground truth?
+- **Context Precision** — Are retrieved contexts relevant?
+- **Context Recall** — Do contexts cover the ground truth?
+
+### Retrieval Metrics
+- **Precision@K** / **Recall@K**
+- **MRR** (Mean Reciprocal Rank)
+- **NDCG** (Normalized Discounted Cumulative Gain)
+- **Hit Rate**
+
+## Retrieval Evaluation
+
+```ruby
+result = EvalRuby.evaluate_retrieval(
+  question: "What is Ruby?",
+  retrieved: ["Ruby is...", "Python is...", "Java is..."],
+  relevant: ["Ruby is..."]
+)
+
+result.precision_at_k(1)  # => 1.0
+result.mrr                # => 1.0
+result.ndcg               # => 0.63
+```
+
+## Batch Evaluation
+
+```ruby
+report = EvalRuby.evaluate_batch(dataset)
+report.summary
+report.worst(5)
+report.failures(threshold: 0.8)
+report.to_csv("results.csv")
+```
+
+## Test Integration
+
+### Minitest
+
+```ruby
+require "eval_ruby/minitest"
+
+class TestRAG < Minitest::Test
+  include EvalRuby::Assertions
+
+  def test_faithfulness
+    assert_faithful answer, context, threshold: 0.8
+  end
+
+  def test_no_hallucination
+    refute_hallucination answer, context
+  end
+end
+```
+
+### RSpec
+
+```ruby
+require "eval_ruby/rspec"
+
+RSpec.describe "RAG" do
+  include EvalRuby::RSpecMatchers
+
+  it "produces faithful answers" do
+    expect(answer).to be_faithful_to(context).with_threshold(0.8)
+  end
+end
+```
+
+## A/B Comparison
+
+```ruby
+comparison = EvalRuby.compare(report_a, report_b)
+comparison.summary
+comparison.significant_improvements  # => [:faithfulness, :context_precision]
+```
+
+## License
+
+MIT

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/eval-ruby.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "lib/eval_ruby/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "eval-ruby"
+  spec.version = EvalRuby::VERSION
+  spec.authors = ["Johannes Dwi Cahyo"]
+  spec.homepage = "https://github.com/johannesdwicahyo/eval-ruby"
+  spec.summary = "Evaluation framework for LLM and RAG applications in Ruby"
+  spec.description = "Measures quality metrics like faithfulness, relevance, context precision, " \
+                     "and answer correctness for LLM and RAG applications. " \
+                     "Think Ragas or DeepEval for Ruby."
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?("test/", "spec/", "examples/", ".git")
+    end
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "csv"
+
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "webmock", "~> 3.0"
+end

data/lib/eval_ruby/comparison.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  class Comparison
+    attr_reader :report_a, :report_b
+
+    def initialize(report_a, report_b)
+      @report_a = report_a
+      @report_b = report_b
+    end
+
+    def summary
+      lines = [
+        format("%-20s | %-10s | %-10s | %-8s | %s", "Metric", "A", "B", "Delta", "p-value"),
+        "-" * 70
+      ]
+
+      all_metrics.each do |metric|
+        stats_a = @report_a.metric_stats[metric]
+        stats_b = @report_b.metric_stats[metric]
+        next unless stats_a && stats_b
+
+        delta = stats_b[:mean] - stats_a[:mean]
+        scores_a = @report_a.results.filter_map { |r| r.scores[metric] }
+        scores_b = @report_b.results.filter_map { |r| r.scores[metric] }
+        t_result = paired_t_test(scores_a, scores_b)
+        sig = significance_marker(t_result[:p_value])
+
+        lines << format(
+          "%-20s | %-10.4f | %-10.4f | %+.4f  | %.4f %s",
+          metric, stats_a[:mean], stats_b[:mean], delta, t_result[:p_value], sig
+        )
+      end
+
+      lines.join("\n")
+    end
+
+    def significant_improvements(alpha: 0.05)
+      all_metrics.select do |metric|
+        scores_a = @report_a.results.filter_map { |r| r.scores[metric] }
+        scores_b = @report_b.results.filter_map { |r| r.scores[metric] }
+        next false if scores_a.empty? || scores_b.empty?
+
+        t_result = paired_t_test(scores_a, scores_b)
+        mean_b = scores_b.sum / scores_b.size.to_f
+        mean_a = scores_a.sum / scores_a.size.to_f
+        t_result[:p_value] < alpha && mean_b > mean_a
+      end
+    end
+
+    private
+
+    def all_metrics
+      (@report_a.metric_stats.keys + @report_b.metric_stats.keys).uniq
+    end
+
+    def paired_t_test(scores_a, scores_b)
+      n = [scores_a.length, scores_b.length].min
+      return {t_stat: 0.0, p_value: 1.0, significant: false} if n < 2
+
+      diffs = scores_a.first(n).zip(scores_b.first(n)).map { |a, b| a - b }
+      mean_diff = diffs.sum / n.to_f
+      std_diff = Math.sqrt(diffs.sum { |d| (d - mean_diff)**2 } / (n - 1).to_f)
+
+      return {t_stat: 0.0, p_value: 1.0, significant: false} if std_diff.zero?
+
+      t_stat = mean_diff / (std_diff / Math.sqrt(n))
+      p_value = 2 * (1 - normal_cdf(t_stat.abs))
+      {t_stat: t_stat, p_value: p_value, significant: p_value < 0.05}
+    end
+
+    def normal_cdf(x)
+      0.5 * (1 + Math.erf(x / Math.sqrt(2)))
+    end
+
+    def significance_marker(p_value)
+      if p_value < 0.001
+        "***"
+      elsif p_value < 0.01
+        "**"
+      elsif p_value < 0.05
+        "*"
+      else
+        ""
+      end
+    end
+  end
+end

data/lib/eval_ruby/configuration.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  class Configuration
+    attr_accessor :judge_llm, :judge_model, :api_key, :default_threshold,
+                  :timeout, :max_retries
+
+    def initialize
+      @judge_llm = :openai
+      @judge_model = "gpt-4o"
+      @api_key = nil
+      @default_threshold = 0.7
+      @timeout = 30
+      @max_retries = 3
+    end
+  end
+end

data/lib/eval_ruby/dataset.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "csv"
+require "json"
+
+module EvalRuby
+  class Dataset
+    include Enumerable
+
+    attr_reader :name, :samples
+
+    def initialize(name = "default")
+      @name = name
+      @samples = []
+    end
+
+    def add(question:, ground_truth: nil, relevant_contexts: [], answer: nil, context: [])
+      @samples << {
+        question: question,
+        answer: answer,
+        context: context.empty? ? relevant_contexts : context,
+        ground_truth: ground_truth
+      }
+      self
+    end
+
+    def each(&block)
+      @samples.each(&block)
+    end
+
+    def size
+      @samples.size
+    end
+
+    def [](index)
+      @samples[index]
+    end
+
+    def self.from_csv(path)
+      dataset = new(File.basename(path, ".*"))
+      CSV.foreach(path, headers: true) do |row|
+        dataset.add(
+          question: row["question"],
+          answer: row["answer"],
+          context: parse_array_field(row["context"]),
+          ground_truth: row["ground_truth"]
+        )
+      end
+      dataset
+    end
+
+    def self.from_json(path)
+      dataset = new(File.basename(path, ".*"))
+      data = JSON.parse(File.read(path))
+      samples = data.is_a?(Array) ? data : data["samples"] || data["data"] || []
+      samples.each do |sample|
+        dataset.add(
+          question: sample["question"],
+          answer: sample["answer"],
+          context: Array(sample["context"]),
+          ground_truth: sample["ground_truth"]
+        )
+      end
+      dataset
+    end
+
+    def to_csv(path)
+      CSV.open(path, "w") do |csv|
+        csv << %w[question answer context ground_truth]
+        @samples.each do |sample|
+          csv << [
+            sample[:question],
+            sample[:answer],
+            JSON.generate(sample[:context]),
+            sample[:ground_truth]
+          ]
+        end
+      end
+    end
+
+    def to_json(path)
+      File.write(path, JSON.pretty_generate({name: @name, samples: @samples}))
+    end
+
+    def self.generate(documents:, questions_per_doc: 5, llm: :openai)
+      config = EvalRuby.configuration.dup
+      config.judge_llm = llm
+      judge = case llm
+      when :openai then Judges::OpenAI.new(config)
+      when :anthropic then Judges::Anthropic.new(config)
+      else raise Error, "Unknown LLM: #{llm}"
+      end
+
+      dataset = new("generated")
+      documents.each do |doc_path|
+        content = File.read(doc_path)
+        prompt = <<~PROMPT
+          Given the following document, generate #{questions_per_doc} question-answer pairs
+          that can be answered using the document content.
+
+          Document:
+          #{content}
+
+          Respond in JSON: {"pairs": [{"question": "...", "answer": "...", "context": "relevant excerpt"}]}
+        PROMPT
+
+        result = judge.call(prompt)
+        next unless result&.key?("pairs")
+
+        result["pairs"].each do |pair|
+          dataset.add(
+            question: pair["question"],
+            answer: pair["answer"],
+            context: [pair["context"] || content],
+            ground_truth: pair["answer"]
+          )
+        end
+      end
+      dataset
+    end
+
+    private_class_method def self.parse_array_field(value)
+      return [] if value.nil? || value.empty?
+
+      JSON.parse(value)
+    rescue JSON::ParserError
+      [value]
+    end
+  end
+end

data/lib/eval_ruby/evaluator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  class Evaluator
+    def initialize(config = EvalRuby.configuration)
+      @config = config
+      @judge = build_judge(config)
+    end
+
+    def evaluate(question:, answer:, context: [], ground_truth: nil)
+      scores = {}
+      details = {}
+
+      # LLM-as-judge metrics
+      faith = Metrics::Faithfulness.new(judge: @judge).call(answer: answer, context: context)
+      scores[:faithfulness] = faith[:score]
+      details[:faithfulness] = faith[:details]
+
+      rel = Metrics::Relevance.new(judge: @judge).call(question: question, answer: answer)
+      scores[:relevance] = rel[:score]
+      details[:relevance] = rel[:details]
+
+      cp = Metrics::ContextPrecision.new(judge: @judge).call(question: question, context: context)
+      scores[:context_precision] = cp[:score]
+      details[:context_precision] = cp[:details]
+
+      if ground_truth
+        corr = Metrics::Correctness.new(judge: @judge).call(answer: answer, ground_truth: ground_truth)
+        scores[:correctness] = corr[:score]
+        details[:correctness] = corr[:details]
+
+        cr = Metrics::ContextRecall.new(judge: @judge).call(context: context, ground_truth: ground_truth)
+        scores[:context_recall] = cr[:score]
+        details[:context_recall] = cr[:details]
+      end
+
+      Result.new(scores: scores, details: details)
+    end
+
+    def evaluate_retrieval(question:, retrieved:, relevant:)
+      RetrievalResult.new(retrieved: retrieved, relevant: relevant)
+    end
+
+    private
+
+    def build_judge(config)
+      case config.judge_llm
+      when :openai
+        Judges::OpenAI.new(config)
+      when :anthropic
+        Judges::Anthropic.new(config)
+      else
+        raise Error, "Unknown judge LLM: #{config.judge_llm}"
+      end
+    end
+  end
+
+  class RetrievalResult
+    def initialize(retrieved:, relevant:)
+      @retrieved = retrieved
+      @relevant = relevant
+    end
+
+    def precision_at_k(k)
+      Metrics::PrecisionAtK.new.call(retrieved: @retrieved, relevant: @relevant, k: k)
+    end
+
+    def recall_at_k(k)
+      Metrics::RecallAtK.new.call(retrieved: @retrieved, relevant: @relevant, k: k)
+    end
+
+    def mrr
+      Metrics::MRR.new.call(retrieved: @retrieved, relevant: @relevant)
+    end
+
+    def ndcg(k: nil)
+      Metrics::NDCG.new.call(retrieved: @retrieved, relevant: @relevant, k: k)
+    end
+
+    def hit_rate
+      @retrieved.any? { |doc| @relevant.include?(doc) } ? 1.0 : 0.0
+    end
+
+    def to_h
+      {
+        precision_at_k: precision_at_k(@retrieved.length),
+        recall_at_k: recall_at_k(@retrieved.length),
+        mrr: mrr,
+        ndcg: ndcg,
+        hit_rate: hit_rate
+      }
+    end
+  end
+end

data/lib/eval_ruby/judges/anthropic.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module EvalRuby
+  module Judges
+    class Anthropic < Base
+      API_URL = "https://api.anthropic.com/v1/messages"
+
+      def call(prompt)
+        uri = URI(API_URL)
+        request = Net::HTTP::Post.new(uri)
+        request["x-api-key"] = @config.api_key
+        request["anthropic-version"] = "2023-06-01"
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate({
+          model: @config.judge_model,
+          max_tokens: 4096,
+          messages: [{role: "user", content: prompt}],
+          temperature: 0.0
+        })
+
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
+                                   read_timeout: @config.timeout) do |http|
+          http.request(request)
+        end
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise Error, "Anthropic API error: #{response.code} - #{response.body}"
+        end
+
+        body = JSON.parse(response.body)
+        content = body.dig("content", 0, "text")
+        parse_json_response(content)
+      end
+    end
+  end
+end

data/lib/eval_ruby/judges/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module EvalRuby
+  module Judges
+    class Base
+      def initialize(config)
+        @config = config
+      end
+
+      def call(prompt)
+        raise NotImplementedError, "#{self.class}#call must be implemented"
+      end
+
+      private
+
+      def parse_json_response(text)
+        match = text.match(/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/m)
+        return nil unless match
+
+        JSON.parse(match[0])
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/eval_ruby/judges/openai.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module EvalRuby
+  module Judges
+    class OpenAI < Base
+      API_URL = "https://api.openai.com/v1/chat/completions"
+
+      def call(prompt)
+        uri = URI(API_URL)
+        request = Net::HTTP::Post.new(uri)
+        request["Authorization"] = "Bearer #{@config.api_key}"
+        request["Content-Type"] = "application/json"
+        request.body = JSON.generate({
+          model: @config.judge_model,
+          messages: [{role: "user", content: prompt}],
+          temperature: 0.0,
+          response_format: {type: "json_object"}
+        })
+
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
+                                   read_timeout: @config.timeout) do |http|
+          http.request(request)
+        end
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise Error, "OpenAI API error: #{response.code} - #{response.body}"
+        end
+
+        body = JSON.parse(response.body)
+        content = body.dig("choices", 0, "message", "content")
+        parse_json_response(content)
+      end
+    end
+  end
+end