rubric_llm 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52a6704521634882ccc1d1779c20e2b65587f398b2cfd00b3c7892d9a97706e1
-  data.tar.gz: 3410606f13f439af21457ac9714b0e67a33752a3231854ccd0f73e759d4ac209
+  metadata.gz: 6ed5a9f9ece0c78f86b4134211e39d8100e046046686ec296697d794cefcce15
+  data.tar.gz: 40e120b95e7d8b5711e495f909e357395fd5ffbae76cb49dd8aa6c5212488196
 SHA512:
-  metadata.gz: 17a9e4d4308627889576b29fbbdb91a93a12e702847780e21fe070cb499ac06d243fa5e688d520e4fd43827eb05ba86b85eafb039598160e31e273838ab69f1c
-  data.tar.gz: 86b21112726b41c2eef29cc2e0fc0f6b6b4d742a6c2797d46f391d0292fb2ad8477d3792029d1b90f562425118f52df496e998f6d434128c4b4017756aaa217d
+  metadata.gz: e07a90dabde33a66007ce21bd2173c577ce61fb1a964ce6946f2df26d03ff29dac18bc5757d209fcf5f8d76b5e8bb72c33562f250ee2335adc9aa1c83b0aada1
+  data.tar.gz: 8fcf6d081679021ac44aa729f63e243aabe344aa93c730a5b660d4fcc981963a08f7017276416b68b7b8b2d390756c537469263cdad92772e5f23a66199f1b0e

data/CHANGELOG.md

@@ -5,7 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.2.0] - 2026-07-11
+
+### Changed
+
+- Enforce schema-backed judge responses through RubyLLM structured output when supported
+- Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
+- Require `ruby_llm ~> 1.13` for named schema payload support in structured output
+- Record judge failures per metric in `RubricLLM.evaluate` and `RubricLLM.evaluate_batch` as a `nil` score with the error message in details and continue the run, while non-judge errors propagate
+- Remove dead score clamping and nil-response guards from LLM metrics now that the judge validates the response contract
+
+## [0.1.2] - 2026-04-30
+
+### Added
+
+- Add standalone LLM-as-Judge examples for RAG scoring, batch reports, model comparison, custom metrics, and Minitest assertions
 
 ## [0.1.1] - 2026-03-24
 

data/README.md

@@ -1,9 +1,10 @@
 # RubricLLM
 
-Lightweight LLM evaluation framework for Ruby, inspired by [Ragas](https://github.com/vibrantlabsai/ragas), powered by [RubyLLM](https://github.com/crmne/ruby_llm).
+Lightweight LLM evaluation framework for Ruby, inspired by [DeepEval](https://github.com/confident-ai/deepeval), powered by [RubyLLM](https://github.com/crmne/ruby_llm).
 
 [![Gem Version](https://badge.fury.io/rb/rubric_llm.svg)](https://badge.fury.io/rb/rubric_llm)
 [![CI](https://github.com/dpaluy/rubric_llm/actions/workflows/ci.yml/badge.svg)](https://github.com/dpaluy/rubric_llm/actions/workflows/ci.yml)
+[![Wiki](https://img.shields.io/badge/docs-wiki-blue)](https://github.com/dpaluy/rubric_llm/wiki)
 
 Provider-agnostic evaluation with pluggable metrics, statistical A/B comparison, and test framework integration — no Rails, no ActiveRecord, no UI. Works anywhere Ruby runs.
 
@@ -27,31 +28,34 @@ gem install rubric_llm
 require "rubric_llm"
 
 RubricLLM.configure do |c|
-  c.judge_model = "gpt-4o"
+  c.judge_model = "gpt-5.5"
   c.judge_provider = :openai
 end
 
 result = RubricLLM.evaluate(
-  question: "What is the capital of France?",
-  answer: "The capital of France is Paris, located on the Seine river.",
-  context: ["Paris is the capital and largest city of France."],
-  ground_truth: "Paris"
+  question: "What is Ruby's core design philosophy?",
+  answer: "Ruby was designed by Yukihiro Matsumoto to optimize for developer happiness and productivity, prioritizing the programmer's joy over machine efficiency.",
+  context: [
+    "Yukihiro Matsumoto, Ruby's creator, has stated that Ruby is designed to make programmers happy. " \
+    "He optimized the language for human readability and developer productivity rather than raw machine performance."
+  ],
+  ground_truth: "Ruby is designed to maximize developer happiness and productivity."
 )
 
-result.faithfulness      # => 0.95
-result.relevance         # => 0.92
-result.correctness       # => 0.98
-result.overall           # => 0.94
+result.correctness       # => 0.99
 result.pass?             # => true
 ```
 
+For runnable LLM-as-Judge examples, including RAG scoring, batch pass/fail output, model comparison, custom metrics, and live
+Minitest assertions, see [examples/README.md](examples/README.md).
+
 ## Configuration
 
 ### Global
 
 ```ruby
 RubricLLM.configure do |c|
-  c.judge_model = "gpt-4o"           # any model RubyLLM supports
+  c.judge_model = "gpt-5.5"           # any model RubyLLM supports
   c.judge_provider = :openai          # :openai, :anthropic, :gemini, etc.
   c.temperature = 0.0                 # deterministic scoring (default)
   c.max_tokens = 4096                 # max tokens for judge response
@@ -80,7 +84,7 @@ config = RubricLLM::Config.from_env
 ### Per-Evaluation Override
 
 ```ruby
-custom = RubricLLM::Config.new(judge_model: "claude-haiku-4-5", judge_provider: :anthropic)
+custom = RubricLLM::Config.new(judge_model: "claude-opus-4-5", judge_provider: :anthropic)
 
 result = RubricLLM.evaluate(question: "...", answer: "...", config: custom)
 report = RubricLLM.evaluate_batch(dataset, config: custom)
@@ -91,7 +95,7 @@ report = RubricLLM.evaluate_batch(dataset, config: custom)
 ```ruby
 # config/initializers/rubric_llm.rb
 RubricLLM.configure do |c|
-  c.judge_model = "gpt-4o"
+  c.judge_model = "gpt-5.5"
   c.judge_provider = :openai
 end
 ```
@@ -104,12 +108,12 @@ These metrics use a judge LLM to evaluate quality. Each sends a structured promp
 
 | Metric | Question it answers | Requires |
 |--------|-------------------|----------|
-| **Faithfulness** | Is every claim in the answer supported by the context? | `context` |
-| **Relevance** | Does the answer address what was asked? | `question` |
 | **Correctness** | Does the answer match the known correct answer? | `ground_truth` |
+| **Relevance** | Does the answer address what was asked? | `question` |
 | **Context Precision** | Are the retrieved context chunks actually relevant? | `question`, `context` |
-| **Context Recall** | Do the contexts cover the information in the ground truth? | `context`, `ground_truth` |
 | **Factual Accuracy** | Are there factual discrepancies between candidate and reference? | `ground_truth` |
+| **Context Recall** | Do the contexts cover the information in the ground truth? | `context`, `ground_truth` |
+| **Faithfulness** | Is every claim in the answer supported by the context? | `context` |
 
 ```ruby
 # Only context — gets faithfulness, relevance, context_precision
@@ -160,8 +164,8 @@ result = RubricLLM.evaluate_retrieval(
 )
 
 result.precision_at_k(3)  # => 0.67
-result.recall_at_k(3)     # => 1.0
-result.mrr                # => 1.0
+result.recall_at_k(3)     # => 0.90
+result.mrr                # => 0.90
 result.ndcg               # => 0.86
 result.hit_rate           # => 1.0
 ```
@@ -203,7 +207,7 @@ report.to_json                        # returns JSON string
 Compare two models with statistical significance testing:
 
 ```ruby
-config_a = RubricLLM::Config.new(judge_model: "gpt-4o")
+config_a = RubricLLM::Config.new(judge_model: "gpt-5.5")
 config_b = RubricLLM::Config.new(judge_model: "claude-sonnet-4-6")
 
 report_a = RubricLLM.evaluate_batch(dataset, config: config_a)
@@ -226,6 +230,8 @@ comparison.significant_regressions    # => []
 
 Significance markers: `*` (p < 0.05), `**` (p < 0.01), `***` (p < 0.001)
 
+For the statistical reasoning behind paired t-tests and how to read these p-values, see [Understanding A/B Comparison](https://github.com/dpaluy/rubric_llm/wiki/Understanding-A-B-Comparison) on the wiki.
+
 ## Test Integration
 
 ### Minitest
@@ -315,8 +321,8 @@ RubricLLM uses LLM-as-Judge — an LLM scores another LLM's output. This is the
 
 Mitigations built into the framework:
 
-- **Cross-model judging.** Configure a different model as judge than the one being evaluated. Don't let GPT-4o grade GPT-4o.
-- **Retrieval metrics are pure math.** `precision_at_k`, `recall_at_k`, `mrr`, `ndcg` — no LLM involved, no judge bias.
+- **Cross-model judging.** Configure a different model as judge than the one being evaluated. Don't let gpt-5.5 grade gpt-5.5.
+- **Retrieval metrics are pure math.** `precision_at_k`, `recall_at_k`, `mrr`, `ndcg` — no LLM involved, no judge bias. See [Why Retrieval Metrics Are Pure Math](https://github.com/dpaluy/rubric_llm/wiki/Why-Retrieval-Metrics-Are-Pure-Math).
 - **Custom non-LLM metrics.** Subclass `Metrics::Base` with regex checks, embedding similarity, or any deterministic logic.
 - **Statistical comparison.** A/B testing with paired t-tests surfaces systematic judge bias across runs.
 
@@ -337,6 +343,13 @@ Ruby has two LLM evaluation options today. Neither fits most use cases:
 | **Pluggable metrics** | No (fixed set) | Yes | Yes |
 | **Retrieval metrics** | Yes | No | Yes |
 
+## Further Reading
+
+Deep dives live in the [project wiki](https://github.com/dpaluy/rubric_llm/wiki):
+
+- [Understanding A/B Comparison](https://github.com/dpaluy/rubric_llm/wiki/Understanding-A-B-Comparison) — what paired t-tests and p-values mean for model comparison
+- [Why Retrieval Metrics Are Pure Math](https://github.com/dpaluy/rubric_llm/wiki/Why-Retrieval-Metrics-Are-Pure-Math) — why `precision_at_k`, `recall_at_k`, `mrr`, `ndcg`, and `hit_rate` are deterministic and bias-free
+
 ## Requirements
 
 - Ruby >= 3.4
@@ -350,3 +363,5 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/dpaluy/
 ## License
 
 [MIT](LICENSE.txt)
+
+Supported by [Majestic Labs](https://majesticlabs.dev/).

data/lib/rubric_llm/evaluator.rb

@@ -30,7 +30,7 @@ module RubricLLM
         result = metric.call(question:, answer:, context:, ground_truth:)
         scores[name] = result[:score]
         details[name] = result[:details]
-      rescue StandardError => e
+      rescue JudgeError => e
         scores[name] = nil
         details[name] = { error: e.message }
       end

data/lib/rubric_llm/judge.rb

@@ -4,6 +4,24 @@ require "json"
 
 module RubricLLM
   class Judge
+    METRIC_RESPONSE_SCHEMA = {
+      name: "rubric_llm_metric_response",
+      strict: false,
+      schema: {
+        type: "object",
+        properties: {
+          score: { type: "number", minimum: 0.0, maximum: 1.0 },
+          reasoning: { type: "string" },
+          claims: { type: "array", items: { type: "object" } },
+          context_scores: { type: "array", items: { type: "object" } },
+          covered_facts: { type: "array", items: { type: "object" } },
+          discrepancies: { type: "array", items: { type: "object" } }
+        },
+        required: ["score"],
+        additionalProperties: true
+      }
+    }.freeze
+
     attr_reader :config
 
     def initialize(config:)
@@ -20,13 +38,19 @@ module RubricLLM
         chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
         chat.with_temperature(config.temperature)
         chat.with_params(max_tokens: config.max_tokens)
+        apply_response_schema(chat)
 
         full_system_prompt = build_system_prompt(system_prompt)
         chat.with_instructions(full_system_prompt)
         response = chat.ask(user_prompt)
-        parse_json(response.content)
+        content = response.content
+        validate_response!(content.is_a?(Hash) ? content : parse_json(content))
       rescue StandardError => e
-        raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries
+        if attempts > config.max_retries
+          raise e if e.is_a?(JudgeError)
+
+          raise JudgeError, "Judge call failed: #{e.message}"
+        end
 
         sleep(config.retry_base_delay * (2**(attempts - 1)))
         retry
@@ -36,25 +60,55 @@ module RubricLLM
     # Parse JSON from LLM output with multiple strategies:
     # 1. Direct JSON.parse
     # 2. Extract from markdown code fence
-    # 3. Return nil (never raises)
+    # 3. Raise JudgeError for malformed output
     def parse_json(text)
-      return nil if text.nil? || text.strip.empty?
+      raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty?
 
-      # Try direct parse
       JSON.parse(text)
-    rescue JSON::ParserError
-      # Try extracting from code fence
+    rescue JSON::ParserError => e
       if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
         begin
-          JSON.parse(match[1])
-        rescue JSON::ParserError
-          nil
+          return JSON.parse(match[1])
+        rescue JSON::ParserError => e
+          raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}"
         end
       end
+
+      raise JudgeError, "Judge response was not valid JSON: #{e.message}"
     end
 
     private
 
+    def apply_response_schema(chat)
+      return chat unless chat.respond_to?(:with_schema)
+      return chat unless structured_output_supported?(chat)
+
+      chat.with_schema(METRIC_RESPONSE_SCHEMA)
+    end
+
+    def structured_output_supported?(chat)
+      return true unless chat.respond_to?(:model)
+      return true unless chat.model.respond_to?(:structured_output?)
+
+      chat.model.structured_output?
+    end
+
+    def validate_response!(response)
+      raise JudgeError, "Judge response must be a JSON object" unless response.is_a?(Hash)
+      raise JudgeError, "Judge response missing required score" unless response.key?("score")
+
+      score = parse_score(response["score"])
+      raise JudgeError, "Judge response score must be between 0.0 and 1.0" unless score.finite? && score.between?(0.0, 1.0)
+
+      response
+    end
+
+    def parse_score(score)
+      Float(score)
+    rescue ArgumentError, TypeError
+      raise JudgeError, "Judge response score must be numeric"
+    end
+
     def build_system_prompt(base_prompt)
       return base_prompt unless config.custom_prompt
 

data/lib/rubric_llm/metrics/base.rb

@@ -19,10 +19,7 @@ module RubricLLM
       private
 
       def judge_eval(system_prompt:, user_prompt:)
-        result = judge.call(system_prompt:, user_prompt:)
-        return { score: nil, details: { error: "No response from judge" } } if result.nil?
-
-        result
+        judge.call(system_prompt:, user_prompt:)
       end
     end
   end

data/lib/rubric_llm/metrics/context_precision.rb

@@ -34,10 +34,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             context_scores: result["context_scores"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/context_recall.rb

@@ -35,10 +35,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             covered_facts: result["covered_facts"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/correctness.rb

@@ -34,10 +34,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

data/lib/rubric_llm/metrics/factual_accuracy.rb

@@ -33,10 +33,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             discrepancies: result["discrepancies"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/faithfulness.rb

@@ -36,10 +36,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             claims: result["claims"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/relevance.rb

@@ -30,10 +30,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

data/lib/rubric_llm/retrieval_result.rb

@@ -51,7 +51,7 @@ module RubricLLM
     end
 
     def hit_rate
-      retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0
+      relevant.intersect?(retrieved) ? 1.0 : 0.0
     end
 
     def to_h

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - David Paluy
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
 description: Provider-agnostic LLM evaluation with pluggable metrics, statistical
   A/B comparison, and test framework integration. Ragas for Ruby, powered by RubyLLM.
 email: