rubric_llm 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4984077f320263925eed14be810301406ff2a6f369f8d3d50ed7678e8d997c9c
-  data.tar.gz: 9457b67015d20bd1e68d2b0f4acfaa836c6842b0a0b567d6393911cdcb012fd4
+  metadata.gz: 6ed5a9f9ece0c78f86b4134211e39d8100e046046686ec296697d794cefcce15
+  data.tar.gz: 40e120b95e7d8b5711e495f909e357395fd5ffbae76cb49dd8aa6c5212488196
 SHA512:
-  metadata.gz: 58e6b00870262214c6a7a1331987e22e7f496490f3b3bd135b4f56b400718d7bfa28727749412b58687b0dc5c16c33d3dfa59fc20383a314fc58b031083232e1
-  data.tar.gz: 7c6c68ffb6ec620c8f02f7bde4993c90eaf707c72057f77d240072108aba267e77b9f3b44618c9b6d8a80d70aaed40b21c84f795ec4a8c8228c08d16ac7f49e9
+  metadata.gz: e07a90dabde33a66007ce21bd2173c577ce61fb1a964ce6946f2df26d03ff29dac18bc5757d209fcf5f8d76b5e8bb72c33562f250ee2335adc9aa1c83b0aada1
+  data.tar.gz: 8fcf6d081679021ac44aa729f63e243aabe344aa93c730a5b660d4fcc981963a08f7017276416b68b7b8b2d390756c537469263cdad92772e5f23a66199f1b0e

data/CHANGELOG.md

@@ -5,7 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.2.0] - 2026-07-11
+
+### Changed
+
+- Enforce schema-backed judge responses through RubyLLM structured output when supported
+- Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
+- Require `ruby_llm ~> 1.13` for named schema payload support in structured output
+- Record judge failures per metric in `RubricLLM.evaluate` and `RubricLLM.evaluate_batch` as a `nil` score with the error message in details and continue the run, while non-judge errors propagate
+- Remove dead score clamping and nil-response guards from LLM metrics now that the judge validates the response contract
 
 ## [0.1.2] - 2026-04-30
 

data/README.md

@@ -1,9 +1,10 @@
 # RubricLLM
 
-Lightweight LLM evaluation framework for Ruby, inspired by [Ragas](https://github.com/vibrantlabsai/ragas), powered by [RubyLLM](https://github.com/crmne/ruby_llm).
+Lightweight LLM evaluation framework for Ruby, inspired by [DeepEval](https://github.com/confident-ai/deepeval), powered by [RubyLLM](https://github.com/crmne/ruby_llm).
 
 [![Gem Version](https://badge.fury.io/rb/rubric_llm.svg)](https://badge.fury.io/rb/rubric_llm)
 [![CI](https://github.com/dpaluy/rubric_llm/actions/workflows/ci.yml/badge.svg)](https://github.com/dpaluy/rubric_llm/actions/workflows/ci.yml)
+[![Wiki](https://img.shields.io/badge/docs-wiki-blue)](https://github.com/dpaluy/rubric_llm/wiki)
 
 Provider-agnostic evaluation with pluggable metrics, statistical A/B comparison, and test framework integration — no Rails, no ActiveRecord, no UI. Works anywhere Ruby runs.
 
@@ -32,16 +33,16 @@ RubricLLM.configure do |c|
 end
 
 result = RubricLLM.evaluate(
-  question: "What is the capital of France?",
-  answer: "The capital of France is Paris, located on the Seine river.",
-  context: ["Paris is the capital and largest city of France."],
-  ground_truth: "Paris"
+  question: "What is Ruby's core design philosophy?",
+  answer: "Ruby was designed by Yukihiro Matsumoto to optimize for developer happiness and productivity, prioritizing the programmer's joy over machine efficiency.",
+  context: [
+    "Yukihiro Matsumoto, Ruby's creator, has stated that Ruby is designed to make programmers happy. " \
+    "He optimized the language for human readability and developer productivity rather than raw machine performance."
+  ],
+  ground_truth: "Ruby is designed to maximize developer happiness and productivity."
 )
 
-result.faithfulness      # => 0.95
-result.relevance         # => 0.92
-result.correctness       # => 0.98
-result.overall           # => 0.94
+result.correctness       # => 0.99
 result.pass?             # => true
 ```
 
@@ -54,7 +55,7 @@ Minitest assertions, see [examples/README.md](examples/README.md).
 
 ```ruby
 RubricLLM.configure do |c|
-  c.judge_model = "gpt-4o"           # any model RubyLLM supports
+  c.judge_model = "gpt-5.5"           # any model RubyLLM supports
   c.judge_provider = :openai          # :openai, :anthropic, :gemini, etc.
   c.temperature = 0.0                 # deterministic scoring (default)
   c.max_tokens = 4096                 # max tokens for judge response
@@ -83,7 +84,7 @@ config = RubricLLM::Config.from_env
 ### Per-Evaluation Override
 
 ```ruby
-custom = RubricLLM::Config.new(judge_model: "claude-haiku-4-5", judge_provider: :anthropic)
+custom = RubricLLM::Config.new(judge_model: "claude-opus-4-5", judge_provider: :anthropic)
 
 result = RubricLLM.evaluate(question: "...", answer: "...", config: custom)
 report = RubricLLM.evaluate_batch(dataset, config: custom)
@@ -94,7 +95,7 @@ report = RubricLLM.evaluate_batch(dataset, config: custom)
 ```ruby
 # config/initializers/rubric_llm.rb
 RubricLLM.configure do |c|
-  c.judge_model = "gpt-4o"
+  c.judge_model = "gpt-5.5"
   c.judge_provider = :openai
 end
 ```
@@ -107,12 +108,12 @@ These metrics use a judge LLM to evaluate quality. Each sends a structured promp
 
 | Metric | Question it answers | Requires |
 |--------|-------------------|----------|
-| **Faithfulness** | Is every claim in the answer supported by the context? | `context` |
-| **Relevance** | Does the answer address what was asked? | `question` |
 | **Correctness** | Does the answer match the known correct answer? | `ground_truth` |
+| **Relevance** | Does the answer address what was asked? | `question` |
 | **Context Precision** | Are the retrieved context chunks actually relevant? | `question`, `context` |
-| **Context Recall** | Do the contexts cover the information in the ground truth? | `context`, `ground_truth` |
 | **Factual Accuracy** | Are there factual discrepancies between candidate and reference? | `ground_truth` |
+| **Context Recall** | Do the contexts cover the information in the ground truth? | `context`, `ground_truth` |
+| **Faithfulness** | Is every claim in the answer supported by the context? | `context` |
 
 ```ruby
 # Only context — gets faithfulness, relevance, context_precision
@@ -163,8 +164,8 @@ result = RubricLLM.evaluate_retrieval(
 )
 
 result.precision_at_k(3)  # => 0.67
-result.recall_at_k(3)     # => 1.0
-result.mrr                # => 1.0
+result.recall_at_k(3)     # => 0.90
+result.mrr                # => 0.90
 result.ndcg               # => 0.86
 result.hit_rate           # => 1.0
 ```
@@ -206,7 +207,7 @@ report.to_json                        # returns JSON string
 Compare two models with statistical significance testing:
 
 ```ruby
-config_a = RubricLLM::Config.new(judge_model: "gpt-4o")
+config_a = RubricLLM::Config.new(judge_model: "gpt-5.5")
 config_b = RubricLLM::Config.new(judge_model: "claude-sonnet-4-6")
 
 report_a = RubricLLM.evaluate_batch(dataset, config: config_a)
@@ -229,6 +230,8 @@ comparison.significant_regressions    # => []
 
 Significance markers: `*` (p < 0.05), `**` (p < 0.01), `***` (p < 0.001)
 
+For the statistical reasoning behind paired t-tests and how to read these p-values, see [Understanding A/B Comparison](https://github.com/dpaluy/rubric_llm/wiki/Understanding-A-B-Comparison) on the wiki.
+
 ## Test Integration
 
 ### Minitest
@@ -318,8 +321,8 @@ RubricLLM uses LLM-as-Judge — an LLM scores another LLM's output. This is the
 
 Mitigations built into the framework:
 
-- **Cross-model judging.** Configure a different model as judge than the one being evaluated. Don't let GPT-4o grade GPT-4o.
-- **Retrieval metrics are pure math.** `precision_at_k`, `recall_at_k`, `mrr`, `ndcg` — no LLM involved, no judge bias.
+- **Cross-model judging.** Configure a different model as judge than the one being evaluated. Don't let gpt-5.5 grade gpt-5.5.
+- **Retrieval metrics are pure math.** `precision_at_k`, `recall_at_k`, `mrr`, `ndcg` — no LLM involved, no judge bias. See [Why Retrieval Metrics Are Pure Math](https://github.com/dpaluy/rubric_llm/wiki/Why-Retrieval-Metrics-Are-Pure-Math).
 - **Custom non-LLM metrics.** Subclass `Metrics::Base` with regex checks, embedding similarity, or any deterministic logic.
 - **Statistical comparison.** A/B testing with paired t-tests surfaces systematic judge bias across runs.
 
@@ -340,6 +343,13 @@ Ruby has two LLM evaluation options today. Neither fits most use cases:
 | **Pluggable metrics** | No (fixed set) | Yes | Yes |
 | **Retrieval metrics** | Yes | No | Yes |
 
+## Further Reading
+
+Deep dives live in the [project wiki](https://github.com/dpaluy/rubric_llm/wiki):
+
+- [Understanding A/B Comparison](https://github.com/dpaluy/rubric_llm/wiki/Understanding-A-B-Comparison) — what paired t-tests and p-values mean for model comparison
+- [Why Retrieval Metrics Are Pure Math](https://github.com/dpaluy/rubric_llm/wiki/Why-Retrieval-Metrics-Are-Pure-Math) — why `precision_at_k`, `recall_at_k`, `mrr`, `ndcg`, and `hit_rate` are deterministic and bias-free
+
 ## Requirements
 
 - Ruby >= 3.4

data/lib/rubric_llm/evaluator.rb

@@ -30,7 +30,7 @@ module RubricLLM
         result = metric.call(question:, answer:, context:, ground_truth:)
         scores[name] = result[:score]
         details[name] = result[:details]
-      rescue StandardError => e
+      rescue JudgeError => e
         scores[name] = nil
         details[name] = { error: e.message }
       end

data/lib/rubric_llm/judge.rb

@@ -4,6 +4,24 @@ require "json"
 
 module RubricLLM
   class Judge
+    METRIC_RESPONSE_SCHEMA = {
+      name: "rubric_llm_metric_response",
+      strict: false,
+      schema: {
+        type: "object",
+        properties: {
+          score: { type: "number", minimum: 0.0, maximum: 1.0 },
+          reasoning: { type: "string" },
+          claims: { type: "array", items: { type: "object" } },
+          context_scores: { type: "array", items: { type: "object" } },
+          covered_facts: { type: "array", items: { type: "object" } },
+          discrepancies: { type: "array", items: { type: "object" } }
+        },
+        required: ["score"],
+        additionalProperties: true
+      }
+    }.freeze
+
     attr_reader :config
 
     def initialize(config:)
@@ -20,13 +38,19 @@ module RubricLLM
         chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
         chat.with_temperature(config.temperature)
         chat.with_params(max_tokens: config.max_tokens)
+        apply_response_schema(chat)
 
         full_system_prompt = build_system_prompt(system_prompt)
         chat.with_instructions(full_system_prompt)
         response = chat.ask(user_prompt)
-        parse_json(response.content)
+        content = response.content
+        validate_response!(content.is_a?(Hash) ? content : parse_json(content))
       rescue StandardError => e
-        raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries
+        if attempts > config.max_retries
+          raise e if e.is_a?(JudgeError)
+
+          raise JudgeError, "Judge call failed: #{e.message}"
+        end
 
         sleep(config.retry_base_delay * (2**(attempts - 1)))
         retry
@@ -36,25 +60,55 @@ module RubricLLM
     # Parse JSON from LLM output with multiple strategies:
     # 1. Direct JSON.parse
     # 2. Extract from markdown code fence
-    # 3. Return nil (never raises)
+    # 3. Raise JudgeError for malformed output
     def parse_json(text)
-      return nil if text.nil? || text.strip.empty?
+      raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty?
 
-      # Try direct parse
       JSON.parse(text)
-    rescue JSON::ParserError
-      # Try extracting from code fence
+    rescue JSON::ParserError => e
       if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
         begin
-          JSON.parse(match[1])
-        rescue JSON::ParserError
-          nil
+          return JSON.parse(match[1])
+        rescue JSON::ParserError => e
+          raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}"
         end
       end
+
+      raise JudgeError, "Judge response was not valid JSON: #{e.message}"
     end
 
     private
 
+    def apply_response_schema(chat)
+      return chat unless chat.respond_to?(:with_schema)
+      return chat unless structured_output_supported?(chat)
+
+      chat.with_schema(METRIC_RESPONSE_SCHEMA)
+    end
+
+    def structured_output_supported?(chat)
+      return true unless chat.respond_to?(:model)
+      return true unless chat.model.respond_to?(:structured_output?)
+
+      chat.model.structured_output?
+    end
+
+    def validate_response!(response)
+      raise JudgeError, "Judge response must be a JSON object" unless response.is_a?(Hash)
+      raise JudgeError, "Judge response missing required score" unless response.key?("score")
+
+      score = parse_score(response["score"])
+      raise JudgeError, "Judge response score must be between 0.0 and 1.0" unless score.finite? && score.between?(0.0, 1.0)
+
+      response
+    end
+
+    def parse_score(score)
+      Float(score)
+    rescue ArgumentError, TypeError
+      raise JudgeError, "Judge response score must be numeric"
+    end
+
     def build_system_prompt(base_prompt)
       return base_prompt unless config.custom_prompt
 

data/lib/rubric_llm/metrics/base.rb

@@ -19,10 +19,7 @@ module RubricLLM
       private
 
       def judge_eval(system_prompt:, user_prompt:)
-        result = judge.call(system_prompt:, user_prompt:)
-        return { score: nil, details: { error: "No response from judge" } } if result.nil?
-
-        result
+        judge.call(system_prompt:, user_prompt:)
       end
     end
   end

data/lib/rubric_llm/metrics/context_precision.rb

@@ -34,10 +34,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             context_scores: result["context_scores"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/context_recall.rb

@@ -35,10 +35,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             covered_facts: result["covered_facts"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/correctness.rb

@@ -34,10 +34,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

data/lib/rubric_llm/metrics/factual_accuracy.rb

@@ -33,10 +33,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             discrepancies: result["discrepancies"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/faithfulness.rb

@@ -36,10 +36,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             claims: result["claims"],
             reasoning: result["reasoning"]

data/lib/rubric_llm/metrics/relevance.rb

@@ -30,10 +30,8 @@ module RubricLLM
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

data/lib/rubric_llm/retrieval_result.rb

@@ -51,7 +51,7 @@ module RubricLLM
     end
 
     def hit_rate
-      retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0
+      relevant.intersect?(retrieved) ? 1.0 : 0.0
     end
 
     def to_h

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - David Paluy
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
 description: Provider-agnostic LLM evaluation with pluggable metrics, statistical
   A/B comparison, and test framework integration. Ragas for Ruby, powered by RubyLLM.
 email: