rubric_llm 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f75b6e37c79f540f62929562809323154ab17055a6e1f6b74c918605dd3f4c0
-  data.tar.gz: 9a4a7752bf87c1e16e4d75ff190d5a45f9c368276cb47a7c148be2c873de9e35
+  metadata.gz: fe5f964e023687c815cf95b56673d05f3103f15a679ebb6f5d272d287c396163
+  data.tar.gz: f082d306d7e50347dc6a53caf453dc7e24abbd0213012711506692516ff38a47
 SHA512:
-  metadata.gz: 3606120936454d38b72f5441ef3329a645a3dd725148728fe3e3b45e595c76028d02aeab176767ec93bacae330709ec9ad1bdff57f06ac4942ddda89a518ad5b
-  data.tar.gz: 667739e0ee4e4568faf3f438b9b3b4455fcd9d7fb0815e2575f172d8e91abb33c7a03fe9f4b436f14cc0a35614c9ce604d23c1d509befaf62e0a023b81aa107c
+  metadata.gz: 8cdba0f5196529eed16d2c19edaca418f8feedca62244724a206e6b4baa1ab9ef13660a3cf24f1711ad52e43856f325648b88ee8443da0cce7bee9a9f9b637d7
+  data.tar.gz: c2828756fca3eb3395567bd61ebac9b80bdffd12dbdd4b410f000b5001b68169be14101a95bde4c0541f98526c9bdc7ff665e22e2d918fda75072af4dd1e4dc2

data/CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2026-09-22
+
+### Changed
+
+- Calculate faithfulness, context precision, and context recall scores from the judge's item-level decisions instead of its suggested score
+- Define scoring criteria for correctness, relevance, and factual accuracy, with factual accuracy measuring contradictions against the reference rather than missing facts
+- Reject missing or malformed metric details, including missing reasoning, empty claim or fact lists, invalid context indices, and factual accuracy scores that conflict with their discrepancy lists. Context indices must be 1-based integers. Evaluation returns `nil` with an error for these responses; direct metric calls raise `JudgeError`
+
 ## [0.7.0] - 2026-09-22
 
 ### Added

data/README.md

@@ -165,14 +165,14 @@ RubyLLM classifies OpenAI's HTTP 429 `insufficient_quota` response as a rate-lim
 
 ### LLM-as-Judge Metrics
 
-These metrics use a judge LLM to evaluate quality. Each sends a structured prompt and parses a JSON response with a 0.0–1.0 score.
+These metrics use a judge LLM to evaluate quality. Each returns a 0.0–1.0 score. Faithfulness counts supported answer claims, context precision counts relevant non-empty chunks, and context recall counts covered reference facts. These three scores are calculated from the judge's item-level decisions, not its suggested score. An empty or malformed item list (including an answer with no factual claims) is an evaluation error, not a quality score. Correctness, relevance, and factual accuracy use judge scores with metric-specific scoring criteria. The judge still decides what counts as a claim, fact, or relevant chunk, so scores are not deterministic across models.
 
 | Metric | Question it answers | Requires |
 |--------|-------------------|----------|
 | **Correctness** | Does the answer match the known correct answer? | `ground_truth` |
 | **Relevance** | Does the answer address what was asked? | `question` |
 | **Context Precision** | Are the retrieved context chunks actually relevant? | `question`, `context` |
-| **Factual Accuracy** | Are there factual discrepancies between candidate and reference? | `ground_truth` |
+| **Factual Accuracy** | Does the candidate contradict the reference (not omit it)? | `ground_truth` |
 | **Context Recall** | Do the contexts cover the information in the ground truth? | `context`, `ground_truth` |
 | **Faithfulness** | Is every claim in the answer supported by the context? | `context` |
 

data/lib/rubric_llm/metrics/base.rb

@@ -35,6 +35,29 @@ module RubricLLM
       def judge_eval(system_prompt:, user_prompt:)
         judge.call(system_prompt:, user_prompt:)
       end
+
+      def checked_items(result, key, label:, value_key:, text_key:, count: nil)
+        items = result[key]
+        valid = items.is_a?(Array) && items.any? && (count.nil? || items.size == count)
+        valid &&= items.all? do |item|
+          item.is_a?(Hash) && item[text_key].is_a?(String) && !item[text_key].strip.empty? &&
+            [true, false].include?(item[value_key])
+        end
+        raise JudgeError, "Judge response has invalid #{label}" unless valid
+
+        items
+      end
+
+      def fraction(items, key)
+        items.count { |item| item[key] }.to_f / items.size
+      end
+
+      def reasoning_for(result)
+        reasoning = result["reasoning"]
+        raise JudgeError, "Judge response missing reasoning" unless reasoning.is_a?(String) && !reasoning.strip.empty?
+
+        reasoning
+      end
     end
   end
 end

data/lib/rubric_llm/metrics/context_precision.rb

@@ -5,7 +5,8 @@ module RubricLLM
     class ContextPrecision < Base
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Assess whether the retrieved contexts are relevant to the question.
-        Context precision measures if the retrieved documents are useful for answering the question.
+        Context precision is the fraction of retrieved chunks useful for answering the question.
+        Classify every numbered chunk once, in order. Use the displayed 1-based index.
 
         Respond with JSON only:
         {
@@ -29,18 +30,20 @@ module RubricLLM
         PROMPT
 
         result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
-        normalize(result)
+        normalize(result, context_chunks.size)
       end
 
       private
 
-      def normalize(result)
+      def normalize(result, count)
+        context_scores = checked_items(result, "context_scores", label: "context scores", count:, value_key: "relevant", text_key: "reason")
+        unless context_scores.each_with_index.all? { |item, index| item["index"].is_a?(Integer) && item["index"] == index + 1 }
+          raise JudgeError, "Judge response has invalid context indices"
+        end
+
         {
-          score: Float(result["score"]),
-          details: {
-            context_scores: result["context_scores"],
-            reasoning: result["reasoning"]
-          }
+          score: fraction(context_scores, "relevant"),
+          details: { context_scores:, reasoning: reasoning_for(result) }
         }
       end
     end

data/lib/rubric_llm/metrics/context_recall.rb

@@ -5,7 +5,10 @@ module RubricLLM
     class ContextRecall < Base
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Assess whether the provided contexts cover the information in the ground truth.
-        Context recall measures if the retrieved documents contain enough information to construct the ground truth answer.
+        Context recall is the fraction of facts in the ground truth supported by the contexts.
+        List every distinct factual claim in the ground truth. Mark it covered only if a numbered
+        context supports it, and use that context's 1-based index as source_context.
+        Use null for source_context when a fact is not covered.
 
         Respond with JSON only:
         {
@@ -31,18 +34,22 @@ module RubricLLM
         PROMPT
 
         result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
-        normalize(result)
+        normalize(result, context_chunks.size)
       end
 
       private
 
-      def normalize(result)
+      def normalize(result, count)
+        covered_facts = checked_items(result, "covered_facts", label: "covered facts", value_key: "covered", text_key: "fact")
+        valid_sources = covered_facts.all? do |item|
+          source = item["source_context"]
+          item["covered"] ? source.is_a?(Integer) && (1..count).cover?(source) : source.nil?
+        end
+        raise JudgeError, "Judge response has invalid source contexts" unless valid_sources
+
         {
-          score: Float(result["score"]),
-          details: {
-            covered_facts: result["covered_facts"],
-            reasoning: result["reasoning"]
-          }
+          score: fraction(covered_facts, "covered"),
+          details: { covered_facts:, reasoning: reasoning_for(result) }
         }
       end
     end

data/lib/rubric_llm/metrics/correctness.rb

@@ -5,7 +5,11 @@ module RubricLLM
     class Correctness < Base
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Assess whether the answer matches the ground truth.
-        Consider semantic equivalence, not just exact string matching.
+        Consider semantic equivalence, not just exact string matching. Score the answer against
+        the question and reference: 1.0 if it fully answers with no material errors, 0.5 if it
+        is partly correct but misses required information or includes a material error, and 0.0
+        if it is wrong or does not answer. Use intermediate values for partial cases and explain
+        what is correct, missing, or wrong. Do not require irrelevant reference details.
 
         Respond with JSON only:
         {
@@ -36,7 +40,7 @@ module RubricLLM
       def normalize(result)
         {
           score: Float(result["score"]),
-          details: { reasoning: result["reasoning"] }
+          details: { reasoning: reasoning_for(result) }
         }
       end
     end

data/lib/rubric_llm/metrics/factual_accuracy.rb

@@ -5,7 +5,12 @@ module RubricLLM
     class FactualAccuracy < Base
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Compare the factual claims in the candidate answer against the reference answer.
-        Identify any discrepancies where the candidate states something different from the reference.
+        Identify contradictions in candidate factual claims against the reference. Do not penalize
+        missing reference facts, which correctness measures. Score 1.0 if there are no contradictions,
+        0.5 if minor factual contradictions affect part of the answer, and 0.0 if major contradictions
+        undermine the answer. Use intermediate values for partial cases. Explain the score and list
+        each contradiction with severity minor or major. If the reference does not establish whether
+        a claim is true, do not call it a contradiction.
 
         Respond with JSON only:
         {
@@ -33,14 +38,25 @@ module RubricLLM
       private
 
       def normalize(result)
+        discrepancies = result["discrepancies"]
+        unless discrepancies.is_a?(Array) && discrepancies.all? { |item| valid_discrepancy?(item) }
+          raise JudgeError, "Judge response has invalid discrepancies"
+        end
+
+        score = Float(result["score"])
+        raise JudgeError, "Judge response score conflicts with no discrepancies" if discrepancies.empty? && score < 1.0
+        raise JudgeError, "Judge response score conflicts with discrepancies" if discrepancies.any? && score >= 1.0
+
         {
-          score: Float(result["score"]),
-          details: {
-            discrepancies: result["discrepancies"],
-            reasoning: result["reasoning"]
-          }
+          score:,
+          details: { discrepancies:, reasoning: reasoning_for(result) }
         }
       end
+
+      def valid_discrepancy?(item)
+        item.is_a?(Hash) && %w[claim reference].all? { |key| item[key].is_a?(String) && !item[key].strip.empty? } &&
+          %w[minor major].include?(item["severity"])
+      end
     end
   end
 end

data/lib/rubric_llm/metrics/faithfulness.rb

@@ -6,6 +6,8 @@ module RubricLLM
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Assess whether the answer is faithful to the provided context.
         A faithful answer only contains information that is supported by the context.
+        List every factual claim in the answer. Mark a claim supported only if the context supports it.
+        The score is the fraction of supported claims. If there are no factual claims, do not invent claims.
 
         Respond with JSON only:
         {
@@ -36,11 +38,12 @@ module RubricLLM
       private
 
       def normalize(result)
+        claims = checked_items(result, "claims", label: "claims", value_key: "supported", text_key: "claim")
         {
-          score: Float(result["score"]),
+          score: fraction(claims, "supported"),
           details: {
-            claims: result["claims"],
-            reasoning: result["reasoning"]
+            claims:,
+            reasoning: reasoning_for(result)
           }
         }
       end

data/lib/rubric_llm/metrics/relevance.rb

@@ -5,7 +5,10 @@ module RubricLLM
     class Relevance < Base
       SYSTEM_PROMPT = <<~PROMPT
         You are an evaluation judge. Assess whether the answer is relevant to the question.
-        A relevant answer directly addresses what was asked.
+        A relevant answer directly addresses what was asked, regardless of its factual accuracy.
+        Score 1.0 if it directly addresses all parts of the question, 0.5 if it addresses only
+        part or is mostly tangential, and 0.0 if it does not address the question. Use intermediate
+        values for partial cases and explain the score. Do not score correctness here.
 
         Respond with JSON only:
         {
@@ -32,7 +35,7 @@ module RubricLLM
       def normalize(result)
         {
           score: Float(result["score"]),
-          details: { reasoning: result["reasoning"] }
+          details: { reasoning: reasoning_for(result) }
         }
       end
     end

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - David Paluy