rubric_llm 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ed5a9f9ece0c78f86b4134211e39d8100e046046686ec296697d794cefcce15
-  data.tar.gz: 40e120b95e7d8b5711e495f909e357395fd5ffbae76cb49dd8aa6c5212488196
+  metadata.gz: 4afa554e2e9ae10520d0cc605f55d59cdc6330c7ba65f38c4d3e933fb11519a4
+  data.tar.gz: e371eaaf0ab368c46d930e352434664bca19315bfc45865601686b0a5525ca5a
 SHA512:
-  metadata.gz: e07a90dabde33a66007ce21bd2173c577ce61fb1a964ce6946f2df26d03ff29dac18bc5757d209fcf5f8d76b5e8bb72c33562f250ee2335adc9aa1c83b0aada1
-  data.tar.gz: 8fcf6d081679021ac44aa729f63e243aabe344aa93c730a5b660d4fcc981963a08f7017276416b68b7b8b2d390756c537469263cdad92772e5f23a66199f1b0e
+  metadata.gz: f635136ea65b903555493b377a14d5902b2be949ef7bc77b036450ac25d8ead70429039c5f47ffdc36c2216470e39409ceb50f35c609df8f4e5648b64569fa38
+  data.tar.gz: a66697e131141d27ff0c406fdd60b6ef103c4c69cce3b85a3c5cf481cbdf55cc489983b10e94ffe7557ab0a637a8c09530e9cd0f317e16e092496862574fed1c

data/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2026-07-11
+
+### Changed
+
+- `Result#pass?` now returns `false` when any metric errored during evaluation (fail-closed for CI gating)
+- `Report#worst` now ranks results with no valid scores as worst instead of best
+
+### Added
+
+- `Result#errors` and `Result#valid?` expose per-metric judge failures
+- `Result#to_h` and report JSON exports include metric error information
+- Report summaries show a metric error count line when evaluation errors occurred
+
 ## [0.2.0] - 2026-07-11
 
 ### Changed

data/lib/rubric_llm/report.rb

@@ -16,7 +16,7 @@ module RubricLLM
     end
 
     def worst(n)
-      results.sort_by { |r| r.overall || Float::INFINITY }.first(n)
+      results.sort_by { |r| r.overall || -Float::INFINITY }.first(n)
     end
 
     def failures(threshold: 0.8)
@@ -34,6 +34,10 @@ module RubricLLM
                         metric, stats[:mean], stats[:std], stats[:min], stats[:max], stats[:count])
       end
 
+      error_count = error_counts.values.sum
+      affected_samples = results.count { |r| r.errors.any? }
+      lines << "Errors: #{error_count} metric errors across #{affected_samples} samples" if error_count.positive?
+
       lines.join("\n")
     end
 
@@ -72,6 +76,7 @@ module RubricLLM
       {
         summary: metric_stats,
         duration:,
+        errors: error_counts,
         results: results.map(&:to_h)
       }
     end
@@ -80,6 +85,12 @@ module RubricLLM
       results.flat_map { |r| r.scores.keys }.uniq
     end
 
+    def error_counts
+      counts = Hash.new(0)
+      results.each { |r| r.errors.each_key { |name| counts[name] += 1 } }
+      counts
+    end
+
     def compute_stats
       all_metric_names.each_with_object({}) do |metric, stats|
         values = results.filter_map { |r| r.scores[metric] }

data/lib/rubric_llm/result.rb

@@ -18,14 +18,26 @@ module RubricLLM
     end
 
     def pass?(threshold: 0.8)
+      return false unless valid?
+
       score = overall
       return false if score.nil?
 
       score >= threshold
     end
 
+    def errors
+      details.each_with_object({}) do |(name, detail), acc|
+        acc[name] = detail[:error] if detail.is_a?(Hash) && detail.key?(:error)
+      end
+    end
+
+    def valid?
+      errors.empty?
+    end
+
     def to_h
-      { scores:, details:, overall: overall }
+      { scores:, details:, overall: overall, errors: }
     end
 
     private

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - David Paluy