rubric_llm 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ed5a9f9ece0c78f86b4134211e39d8100e046046686ec296697d794cefcce15
-  data.tar.gz: 40e120b95e7d8b5711e495f909e357395fd5ffbae76cb49dd8aa6c5212488196
+  metadata.gz: 16ccb30778e4ace0b35900e0d488ac884042acf124e139181e4e364a6a5829ed
+  data.tar.gz: f5c1df6b47de48bd4c976a9986836aa2d2c66c6614dc94e4b6f4f8b12c090144
 SHA512:
-  metadata.gz: e07a90dabde33a66007ce21bd2173c577ce61fb1a964ce6946f2df26d03ff29dac18bc5757d209fcf5f8d76b5e8bb72c33562f250ee2335adc9aa1c83b0aada1
-  data.tar.gz: 8fcf6d081679021ac44aa729f63e243aabe344aa93c730a5b660d4fcc981963a08f7017276416b68b7b8b2d390756c537469263cdad92772e5f23a66199f1b0e
+  metadata.gz: d267c7eceba64ebf8b9f5829b960ebf45aaec98bbd8545a393f16f8dfdf0070fb98ee8038e0fe9ddb1428164bbbcb677b6e8da0afa86513865187a358aad45a7
+  data.tar.gz: d0fff2333050e27bb785820f128ebaee7161f15e446f728beffbe2744078d95cd0de077b4a2674b26d8136391ee9c977f5cc87a1608f48ff96d9e0ed8cd04dd8

data/CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2026-07-11
+
+### Added
+
+- `RubricLLM.evaluate_batch` validates every sample upfront (must be a Hash with non-nil `:question` and `:answer`, string or symbol keys) and raises `ArgumentError` with the offending index before any LLM call, so sequential and concurrent modes fail identically and without API spend
+
+### Changed
+
+- Add `csv` as a runtime dependency; `csv` moved from a default gem to a bundled gem in Ruby 3.4, so consumers previously hit a `LoadError` on `Report#export_csv`
+- Require `ruby_llm ~> 1.16`
+
+## [0.3.0] - 2026-07-11
+
+### Changed
+
+- `Result#pass?` now returns `false` when any metric errored during evaluation (fail-closed for CI gating)
+- `Report#worst` now ranks results with no valid scores as worst instead of best
+
+### Added
+
+- `Result#errors` and `Result#valid?` expose per-metric judge failures
+- `Result#to_h` and report JSON exports include metric error information
+- Report summaries show a metric error count line when evaluation errors occurred
+
 ## [0.2.0] - 2026-07-11
 
 ### Changed

data/lib/rubric_llm/report.rb

@@ -16,7 +16,7 @@ module RubricLLM
     end
 
     def worst(n)
-      results.sort_by { |r| r.overall || Float::INFINITY }.first(n)
+      results.sort_by { |r| r.overall || -Float::INFINITY }.first(n)
     end
 
     def failures(threshold: 0.8)
@@ -34,11 +34,15 @@ module RubricLLM
                         metric, stats[:mean], stats[:std], stats[:min], stats[:max], stats[:count])
       end
 
+      error_count = error_counts.values.sum
+      affected_samples = results.count { |r| r.errors.any? }
+      lines << "Errors: #{error_count} metric errors across #{affected_samples} samples" if error_count.positive?
+
       lines.join("\n")
     end
 
     def export_csv(path)
-      require "csv" # optional dependency — add `gem "csv"` to your Gemfile if missing
+      require "csv"
       metrics = all_metric_names
       CSV.open(path, "w") do |csv|
         csv << ["question", "answer", "overall", *metrics]
@@ -72,6 +76,7 @@ module RubricLLM
       {
         summary: metric_stats,
         duration:,
+        errors: error_counts,
         results: results.map(&:to_h)
       }
     end
@@ -80,6 +85,12 @@ module RubricLLM
       results.flat_map { |r| r.scores.keys }.uniq
     end
 
+    def error_counts
+      counts = Hash.new(0)
+      results.each { |r| r.errors.each_key { |name| counts[name] += 1 } }
+      counts
+    end
+
     def compute_stats
       all_metric_names.each_with_object({}) do |metric, stats|
         values = results.filter_map { |r| r.scores[metric] }

data/lib/rubric_llm/result.rb

@@ -18,14 +18,26 @@ module RubricLLM
     end
 
     def pass?(threshold: 0.8)
+      return false unless valid?
+
       score = overall
       return false if score.nil?
 
       score >= threshold
     end
 
+    def errors
+      details.each_with_object({}) do |(name, detail), acc|
+        acc[name] = detail[:error] if detail.is_a?(Hash) && detail.key?(:error)
+      end
+    end
+
+    def valid?
+      errors.empty?
+    end
+
     def to_h
-      { scores:, details:, overall: overall }
+      { scores:, details:, overall: overall, errors: }
     end
 
     private

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/rubric_llm.rb

@@ -56,6 +56,7 @@ module RubricLLM
     #   report = RubricLLM.evaluate_batch(dataset)
     #   report = RubricLLM.evaluate_batch(dataset, concurrency: 4)
     def evaluate_batch(dataset, metrics: nil, config: self.config, custom_prompt: nil, concurrency: nil)
+      validate_dataset!(dataset)
       config = apply_custom_prompt(config, custom_prompt)
       pool_size = concurrency || config.concurrency
       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -87,6 +88,22 @@ module RubricLLM
 
     private
 
+    def validate_dataset!(dataset)
+      dataset.each_with_index do |sample, index|
+        raise ArgumentError, "sample at index #{index} is not a Hash" unless sample.is_a?(Hash)
+
+        question_present = sample.key?(:question) || sample.key?("question")
+        answer_present = sample.key?(:answer) || sample.key?("answer")
+        raise ArgumentError, "sample at index #{index} is missing :question" unless question_present
+        raise ArgumentError, "sample at index #{index} is missing :answer" unless answer_present
+
+        question_provided = !sample[:question].nil? || !sample["question"].nil?
+        answer_provided = !sample[:answer].nil? || !sample["answer"].nil?
+        raise ArgumentError, "sample at index #{index} has nil :question" unless question_provided
+        raise ArgumentError, "sample at index #{index} has nil :answer" unless answer_provided
+      end
+    end
+
     def evaluate_sample(evaluator, sample)
       sample = normalize_sample(sample)
       evaluator.call(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - David Paluy
@@ -9,20 +9,34 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby_llm
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '1.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '1.16'
 description: Provider-agnostic LLM evaluation with pluggable metrics, statistical
   A/B comparison, and test framework integration. Ragas for Ruby, powered by RubyLLM.
 email: