rubric_llm 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1129b7b842324735b232caf6a116451cb04b599187aac713ce20c5befda4295
-  data.tar.gz: 33fc34b74d756ded10b00227cc9c46de94aa3da8a17dd798dc49e94edee5ba15
+  metadata.gz: 3f75b6e37c79f540f62929562809323154ab17055a6e1f6b74c918605dd3f4c0
+  data.tar.gz: 9a4a7752bf87c1e16e4d75ff190d5a45f9c368276cb47a7c148be2c873de9e35
 SHA512:
-  metadata.gz: 19c150856bbb65ab6d1c85e0a80ab92c309fc4524f20ab2057323715f6dd506d24408e37263e37b13eb0d3b6410f7c56a47a2511cbb25d5d0a1d53872117e4ab
-  data.tar.gz: e3c83d54f10befc47c0a403f9784f33a634debae737201cf39b758784bc3a0c1f6760a08aab677d00c101eab6a5b67ee8d05626a39360d1d68fea5ab1304c767
+  metadata.gz: 3606120936454d38b72f5441ef3329a645a3dd725148728fe3e3b45e595c76028d02aeab176767ec93bacae330709ec9ad1bdff57f06ac4942ddda89a518ad5b
+  data.tar.gz: 667739e0ee4e4568faf3f438b9b3b4455fcd9d7fb0815e2575f172d8e91abb33c7a03fe9f4b436f14cc0a35614c9ce604d23c1d509befaf62e0a023b81aa107c

data/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0] - 2026-09-22
+
+### Added
+
+- Optional `thinking_effort` configuration and `RUBRIC_THINKING_EFFORT` environment variable for judge reasoning models, passed to RubyLLM's `with_thinking(effort: ...)`. The default leaves thinking controls unchanged; explicit `nil` ignores the environment variable
+
 ## [0.6.0] - 2026-09-18
 
 ### Changed

data/README.md

@@ -8,14 +8,14 @@ Lightweight LLM evaluation framework for Ruby, inspired by [DeepEval](https://gi
 
 Provider-agnostic evaluation with pluggable metrics, statistical A/B comparison, and test framework integration: no Rails, no ActiveRecord, no UI. Works anywhere Ruby runs.
 
-RubricLLM `0.6.0` supports RubyLLM `~> 2.0` and Ruby 3.4 or later. Applications that use RubyLLM 1.x should stay on RubricLLM `0.5.x`.
+RubricLLM `0.7.0` supports RubyLLM `~> 2.0` and Ruby 3.4 or later. Applications that use RubyLLM 1.x should stay on RubricLLM `0.5.x`.
 
 ## Installation
 
 Update both gem constraints together:
 
 ```ruby
-gem "rubric_llm", "~> 0.6.0"
+gem "rubric_llm", "~> 0.7.0"
 gem "ruby_llm", "~> 2.0"
 ```
 
@@ -30,7 +30,7 @@ If the application still has a RubyLLM 1.x constraint, update both constraints b
 Or install directly:
 
 ```bash
-gem install rubric_llm --version 0.6.0
+gem install rubric_llm --version 0.7.0
 ```
 
 ## Quick Start
@@ -92,6 +92,28 @@ end
 
 Structured output support depends on the selected provider and model. RubricLLM sends its schema when RubyLLM reports structured output support. Otherwise it requests JSON text and validates the response object and score locally. Check the target provider's support before relying on a schema or a specific protocol. See RubyLLM's [2.0 upgrade guide](https://rubyllm.com/next/upgrading/), [request control guide](https://rubyllm.com/next/chat-request-control/), and [structured output support](https://rubyllm.com/next/structured-output/).
 
+### Judge Thinking Effort
+
+Set `thinking_effort` when using a reasoning model to control the trade-off between judge quality, latency, and cost:
+
+```ruby
+RubricLLM.configure do |c|
+  c.judge_model = "gpt-6-astra"
+  c.judge_provider = :openai
+  c.temperature = nil
+  c.thinking_effort = :high
+end
+
+# Or configure an individual evaluation:
+config = RubricLLM::Config.new(judge_model: "gpt-6-astra", temperature: nil, thinking_effort: "high")
+```
+
+RubricLLM passes the value to RubyLLM's [`with_thinking(effort: ...)`](https://rubyllm.com/thinking/). Strings and symbols are accepted. Supported values depend on the model and provider; RubricLLM does not translate values or silently ignore unsupported settings. Request errors are reported as judge errors.
+
+The default is `nil`: RubricLLM does not call `with_thinking`, so existing RubyLLM and provider behavior is unchanged. This does **not** disable reasoning. An omitted setting reads `RUBRIC_THINKING_EFFORT`; an explicit `thinking_effort: nil` ignores that environment variable. Use `:none` only when the selected model supports it.
+
+Higher effort does not guarantee better scores. Compare results on your evaluation dataset. For providers that count reasoning tokens toward the output limit, increase `max_tokens` if the default 4,096 tokens leaves too little room for the JSON response.
+
 ### Environment Variables
 
 All config fields can be set via environment variables:
@@ -101,6 +123,7 @@ All config fields can be set via environment variables:
 | `RUBRIC_JUDGE_MODEL` | `gpt-4o` | Judge LLM model name |
 | `RUBRIC_JUDGE_PROVIDER` | `openai` | RubyLLM provider |
 | `RUBRIC_TEMPERATURE` | `0.0` | Judge temperature |
+| `RUBRIC_THINKING_EFFORT` | unset | Model-specific reasoning effort, such as `high` |
 | `RUBRIC_MAX_TOKENS` | `4096` | Max response tokens |
 | `RUBRIC_MAX_RETRIES` | `2` | Max retries on transient failures |
 | `RUBRIC_RETRY_BASE_DELAY` | `1.0` | Base delay (seconds) for exponential backoff |
@@ -403,7 +426,7 @@ Deep dives live in the [project wiki](https://github.com/dpaluy/rubric_llm/wiki)
 ## Requirements
 
 - Ruby >= 3.4
-- [ruby_llm](https://github.com/crmne/ruby_llm) ~> 2.0 for RubricLLM 0.6.0
+- [ruby_llm](https://github.com/crmne/ruby_llm) ~> 2.0 for RubricLLM 0.7.0
 - An API key for your chosen LLM provider (set via RubyLLM configuration)
 
 ## Contributing

data/lib/rubric_llm/config.rb

@@ -3,14 +3,16 @@
 module RubricLLM
   class Config
     attr_accessor :judge_model, :judge_provider, :temperature, :max_tokens, :custom_prompt,
-                  :max_retries, :retry_base_delay, :concurrency
+                  :max_retries, :retry_base_delay, :concurrency, :thinking_effort
 
     def initialize(judge_model: nil, judge_provider: nil, # rubocop:disable Metrics/ParameterLists
                    temperature: Float(ENV.fetch("RUBRIC_TEMPERATURE", "0.0")), max_tokens: nil,
-                   custom_prompt: nil, max_retries: nil, retry_base_delay: nil, concurrency: nil, validate: false)
+                   custom_prompt: nil, max_retries: nil, retry_base_delay: nil, concurrency: nil, validate: false,
+                   thinking_effort: ENV.fetch("RUBRIC_THINKING_EFFORT", nil))
       @judge_model = judge_model || ENV.fetch("RUBRIC_JUDGE_MODEL", "gpt-4o")
       @judge_provider = (judge_provider || ENV.fetch("RUBRIC_JUDGE_PROVIDER", "openai")).to_sym
       @temperature = temperature
+      @thinking_effort = thinking_effort
       @max_tokens = max_tokens || Integer(ENV.fetch("RUBRIC_MAX_TOKENS", "4096"))
       @custom_prompt = custom_prompt
       @max_retries = max_retries || Integer(ENV.fetch("RUBRIC_MAX_RETRIES", "2"))
@@ -27,6 +29,7 @@ module RubricLLM
       validate_judge_model
       validate_judge_provider
       validate_temperature
+      validate_thinking_effort
       validate_max_tokens
       validate_max_retries
       validate_retry_base_delay
@@ -39,6 +42,7 @@ module RubricLLM
         judge_model:,
         judge_provider:,
         temperature:,
+        thinking_effort:,
         max_tokens:,
         custom_prompt:,
         max_retries:,
@@ -67,6 +71,13 @@ module RubricLLM
       raise ConfigurationError, "temperature must be nil or between 0.0 and 2.0"
     end
 
+    def validate_thinking_effort
+      return if thinking_effort.nil?
+      return if (thinking_effort.is_a?(String) || thinking_effort.is_a?(Symbol)) && !thinking_effort.to_s.strip.empty?
+
+      raise ConfigurationError, "thinking_effort must be nil or a non-empty string or symbol"
+    end
+
     def validate_max_tokens
       return if max_tokens.is_a?(Integer) && max_tokens.positive?
 

data/lib/rubric_llm/judge.rb

@@ -52,6 +52,7 @@ module RubricLLM
         chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
         chat.with_temperature(config.temperature)
         chat.with_max_output_tokens(config.max_tokens)
+        chat.with_thinking(effort: config.thinking_effort) unless config.thinking_effort.nil?
         apply_response_schema(chat)
 
         full_system_prompt = build_system_prompt(system_prompt)

data/lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubric_llm
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - David Paluy