ruby_llm-contract 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 35a61fe65d6a7939e3ef22bdd37732d2ae6cd5643f51d595a3f26b4281eea396
-  data.tar.gz: 9b1b95b29c31e433af60c25e85dfdebf3e8e71cb85c0e568835309a7cd855926
+  metadata.gz: dee963c252704634b8b9452e4e0460561e7795385e2dc59f4d5cc089a16d9210
+  data.tar.gz: ce289e0f1dee22a75d7079b28775c6dd0e5d85b01a54e5a97e4f47b40c2f5741
 SHA512:
-  metadata.gz: 0bb0333b6c362b1687b51f6bf360fd6d659c066a2a5b4b539bab4795150e5c1c8dbebe8dac6d05791b62958058d60418e5ff1f2b5db1f050f29412ed136494a5
-  data.tar.gz: ff5a8e7c30344993617bdd5f85d857e91d0cb633e2b7fe35a08aadf0790a4c7c0389cb017f92a192d199fe1eaba9526c509d5731321b36bd2c6e5fdedb5ca6d0
+  metadata.gz: d10ff4021462051d80cb5205174a24f9c5093ee096fc5add7d5bfacc88fb936a364d474871c05d87dad404ffc9577c998e7a1ae73cc8a8e0a5868e7cef629c83
+  data.tar.gz: 914a370baf65d5e8fc62f78a22e3bc6ee9eba83b78257ac95b87c8d5965ae23e54dbb7a66de7b2b6c7dc3c848a513be22c2e37e76445d9094dd576f3d3867215

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.3.7 (2026-03-24)
+
+- **Trait missing key = error** — `expected_traits: { title: 0..5 }` on output `{}` now fails instead of silently passing.
+- **nil input in dynamic prompts** — `run(nil)` with `prompt { |input| ... }` correctly passes nil to block.
+- **Defensive sample pre-validation** — `sample_response` uses the same parser as runtime (handles code fences, BOM, prose around JSON).
+- **Baseline diff excludes skipped** — self-compare with skipped cases no longer shows artificial score delta.
+- **Zeitwerk eval/ ignore** — `eager_load_contract_dirs!` ignores `eval/` subdirs before eager load.
+
 ## 0.3.6 (2026-03-24)
 
 - **Recursive array/object validation** — nested arrays (`array of array of string`) validated recursively. Object items validated even without `:properties` (e.g. `additionalProperties: false`).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ruby_llm-contract (0.3.6)
+    ruby_llm-contract (0.3.7)
       dry-types (~> 1.7)
       ruby_llm (~> 1.0)
       ruby_llm-schema (~> 0.3)
@@ -165,7 +165,7 @@ CHECKSUMS
   rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   ruby_llm (1.14.0) sha256=57c6f7034fc4a44504ea137d70f853b07824f1c1cdbe774ab3ab3522e7098deb
-  ruby_llm-contract (0.3.6)
+  ruby_llm-contract (0.3.7)
   ruby_llm-schema (0.3.0) sha256=a591edc5ca1b7f0304f0e2261de61ba4b3bea17be09f5cf7558153adfda3dec6
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
   unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f

data/README.md

@@ -6,7 +6,7 @@ Companion gem for [ruby_llm](https://github.com/crmne/ruby_llm).
 
 ## The problem
 
-You call an LLM. It returns bad JSON, wrong values, or costs 4x more than it should. You switch models and quality drops silently. You have no data to decide which model to use.
+Which model should you use? The expensive one is accurate but costs 4x more. The cheap one is fast but hallucinates on edge cases. You tweak a prompt — did accuracy improve or drop? You have no data. Just gut feeling.
 
 ## The fix
 

data/lib/ruby_llm/contract/eval/baseline_diff.rb

@@ -9,8 +9,8 @@ module RubyLLM
         def initialize(baseline_cases:, current_cases:)
           @baseline = index_by_name(baseline_cases)
           @current = index_by_name(current_cases)
-          @baseline_score = baseline_cases.empty? ? 0.0 : baseline_cases.sum { |c| c[:score] } / baseline_cases.length
-          @current_score = current_cases.empty? ? 0.0 : current_cases.sum { |c| c[:score] } / current_cases.length
+          @baseline_score = compute_score(baseline_cases)
+          @current_score = compute_score(current_cases)
           freeze
         end
 
@@ -78,6 +78,14 @@ module RubyLLM
 
         private
 
+        def compute_score(cases)
+          # Exclude skipped cases from score (consistent with Report#score)
+          evaluated = cases.reject { |c| c[:details]&.start_with?("skipped:") }
+          return 0.0 if evaluated.empty?
+
+          evaluated.sum { |c| c[:score] } / evaluated.length
+        end
+
         def index_by_name(cases)
           cases.each_with_object({}) { |c, h| h[c[:name]] = c }
         end

data/lib/ruby_llm/contract/eval/eval_definition.rb

@@ -106,15 +106,14 @@ module RubyLLM
           return if errors.empty?
 
           raise ArgumentError, "sample_response does not satisfy step schema: #{errors.join(", ")}"
-        rescue JSON::ParserError => e
-          # Non-JSON string with a structured schema = clear error
+        rescue JSON::ParserError, RubyLLM::Contract::ParseError => e
           raise ArgumentError, "sample_response is not valid JSON: #{e.message}"
         end
 
         def validate_sample_against_schema(schema)
           parsed = case @sample_response
                    when Hash, Array then @sample_response
-                   when String then JSON.parse(@sample_response)
+                   when String then Parser.parse(@sample_response, strategy: :json)
                    else @sample_response
                    end
           symbolized = deep_symbolize(parsed)

data/lib/ruby_llm/contract/eval/trait_evaluator.rb

@@ -19,8 +19,11 @@ module RubyLLM
         end
 
         def check_trait(output, key, expectation, errors)
-          value = output.is_a?(Hash) ? output[key] : nil
-          error_msg = trait_error(key, value, expectation)
+          unless output.is_a?(Hash) && output.key?(key)
+            errors << "#{key}: missing key"
+            return
+          end
+          error_msg = trait_error(key, output[key], expectation)
           errors << error_msg if error_msg
         end
 

data/lib/ruby_llm/contract/prompt/builder.rb

@@ -4,14 +4,16 @@ module RubyLLM
   module Contract
     module Prompt
       class Builder
+        NOT_PROVIDED = Object.new.freeze
+
         def initialize(block)
           @block = block
           @nodes = []
         end
 
-        def build(input = nil)
+        def build(input = NOT_PROVIDED)
           @nodes = []
-          if !input.nil? && @block.arity >= 1
+          if input != NOT_PROVIDED && @block.arity >= 1
             instance_exec(input, &@block)
           else
             instance_eval(&@block)
@@ -39,7 +41,7 @@ module RubyLLM
           @nodes << Nodes::SectionNode.new(name, text)
         end
 
-        def self.build(input: nil, &block)
+        def self.build(input: NOT_PROVIDED, &block)
           new(block).build(input)
         end
       end

data/lib/ruby_llm/contract/step/base.rb

@@ -82,7 +82,8 @@ module RubyLLM
 
           def build_messages(input)
             dynamic = prompt.arity >= 1
-            ast = Prompt::Builder.build(input: dynamic ? input : nil, &prompt)
+            builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
+            ast = Prompt::Builder.build(input: builder_input, &prompt)
             variables = dynamic ? {} : { input: input }
             variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
             Prompt::Renderer.render(ast, variables: variables)

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.3.6"
+    VERSION = "0.3.7"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -88,9 +88,12 @@ module RubyLLM
           full = ::Rails.root.join(path)
           next unless full.exist?
 
+          # Ignore eval/ subdirs — they don't define Zeitwerk-compatible
+          # constants and are loaded separately by load_evals!
+          eval_dir = full.join("eval")
+          ::Rails.autoloaders.main.ignore(eval_dir.to_s) if eval_dir.exist?
           ::Rails.autoloaders.main.eager_load_dir(full.to_s)
         rescue StandardError
-          # Zeitwerk not available or dir not managed — skip
           nil
         end
       end

data/ruby_llm-contract.gemspec

@@ -7,9 +7,10 @@ Gem::Specification.new do |spec|
   spec.version = RubyLLM::Contract::VERSION
   spec.authors = ["Justyna"]
 
-  spec.summary = "Contract-first LLM step execution for RubyLLM"
-  spec.description = "Turn RubyLLM calls into contracted, validated, testable steps with schema enforcement, " \
-                     "retry with model escalation, and eval."
+  spec.summary = "Know which LLM model to use, what it costs, and when accuracy drops"
+  spec.description = "Compare LLM models by accuracy and cost. Regression-test prompts in CI. " \
+                     "Start on nano, auto-escalate to bigger models when quality drops. " \
+                     "Companion gem for ruby_llm."
   spec.homepage = "https://github.com/justi/ruby_llm-contract"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 3.2.0"
@@ -17,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
   spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["documentation_uri"] = "#{spec.homepage}#readme"
   spec.metadata["rubygems_mfa_required"] = "true"
 
   spec.files = Dir.chdir(__dir__) do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Justyna
@@ -51,8 +51,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-description: Turn RubyLLM calls into contracted, validated, testable steps with schema
-  enforcement, retry with model escalation, and eval.
+description: Compare LLM models by accuracy and cost. Regression-test prompts in CI.
+  Start on nano, auto-escalate to bigger models when quality drops. Companion gem
+  for ruby_llm.
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -154,6 +155,7 @@ metadata:
   homepage_uri: https://github.com/justi/ruby_llm-contract
   source_code_uri: https://github.com/justi/ruby_llm-contract
   changelog_uri: https://github.com/justi/ruby_llm-contract/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/justi/ruby_llm-contract#readme
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -171,5 +173,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.7
 specification_version: 4
-summary: Contract-first LLM step execution for RubyLLM
+summary: Know which LLM model to use, what it costs, and when accuracy drops
 test_files: []