ruby_llm-contract 1.2.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fead48c5516f8e0aa738d3ab1633ca2d07f7173aba6466a6d29e37634263b857
-  data.tar.gz: 4ae4c314cd6365c5f5de455f9dc5ddc0741abd26fa751804b22c5ffded0e6cb1
+  metadata.gz: b3d6985e56b8e410f796207f23b52e910b925f1bf4263353da65ca3f540c9535
+  data.tar.gz: 4cb5f7471dc5d97f3d3bd78fe41110cb6a9f28139e90029872fa612ed4ef3e9e
 SHA512:
-  metadata.gz: 9b66df1e490e238fdacc84a728b874a008b80c8b44cdd882b444a84389bb77996ee6132efb6adaad79c91a04bee30eb7351241a4989ff1a4ed5df8d7a6cff619
-  data.tar.gz: a055fd3d06ee4fa9591fa2a6cfed6872142febd950fe3a455daa6e588f04e157a585701456f4815483ecc3accb803cf3bbcffa9308c6dfa5011c00f6a7143274
+  metadata.gz: 62e15223ae05cfa64fd64959eb62858ace305a6f24e23e2c3f466219681eae4a1cd4c08addaf9d7d309f554c4ead57e1051d9b4b1eb55bcea4542ce8df23d789
+  data.tar.gz: e4c07a0a8b4706cbc062ca713ab9c8a2e87906b2fa4126a3b6d10f72ef30625ce6c8c5ea0a29ac30fef0b5251a06fd451bd1462c45289a0ab1cc939cdc25b989

data/CHANGELOG.md

@@ -1,8 +1,44 @@
 # Changelog
 
+## 1.3.1 (2026-10-11)
+
+### Fixed
+
+- **`Eval::Recommender` priced a call N times too low on a `compare_models(runs: N)`
+  comparison.** With `runs > 1` each report is an `AggregatedReport`: its results
+  hold the cases of all N runs, while `total_cost` is the mean cost of one run. The
+  cost per call divided one run's cost by the cases of every run, so with `runs: 3`
+  a $0.002 call showed as $0.0007. That skewed `best`, the retry chain, the rationale
+  and `savings`, and a candidate measured over more runs looked cheaper than one
+  measured once. Cases are now counted per run. `Step.recommend` always compares
+  with one run, so only a `Recommender` built by hand on a multi-run comparison
+  was affected.
+
+## 1.3.0 (2026-10-11)
+
+### Changed - may turn a green CI red
+
+- **A non-null value under `anyOf` / `oneOf` is now validated against the branches.**
+  0.10.3 taught the validator to accept `nil` through a null branch, but a non-null
+  value was never checked at all: the child node carried the composition wrapper,
+  which has no `type` or `properties`, so nothing ran. A nullable object block
+  (`any_of :block do object do ... end; null end`) missing a required field, with
+  a wrong-typed field or with an extra key came back `:ok`. Measured in an adopter:
+  a page-analysis block without `score` was `:ok` and became a 0/10 fail downstream.
+  `anyOf` now passes when at least one branch validates and `oneOf` when exactly one
+  does. When none does, the errors of the one branch whose type fits the value are
+  reported (`missing required field: block.score`); otherwise
+  `block: expected one of object, null, got String`. A `oneOf` value matching two
+  branches fails with `matches 2 oneOf branches, expected exactly 1`. Steps whose
+  model already returns schema-conforming output (OpenAI strict mode) are
+  unaffected; output that slipped through before now fails as `:validation_failed`.
+- **`type` given as an array is checked.** `type: ["string", "null"]` accepted any
+  value; it now accepts a value of any listed type and reports
+  `x: expected string or null, got Integer` otherwise. `"null"` is a known type.
+
 ## 1.2.0 (2026-10-10)
 
-Everything new is opt-in, except the temperature fix below.
+Everything new is opt-in. Two changes under "Changed" can affect existing steps: the temperature fix and `retry_on` rejecting conditions it cannot match.
 
 ### Added
 

data/lib/ruby_llm/contract/contract/schema_validator/node.rb

@@ -29,6 +29,22 @@ module RubyLLM
           Array(schema[:required]).map(&:to_s)
         end
 
+        def composition
+          SchemaValidator::COMPOSITION_KEYWORDS.each do |keyword|
+            branches = schema[keyword]
+            return [keyword, branches] if branches.is_a?(Array) && !branches.empty?
+          end
+          nil
+        end
+
+        def with_schema(branch_schema)
+          with(schema: branch_schema)
+        end
+
+        def label
+          path || "root"
+        end
+
         def items_schema
           schema[:items]
         end

data/lib/ruby_llm/contract/contract/schema_validator/type_rule.rb

@@ -9,12 +9,12 @@ module RubyLLM
         end
 
         def validate(node)
-          expected_type = node.expected_type
+          expected_types = Array(node.schema[:type]).map(&:to_s)
           value = node.value
-          return unless expected_type
-          return if type_valid?(expected_type, value)
+          return if expected_types.empty?
+          return if expected_types.any? { |type| type_valid?(type, value) }
 
-          @errors << "#{node.path}: expected #{expected_type}, got #{value.class}"
+          @errors << "#{node.path}: expected #{expected_types.join(" or ")}, got #{value.class}"
         end
 
         private

data/lib/ruby_llm/contract/contract/schema_validator.rb

@@ -25,8 +25,10 @@ module RubyLLM
         "number" => ->(value) { value.is_a?(Numeric) },
         "boolean" => ->(value) { value.is_a?(TrueClass) || value.is_a?(FalseClass) },
         "array" => ->(value) { value.is_a?(Array) },
-        "object" => ->(value) { value.is_a?(Hash) }
+        "object" => ->(value) { value.is_a?(Hash) },
+        "null" => lambda(&:nil?)
       }.freeze
+      COMPOSITION_KEYWORDS = %i[anyOf oneOf].freeze
 
       def self.validate(parsed_output, schema)
         new(parsed_output, schema).validate
@@ -59,11 +61,52 @@ module RubyLLM
       end
 
       def validate_node(node)
+        return validate_composition(node) if node.composition
+
         @scalar_rules.validate(node)
         @object_rules.validate(node) { |child| validate_node(child) } if node.hash? && node.object_schema?
         validate_array(node) if node.array?
       end
 
+      # anyOf passes when at least one branch validates, oneOf when exactly one
+      # does. When none does, the errors of the one branch whose type fits the
+      # value are the useful ones; otherwise the value matches no branch type.
+      def validate_composition(node)
+        keyword, branches = node.composition
+        branch_errors = branches.map { |branch| errors_for(node.with_schema(branch)) }
+        passing = branch_errors.count(&:empty?)
+        return if keyword == :anyOf ? passing.positive? : passing == 1
+
+        if passing > 1
+          @errors << "#{node.label}: matches #{passing} oneOf branches, expected exactly 1"
+        else
+          report_no_matching_branch(node, branches, branch_errors)
+        end
+      end
+
+      def report_no_matching_branch(node, branches, branch_errors)
+        fitting = branches.each_index.select { |i| branch_type_fits?(branches[i], node.value) }
+        return @errors.concat(branch_errors[fitting.first]) if fitting.size == 1
+
+        types = branches.flat_map { |branch| branch_types(branch) }.uniq
+        @errors << "#{node.label}: expected one of #{types.join(", ")}, got #{node.value.class}"
+      end
+
+      def errors_for(node)
+        before = @errors.size
+        validate_node(node)
+        @errors.slice!(before..)
+      end
+
+      def branch_type_fits?(branch, value)
+        branch_types(branch).any? { |type| TYPE_CHECKS.fetch(type, ->(_) { true }).call(value) }
+      end
+
+      def branch_types(branch)
+        types = Array(branch[:type]).map(&:to_s)
+        types.empty? && branch.key?(:properties) ? ["object"] : types
+      end
+
       def validate_array(node)
         items_schema = node.items_schema
         return unless items_schema.is_a?(Hash)

data/lib/ruby_llm/contract/eval/recommender.rb

@@ -40,15 +40,11 @@ module RubyLLM
             report = @comparison.reports[label]
             next nil unless report
 
-            evaluated_count = report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
-            cases_count = [evaluated_count, 1].max
-            cost_per_call = report.total_cost.to_f / cases_count
-
             {
               label: label,
               config: config,
               score: report.score,
-              cost_per_call: cost_per_call,
+              cost_per_call: cost_per_call(report),
               latency: report.avg_latency_ms || Float::INFINITY,
               pass_rate_ratio: report.pass_rate_ratio,
               total_cost: report.total_cost,
@@ -117,15 +113,20 @@ module RubyLLM
           current_report = @comparison.reports[current_label]
           return {} if current_report.nil? || unknown_cost?(current_report)
 
-          current_evaluated = current_report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
-          current_cases = [current_evaluated, 1].max
-          current_cost = current_report.total_cost.to_f / current_cases
-          diff = current_cost - best[:cost_per_call]
+          diff = cost_per_call(current_report) - best[:cost_per_call]
           return {} unless diff.positive?
 
           { per_call: diff.round(6), monthly_at: { 10_000 => (diff * 10_000).round(2) } }
         end
 
+        # An AggregatedReport's results hold the cases of every run while its
+        # total_cost is the mean of one run, so cases are counted per run.
+        def cost_per_call(report)
+          runs = report.respond_to?(:runs) ? report.runs.length : 1
+          evaluated = report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
+          report.total_cost.to_f * runs / [evaluated, 1].max
+        end
+
         # A subtotal over unpriced cases is not a cost to rank by. Zero stays
         # "unknown pricing" too: offline runs report $0 without any call.
         def cost_known?(scored_candidate)

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "1.2.0"
+    VERSION = "1.3.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Justyna