ruby_llm-contract 0.2.0

data/lib/ruby_llm/contract/contract/parser.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RubyLLM
+  module Contract
+    class Parser
+      extend Concerns::DeepSymbolize
+
+      def self.symbolize_keys(obj)
+        deep_symbolize(obj)
+      end
+
+      def self.parse(raw_output, strategy:)
+        case strategy
+        when :json then parse_json(raw_output)
+        when :text then raw_output
+        else raise ArgumentError, "Unknown parse strategy: #{strategy}"
+        end
+      end
+
+      def self.parse_json(raw_output)
+        return deep_symbolize(raw_output) if raw_output.is_a?(Hash) || raw_output.is_a?(Array)
+
+        # Coerce non-String scalars (boolean, numeric) to their JSON representation
+        # to prevent TypeError from JSON.parse on non-string input.
+        coerced = raw_output.is_a?(String) ? raw_output : raw_output&.to_s
+        text = strip_code_fences(strip_bom(coerced))
+        raise RubyLLM::Contract::ParseError.new("Failed to parse JSON: nil content", details: raw_output) if text.nil?
+
+        parse_json_text(text, raw_output)
+      end
+
+      def self.parse_json_text(text, raw_output)
+        JSON.parse(text, symbolize_names: true)
+      rescue JSON::ParserError
+        parse_json_with_extraction(text, raw_output)
+      end
+      private_class_method :parse_json_text
+
+      # Fallback: attempt to extract the first JSON object or array from prose
+      def self.parse_json_with_extraction(text, raw_output)
+        extracted = extract_json(text)
+        unless extracted
+          raise RubyLLM::Contract::ParseError.new(
+            "Failed to parse JSON: no valid JSON found in output", details: raw_output
+          )
+        end
+
+        JSON.parse(extracted, symbolize_names: true)
+      rescue JSON::ParserError => e
+        raise RubyLLM::Contract::ParseError.new("Failed to parse JSON: #{e.message}", details: raw_output)
+      end
+      private_class_method :parse_json_with_extraction
+
+      # Strip UTF-8 BOM (Byte Order Mark) that some LLMs/APIs prepend to output
+      UTF8_BOM = "\xEF\xBB\xBF"
+      def self.strip_bom(text)
+        return text unless text.is_a?(String)
+
+        text.delete_prefix(UTF8_BOM)
+      end
+
+      # Strip markdown code fences that LLMs commonly wrap around JSON output
+      # Handles ```json ... ```, ``` ... ```, with optional trailing whitespace
+      CODE_FENCE_PATTERN = /\A\s*```(?:json|JSON)?\s*\n(.*?)\n\s*```\s*\z/m
+
+      def self.strip_code_fences(text)
+        return text unless text.is_a?(String)
+
+        match = text.match(CODE_FENCE_PATTERN)
+        match ? match[1] : text
+      end
+
+      # Extract the first JSON object or array from text that may contain prose.
+      # Uses bracket-matching to find the outermost balanced { } or [ ] block.
+      JSON_START_PATTERN = /[{\[]/
+
+      def self.extract_json(text)
+        return nil unless text.is_a?(String)
+
+        start_idx = text.index(JSON_START_PATTERN)
+        return nil unless start_idx
+
+        scan_for_balanced_json(text, start_idx)
+      end
+
+      def self.scan_for_balanced_json(text, start_idx)
+        opening = text[start_idx]
+        closing = opening == "{" ? "}" : "]"
+        state = { depth: 0, in_string: false, escape_next: false }
+
+        (start_idx...text.length).each do |pos|
+          result = process_json_char(text[pos], opening, closing, state)
+          return text[start_idx..pos] if result == :matched
+        end
+
+        nil
+      end
+      private_class_method :scan_for_balanced_json
+
+      def self.process_json_char(char, opening, closing, state)
+        if state[:escape_next]
+          state[:escape_next] = false
+          return nil
+        end
+
+        return handle_backslash(state) if char == "\\"
+        return handle_quote(state) if char == '"'
+        return nil if state[:in_string]
+
+        handle_bracket(char, opening, closing, state)
+      end
+      private_class_method :process_json_char
+
+      def self.handle_backslash(state)
+        state[:escape_next] = true if state[:in_string]
+        nil
+      end
+      private_class_method :handle_backslash
+
+      def self.handle_quote(state)
+        state[:in_string] = !state[:in_string]
+        nil
+      end
+      private_class_method :handle_quote
+
+      def self.handle_bracket(char, opening, closing, state)
+        return adjust_depth(state, 1) if char == opening
+        return adjust_depth(state, -1) if char == closing
+
+        nil
+      end
+      private_class_method :handle_bracket
+
+      def self.adjust_depth(state, delta)
+        state[:depth] += delta
+        state[:depth].zero? ? :matched : nil
+      end
+      private_class_method :adjust_depth
+    end
+  end
+end

data/lib/ruby_llm/contract/contract/schema_validator.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    # Client-side validation of parsed output against an output_schema.
+    # Checks required fields, enum constraints, number ranges, and nested objects.
+    # This complements provider-side enforcement (with_schema) and catches
+    # violations when using Test adapter or providers that ignore schemas.
+    class SchemaValidator # rubocop:disable Metrics/ClassLength
+      include Concerns::DeepSymbolize
+
+      # Bundles field path, value, and constraints to reduce parameter passing
+      FieldCheck = Struct.new(:qualified, :value, :constraints)
+
+      SIZE_BOUNDS = {
+        string: { min_key: :minLength, max_key: :maxLength, metric: "length" },
+        array: { min_key: :minItems, max_key: :maxItems, metric: "array length" }
+      }.freeze
+
+      def self.validate(parsed_output, schema)
+        new(parsed_output, schema).validate
+      end
+
+      def initialize(parsed_output, schema)
+        @output = parsed_output
+        @json_schema = extract_schema(schema)
+        @errors = []
+      end
+
+      def validate
+        return [] unless @json_schema.is_a?(Hash)
+
+        return validate_non_hash_output unless @output.is_a?(Hash)
+
+        validate_object(@output, @json_schema, prefix: nil)
+        @errors
+      end
+
+      private
+
+      def validate_non_hash_output
+        expected_type = @json_schema[:type]&.to_s
+        if expected_type == "object" || @json_schema.key?(:properties)
+          ["expected object, got #{@output.class}"]
+        else
+          []
+        end
+      end
+
+      def extract_schema(schema)
+        instance = schema.is_a?(Class) ? schema.new : schema
+        json = if instance.respond_to?(:to_json_schema)
+                 schema_data = instance.to_json_schema
+                 schema_data[:schema] || schema_data["schema"] || schema_data
+               else
+                 schema
+               end
+        deep_symbolize(json)
+      end
+
+      def validate_object(output, schema, prefix:)
+        return unless output.is_a?(Hash) && schema.is_a?(Hash)
+
+        properties = schema[:properties] || {}
+        required = schema[:required] || []
+
+        check_required(required, output, prefix: prefix)
+        check_properties(properties, output, prefix: prefix, required_fields: required)
+        check_additional_properties(output, schema, prefix: prefix)
+      end
+
+      def check_required(required, output, prefix:)
+        required.each do |field|
+          key = field.to_s.to_sym
+          qualified = qualify(prefix, field)
+          @errors << "missing required field: #{qualified}" unless output.key?(key)
+        end
+      end
+
+      def check_properties(properties, output, prefix:, required_fields: [])
+        required_syms = required_fields.map { |field| field.to_s.to_sym }
+
+        properties.each do |field, constraints|
+          key = field.to_sym
+          value = output[key]
+          qualified = qualify(prefix, field)
+
+          if value.nil?
+            check_nil_required(qualified, key, constraints, required_syms, output)
+            next
+          end
+
+          validate_field(FieldCheck.new(qualified: qualified, value: value, constraints: constraints))
+        end
+      end
+
+      def check_nil_required(qualified, key, constraints, required_syms, output)
+        return unless required_syms.include?(key) && output.key?(key)
+
+        expected = constraints[:type] || "non-null"
+        @errors << "#{qualified}: expected #{expected}, got nil"
+      end
+
+      def check_additional_properties(output, schema, prefix:)
+        return unless schema[:additionalProperties] == false
+
+        allowed_keys = (schema[:properties] || {}).keys.map { |prop_key| prop_key.to_s.to_sym }
+        extra_keys = output.keys - allowed_keys
+
+        extra_keys.each do |extra_key|
+          @errors << "#{qualify(prefix, extra_key)}: additional property not allowed"
+        end
+      end
+
+      def validate_field(field_check)
+        check_enum(field_check)
+        check_number_range(field_check)
+        check_type_constraint(field_check)
+        check_string_length(field_check)
+        check_nested(field_check)
+      end
+
+      def check_enum(field_check)
+        qualified, value, constraints = field_check.to_a
+        enum = constraints[:enum]
+        return unless enum
+
+        @errors << "#{qualified}: #{value.inspect} is not in enum #{enum.inspect}" unless enum.include?(value)
+      end
+
+      def check_number_range(field_check)
+        qualified, value, constraints = field_check.to_a
+        return unless value.is_a?(Numeric)
+
+        check_minimum(qualified, value, constraints[:minimum])
+        check_maximum(qualified, value, constraints[:maximum])
+      end
+
+      def check_type_constraint(field_check)
+        qualified, value, constraints = field_check.to_a
+        expected_type = constraints[:type]&.to_s
+        return unless expected_type
+
+        @errors << "#{qualified}: expected #{expected_type}, got #{value.class}" unless type_valid?(expected_type,
+                                                                                                    value)
+      end
+
+      def type_valid?(expected_type, value)
+        case expected_type
+        when "string" then value.is_a?(String)
+        when "number" then value.is_a?(Numeric)
+        when "integer" then value.is_a?(Integer)
+        when "boolean" then [true, false].include?(value)
+        when "array" then value.is_a?(Array)
+        when "object" then value.is_a?(Hash)
+        else true
+        end
+      end
+
+      def check_nested(field_check)
+        qualified, value, constraints = field_check.to_a
+        nested_type = constraints[:type]&.to_s
+
+        case nested_type
+        when "object"
+          validate_object(value, constraints, prefix: qualified) if value.is_a?(Hash)
+        when "array"
+          check_array_items(qualified, value, constraints) if value.is_a?(Array)
+        end
+      end
+
+      def check_string_length(field_check)
+        qualified, value, constraints = field_check.to_a
+        check_size_bounds(qualified, value.length, constraints, :string) if value.is_a?(String)
+      end
+
+      def check_array_length(qualified, value, constraints)
+        check_size_bounds(qualified, value.length, constraints, :array) if value.is_a?(Array)
+      end
+
+      def check_size_bounds(qualified, actual, constraints, kind)
+        bounds = SIZE_BOUNDS[kind]
+        check_size_minimum(qualified, actual, constraints[bounds[:min_key]], bounds)
+        check_size_maximum(qualified, actual, constraints[bounds[:max_key]], bounds)
+      end
+
+      def check_array_items(qualified, value, constraints)
+        check_array_length(qualified, value, constraints)
+
+        items_schema = constraints[:items]
+        return unless items_schema.is_a?(Hash)
+
+        value.each_with_index do |item, idx|
+          validate_array_item("#{qualified}[#{idx}]", item, items_schema)
+        end
+      end
+
+      def validate_array_item(item_key, item, items_schema)
+        item_type = items_schema[:type]&.to_s
+
+        if item_type == "object" && item.is_a?(Hash)
+          validate_object(item, items_schema, prefix: item_key)
+        elsif item_type == "array" && item.is_a?(Array)
+          check_array_items(item_key, item, items_schema)
+        else
+          validate_field(FieldCheck.new(qualified: item_key, value: item, constraints: items_schema))
+        end
+      end
+
+      def check_minimum(qualified, actual, limit)
+        return unless limit && actual < limit
+
+        @errors << "#{qualified}: #{actual} is below minimum #{limit}"
+      end
+
+      def check_maximum(qualified, actual, limit)
+        return unless limit && actual > limit
+
+        @errors << "#{qualified}: #{actual} is above maximum #{limit}"
+      end
+
+      def check_size_minimum(qualified, actual, limit, bounds)
+        return unless limit && actual < limit
+
+        @errors << "#{qualified}: #{bounds[:metric]} #{actual} is below #{bounds[:min_key]} #{limit}"
+      end
+
+      def check_size_maximum(qualified, actual, limit, bounds)
+        return unless limit && actual > limit
+
+        @errors << "#{qualified}: #{bounds[:metric]} #{actual} is above #{bounds[:max_key]} #{limit}"
+      end
+
+      def qualify(prefix, field)
+        prefix ? "#{prefix}.#{field}" : field.to_s
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/contract/validator.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    class Validator
+      def validate(raw_output:, definition:, output_type:, input: nil, schema: nil)
+        effective_definition = schema ? with_json_strategy(definition) : definition
+        parsed_output = parse_output(raw_output, effective_definition)
+        return parsed_output if parse_error?(parsed_output)
+
+        effective_output = coerce_and_freeze(parsed_output, output_type, schema)
+        errors = collect_errors(effective_output, schema, definition, input)
+
+        { parsed_output: effective_output[:value], errors: errors, status: errors.empty? ? :ok : :validation_failed }
+      end
+
+      def self.validate(raw_output:, definition:, output_type:, input: nil, schema: nil)
+        new.validate(raw_output: raw_output, definition: definition, output_type: output_type,
+                     input: input, schema: schema)
+      end
+
+      private
+
+      def parse_error?(parsed_output)
+        parsed_output.is_a?(Hash) && parsed_output[:status] == :parse_error
+      end
+
+      def coerce_and_freeze(parsed_output, output_type, schema)
+        coerced_output, type_errors = validate_type(parsed_output, output_type, !schema.nil?)
+        effective = type_errors.empty? ? coerced_output : parsed_output
+        deep_freeze(effective)
+        { value: effective, type_errors: type_errors }
+      end
+
+      def collect_errors(effective_output, schema, definition, input)
+        effective_output[:type_errors] +
+          validate_schema(effective_output[:value], schema) +
+          validate_invariants(effective_output[:value], definition, input)
+      end
+
+      def validate_type(parsed_output, output_type, has_schema)
+        return [parsed_output, []] if has_schema
+
+        if output_type.is_a?(Class) && !output_type.respond_to?(:[])
+          raise TypeError, "expected #{output_type}, got #{parsed_output.class}" unless parsed_output.is_a?(output_type)
+
+          [parsed_output, []]
+        else
+          coerced = output_type[parsed_output]
+          [coerced, []]
+        end
+      rescue Dry::Types::CoercionError, TypeError, ArgumentError => e
+        [parsed_output, [e.message]]
+      end
+
+      def validate_schema(parsed_output, schema)
+        return [] unless schema
+
+        SchemaValidator.validate(parsed_output, schema)
+      end
+
+      def validate_invariants(parsed_output, definition, input)
+        definition.invariants.each_with_object([]) do |inv, errors|
+          passed = inv.call(parsed_output, input: input)
+          if passed.nil?
+            warn "[ruby_llm-contract] validate(\"#{inv.description}\") returned nil. " \
+                 "This usually means a key mismatch (string vs symbol). " \
+                 "Output keys are always symbols."
+          end
+          errors << inv.description unless passed
+        rescue StandardError => e
+          errors << "#{inv.description} (raised #{e.class}: #{e.message})"
+        end
+      end
+
+      def parse_output(raw_output, definition)
+        Parser.parse(raw_output, strategy: definition.parse_strategy)
+      rescue RubyLLM::Contract::ParseError => e
+        { parsed_output: nil, errors: [e.message], status: :parse_error }
+      end
+
+      def with_json_strategy(definition)
+        return definition if definition.parse_strategy == :json
+
+        Definition.new { parse :json }
+      end
+
+      def deep_freeze(obj)
+        case obj
+        when Hash
+          obj.each_value { |element| deep_freeze(element) }
+          obj.freeze
+        when Array
+          obj.each { |element| deep_freeze(element) }
+          obj.freeze
+        when String
+          obj.freeze
+        else
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/contract.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative "contract/invariant"
+require_relative "contract/parser"
+require_relative "contract/definition"
+require_relative "contract/schema_validator"
+require_relative "contract/validator"

data/lib/ruby_llm/contract/cost_calculator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module CostCalculator
+      def self.calculate(model_name:, usage:)
+        return nil unless model_name && usage.is_a?(Hash)
+
+        model_info = find_model(model_name)
+        return nil unless model_info
+
+        compute_cost(model_info, usage)
+      rescue StandardError
+        nil
+      end
+
+      def self.compute_cost(model_info, usage)
+        input_cost = token_cost(usage[:input_tokens], model_info.input_price_per_million)
+        output_cost = token_cost(usage[:output_tokens], model_info.output_price_per_million)
+        (input_cost + output_cost).round(6)
+      end
+
+      def self.token_cost(tokens, price_per_million)
+        (tokens || 0) * (price_per_million || 0) / 1_000_000.0
+      end
+
+      def self.find_model(model_name)
+        return nil unless defined?(RubyLLM)
+
+        RubyLLM.models.find(model_name)
+      rescue StandardError
+        nil
+      end
+
+      private_class_method :compute_cost, :token_cost, :find_model
+    end
+  end
+end

data/lib/ruby_llm/contract/dsl.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    # Include this module to get `Types` constant as a shortcut for RubyLLM::Contract::Types.
+    # Usage: `include RubyLLM::Contract::DSL` at the top of your file or class.
+    module DSL
+      def self.included(base)
+        base.const_set(:Types, RubyLLM::Contract::Types) unless base.const_defined?(:Types)
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/errors.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    class Error < StandardError
+      attr_reader :details
+
+      def initialize(message = nil, details: nil)
+        @details = details
+        super(message)
+      end
+    end
+
+    class InputError < Error; end
+    class ParseError < Error; end
+    class ContractError < Error; end
+    class AdapterError < Error; end
+  end
+end

data/lib/ruby_llm/contract/eval/case_result.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class CaseResult
+        attr_reader :name, :input, :output, :expected, :step_status,
+                    :score, :details, :duration_ms, :cost
+
+        def initialize(name:, input:, output:, expected:, step_status:,
+                       score:, passed:, label: nil, details: nil, duration_ms: nil, cost: nil)
+          @name = name
+          @input = input
+          @output = output
+          @expected = expected
+          @step_status = step_status
+          @score = score.to_f.clamp(0.0, 1.0)
+          @passed = passed
+          @label = label
+          @details = details
+          @duration_ms = duration_ms
+          @cost = cost
+          freeze
+        end
+
+        def passed?
+          @passed
+        end
+
+        def failed?
+          !@passed
+        end
+
+        def label
+          @label || (@passed ? "PASS" : "FAIL")
+        end
+
+        def mismatches
+          return {} unless @expected.is_a?(Hash) && @output.is_a?(Hash)
+
+          @expected.each_with_object({}) do |(key, value), result|
+            actual = @output[key]
+            next if match?(value, actual)
+
+            result[key] = { expected: value, got: actual }
+          end
+        end
+
+        def to_h
+          {
+            name: @name,
+            input: @input,
+            output: @output,
+            expected: @expected,
+            step_status: @step_status,
+            score: @score,
+            passed: @passed,
+            label: label,
+            details: @details,
+            duration_ms: @duration_ms,
+            cost: @cost
+          }
+        end
+
+        private
+
+        def match?(expected_value, actual)
+          case expected_value
+          when ::Regexp then actual.to_s.match?(expected_value)
+          else expected_value == actual
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/contract_detail_builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Extracted from Runner to reduce class length.
+      # Builds contract detail strings for contract-only evaluation.
+      module ContractDetailBuilder
+        private
+
+        def build_contract_details
+          parts = ["contract passed"]
+          append_schema_details(parts)
+          append_invariant_details(parts)
+          parts.join(", ")
+        end
+
+        def append_schema_details(parts)
+          return unless @step.respond_to?(:output_schema)
+
+          schema = @step.output_schema
+          return unless schema
+
+          field_count = begin
+            schema.properties.size
+          rescue StandardError
+            0
+          end
+          parts << "schema: #{field_count} fields" if field_count.positive?
+        end
+
+        def append_invariant_details(parts)
+          return unless @step.respond_to?(:contract)
+
+          invariant_count = begin
+            @step.contract.invariants.size
+          rescue StandardError
+            0
+          end
+          class_validates = @step.instance_variable_get(:@class_validates)&.size || 0
+          total = invariant_count + class_validates
+          parts << "validates: #{total} passed" if total.positive?
+        end
+      end
+    end
+  end
+end