ruby_llm-contract 0.2.0

data/lib/ruby_llm/contract/prompt/ast.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      class AST
+        include Enumerable
+
+        attr_reader :nodes
+
+        def initialize(nodes)
+          @nodes = nodes.dup.freeze
+          freeze
+        end
+
+        def each(&)
+          @nodes.each(&)
+        end
+
+        def size
+          @nodes.size
+        end
+
+        def [](index)
+          @nodes[index]
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && nodes == other.nodes
+        end
+
+        def to_a
+          @nodes.map(&:to_h)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      class Builder
+        def initialize(block)
+          @block = block
+          @nodes = []
+        end
+
+        def build(input = nil)
+          if input && @block.arity >= 1
+            instance_exec(input, &@block)
+          else
+            instance_eval(&@block)
+          end
+          AST.new(@nodes)
+        end
+
+        def system(text)
+          @nodes << Nodes::SystemNode.new(text)
+        end
+
+        def rule(text)
+          @nodes << Nodes::RuleNode.new(text)
+        end
+
+        def example(input:, output:)
+          @nodes << Nodes::ExampleNode.new(input: input, output: output)
+        end
+
+        def user(text)
+          @nodes << Nodes::UserNode.new(text)
+        end
+
+        def section(name, text)
+          @nodes << Nodes::SectionNode.new(name, text)
+        end
+
+        def self.build(input: nil, &block)
+          new(block).build(input)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/node.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      class Node
+        attr_reader :type, :content
+
+        def initialize(type:, content:)
+          @type = type.freeze
+          @content = content.freeze
+          freeze
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && type == other.type && content == other.content
+        end
+
+        def to_h
+          { type: @type, content: @content }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes/example_node.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      module Nodes
+        class ExampleNode < Node
+          attr_reader :input, :output
+
+          def initialize(input:, output:)
+            @input = input.freeze
+            @output = output.freeze
+            super(type: :example, content: nil)
+          end
+
+          def ==(other)
+            other.is_a?(self.class) && type == other.type && input == other.input && output == other.output
+          end
+
+          def to_h
+            { type: :example, input: @input, output: @output }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes/rule_node.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      module Nodes
+        class RuleNode < Node
+          def initialize(content)
+            super(type: :rule, content: content)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes/section_node.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      module Nodes
+        class SectionNode < Node
+          attr_reader :name
+
+          def initialize(name, content)
+            @name = name.freeze
+            super(type: :section, content: content)
+          end
+
+          def ==(other)
+            other.is_a?(self.class) && type == other.type && name == other.name && content == other.content
+          end
+
+          def to_h
+            { type: :section, name: @name, content: @content }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes/system_node.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      module Nodes
+        class SystemNode < Node
+          def initialize(content)
+            super(type: :system, content: content)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes/user_node.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Prompt
+      module Nodes
+        class UserNode < Node
+          def initialize(content)
+            super(type: :user, content: content)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/prompt/nodes.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative "nodes/system_node"
+require_relative "nodes/rule_node"
+require_relative "nodes/example_node"
+require_relative "nodes/user_node"
+require_relative "nodes/section_node"

data/lib/ruby_llm/contract/prompt/renderer.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RubyLLM
+  module Contract
+    module Prompt
+      class Renderer
+        def render(ast, variables: {})
+          ast.each_with_object([]) do |node, messages|
+            render_node(node, variables, messages)
+          end
+        end
+
+        def self.render(ast, variables: {})
+          new.render(ast, variables: variables)
+        end
+
+        private
+
+        def render_node(node, variables, messages)
+          case node
+          when Nodes::SystemNode, Nodes::RuleNode
+            append_message(messages, :system, node.content, variables)
+          when Nodes::ExampleNode
+            append_message(messages, :user, node.input, variables)
+            append_message(messages, :assistant, node.output, variables)
+          when Nodes::UserNode
+            append_message(messages, :user, node.content, variables)
+          when Nodes::SectionNode
+            render_section_node(node, variables, messages)
+          end
+        end
+
+        def append_message(messages, role, raw_content, variables)
+          content = interpolate(raw_content, variables)
+          messages << { role: role, content: content } if content_present?(content)
+        end
+
+        def render_section_node(node, variables, messages)
+          section_content = node.content.is_a?(Hash) || node.content.is_a?(Array) ? node.content.to_json : node.content
+          return unless content_present?(section_content)
+
+          safe_name = sanitize_section_name(node.name)
+          body = interpolate(section_content, variables)
+          messages << { role: :system, content: "[#{safe_name}]\n#{body}" }
+        end
+
+        def content_present?(content)
+          content.to_s.strip != ""
+        end
+
+        def sanitize_section_name(name)
+          name.to_s.gsub(/[\[\]\n\r]/, " ").strip
+        end
+
+        def interpolate(text, variables)
+          return text if text.nil?
+          return text.to_json if text.is_a?(Hash) || text.is_a?(Array)
+
+          # Coerce non-String content (Integer, Symbol, etc.) to String before gsub
+          text = text.to_s unless text.is_a?(String)
+
+          text.gsub(/\{(\w+)\}/) do |match|
+            key = ::Regexp.last_match(1).to_sym
+            variables.key?(key) ? serialize_value(variables[key]) : match
+          end
+        end
+
+        def serialize_value(value)
+          value.is_a?(Hash) || value.is_a?(Array) ? value.to_json : value.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/railtie.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    class Railtie < ::Rails::Railtie
+      # Eval files (e.g. classify_threads_eval.rb) don't define Zeitwerk-compatible
+      # constants — they call define_eval on an existing Step class. We use `load`
+      # after initialization, and hook into the reloader for development.
+
+      config.after_initialize do
+        RubyLLM::Contract.load_evals!
+      end
+
+      # Re-load eval files on code reload in development (Spring, zeitwerk:check, etc.)
+      config.to_prepare do
+        RubyLLM::Contract.load_evals!
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/rake_task.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "rake"
+require "rake/tasklib"
+
+module RubyLLM
+  module Contract
+    class RakeTask < ::Rake::TaskLib
+      attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost, :eval_dirs
+
+      def initialize(name = :"ruby_llm_contract:eval", &block)
+        super()
+        @name = name
+        @context = {}
+        @fail_on_empty = true
+        @minimum_score = nil # nil = require 100%; float = threshold
+        @maximum_cost = nil  # nil = no cost limit; float = budget cap (suite-level)
+        @eval_dirs = []      # directories to load eval files from (non-Rails)
+        block&.call(self)
+        define_task
+      end
+
+      private
+
+      def define_task
+        desc "Run all ruby_llm-contract evals"
+        task(@name => task_prerequisites) do
+          require "ruby_llm/contract"
+          @eval_dirs.each { |dir| RubyLLM::Contract.load_evals!(dir) }
+          RubyLLM::Contract.load_evals!
+
+          results = RubyLLM::Contract.run_all_evals(context: @context)
+
+          if results.empty?
+            if @fail_on_empty
+              abort "No evals defined. Define evals with define_eval or set fail_on_empty = false."
+            else
+              puts "No evals defined."
+              next
+            end
+          end
+
+          gate_passed = true
+          suite_cost = 0.0
+
+          results.each do |host, reports|
+            puts "\n#{host.name || host.to_s}"
+            reports.each_value do |report|
+              report.print_summary
+              suite_cost += report.total_cost
+              gate_passed = false unless report_meets_score?(report)
+            end
+          end
+
+          if @maximum_cost && suite_cost > @maximum_cost
+            abort "\nEval suite FAILED: total cost $#{format("%.4f", suite_cost)} " \
+                  "exceeds budget $#{format("%.4f", @maximum_cost)}"
+          end
+
+          abort "\nEval suite FAILED" unless gate_passed
+          puts "\nAll evals passed."
+        end
+      end
+
+      def report_meets_score?(report)
+        if @minimum_score
+          report.score >= @minimum_score
+        else
+          report.passed?
+        end
+      end
+
+      def task_prerequisites
+        Rake::Task.task_defined?(:environment) ? [:environment] : []
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module RSpec
+      # Helper methods for the pass_eval matcher to keep the block short.
+      module PassEvalHelpers
+        def format_failure_message(eval_name, error, report, minimum_score, maximum_cost)
+          return format_error_message(eval_name, error) if error
+
+          format_report_message(eval_name, report, minimum_score, maximum_cost)
+        end
+
+        def format_error_message(eval_name, error)
+          "expected #{eval_name} eval to pass, but it raised an error:\n  #{error.class}: #{error.message}"
+        end
+
+        def format_report_message(eval_name, report, minimum_score, maximum_cost)
+          lines = build_header(eval_name, report, minimum_score, maximum_cost)
+          lines << ""
+
+          report.results.each do |result|
+            cost_str = result.cost ? " $#{format("%.4f", result.cost)}" : ""
+            lines << "  #{result.label}  #{result.name} (score: #{result.score})#{cost_str}"
+            lines << "        #{result.details}" if result.details && result.failed?
+          end
+
+          lines.join("\n")
+        end
+
+        private
+
+        def build_header(eval_name, report, minimum_score, maximum_cost)
+          cost_str = report.total_cost.positive? ? ", cost: $#{format("%.4f", report.total_cost)}" : ""
+
+          if maximum_cost && report.total_cost > maximum_cost
+            ["expected #{eval_name} eval cost <= $#{format("%.4f", maximum_cost)}, " \
+             "but got: $#{format("%.4f", report.total_cost)} (#{report.pass_rate})"]
+          elsif minimum_score
+            ["expected #{eval_name} eval score >= #{minimum_score}, " \
+             "but got: #{report.score.round(2)} (#{report.pass_rate}#{cost_str})"]
+          else
+            ["expected #{eval_name} eval to pass, " \
+             "but got score: #{report.score.round(2)} (#{report.pass_rate}#{cost_str})"]
+          end
+        end
+      end
+    end
+  end
+end
+
+RSpec::Matchers.define :pass_eval do |eval_name|
+  include RubyLLM::Contract::RSpec::PassEvalHelpers
+
+  chain :with_context do |ctx|
+    @context = ctx
+  end
+
+  chain :with_minimum_score do |score|
+    @minimum_score = score
+  end
+
+  chain :with_maximum_cost do |cost|
+    @maximum_cost = cost
+  end
+
+  match do |step_or_pipeline|
+    @eval_name = eval_name
+    @context ||= {}
+    @minimum_score ||= nil
+    @maximum_cost ||= nil
+    @error = nil
+    @report = step_or_pipeline.run_eval(eval_name, context: @context)
+
+    score_ok = if @minimum_score
+                 @report.score >= @minimum_score
+               else
+                 @report.passed?
+               end
+
+    cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
+
+    score_ok && cost_ok
+  rescue StandardError => e
+    @error = e
+    false
+  end
+
+  failure_message do
+    format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+  end
+
+  failure_message_when_negated do
+    "expected #{@eval_name} eval NOT to pass, but it passed with score: #{@report.score.round(2)}"
+  end
+end

data/lib/ruby_llm/contract/rspec/satisfy_contract.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+RSpec::Matchers.define :satisfy_contract do
+  match do |result|
+    @result = result
+    result.respond_to?(:ok?) && result.ok?
+  end
+
+  failure_message do
+    lines = ["expected step result to satisfy contract, but got status: #{@result.status}"]
+
+    if @result.respond_to?(:validation_errors) && @result.validation_errors.any?
+      lines << ""
+      lines << "Validation errors:"
+      @result.validation_errors.each { |e| lines << "  - #{e}" }
+    end
+
+    if @result.respond_to?(:raw_output) && @result.raw_output
+      output = @result.raw_output.to_s
+      output = "#{output[0, 200]}..." if output.size > 200
+      lines << ""
+      lines << "Raw output: #{output}"
+    end
+
+    lines.join("\n")
+  end
+
+  failure_message_when_negated do
+    "expected step result NOT to satisfy contract, but it passed with status: :ok"
+  end
+end

data/lib/ruby_llm/contract/rspec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "ruby_llm/contract"
+
+require_relative "rspec/satisfy_contract"
+require_relative "rspec/pass_eval"

data/lib/ruby_llm/contract/step/base.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class Base
+        def self.inherited(subclass)
+          super
+          Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
+        end
+
+        class << self
+          include Concerns::EvalHost
+          include RetryExecutor
+          include Dsl
+
+          def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
+            dataset = Eval::Dataset.define("single_case") do
+              add_case("inline", input: input, expected: expected,
+                                 expected_traits: expected_traits, evaluator: evaluator)
+            end
+            report = Eval::Runner.run(step: self, dataset: dataset, context: context)
+            report.results.first
+          end
+
+          def estimate_cost(input:, model: nil)
+            model_name = model || RubyLLM::Contract.configuration.default_model
+            messages = build_messages(input)
+            input_tokens = TokenEstimator.estimate(messages)
+            output_tokens = max_output || 256 # conservative default
+
+            model_info = CostCalculator.send(:find_model, model_name)
+            return nil unless model_info
+
+            estimated = CostCalculator.send(:compute_cost, model_info,
+                                            { input_tokens: input_tokens, output_tokens: output_tokens })
+            {
+              model: model_name,
+              input_tokens: input_tokens,
+              output_tokens_estimate: output_tokens,
+              estimated_cost: estimated
+            }
+          end
+
+          def estimate_eval_cost(eval_name, models: nil)
+            defn = send(:all_eval_definitions)[eval_name.to_s]
+            raise ArgumentError, "No eval '#{eval_name}' defined" unless defn
+
+            model_list = models || [RubyLLM::Contract.configuration.default_model].compact
+            cases = defn.build_dataset.cases
+
+            model_list.each_with_object({}) do |model_name, result|
+              per_case = cases.sum do |c|
+                est = estimate_cost(input: c.input, model: model_name)
+                est ? est[:estimated_cost] : 0.0
+              end
+              result[model_name] = per_case.round(6)
+            end
+          end
+
+          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens schema provider assume_model_exists].freeze
+
+          def run(input, context: {})
+            warn_unknown_context_keys(context)
+            adapter = resolve_adapter(context)
+            default_model = context[:model] || RubyLLM::Contract.configuration.default_model
+            policy = retry_policy
+
+            if policy
+              run_with_retry(input, adapter: adapter, default_model: default_model, policy: policy)
+            else
+              run_once(input, adapter: adapter, model: default_model)
+            end
+          end
+
+          private
+
+          def warn_unknown_context_keys(context)
+            unknown = context.keys - KNOWN_CONTEXT_KEYS
+            return if unknown.empty?
+
+            warn "[ruby_llm-contract] Unknown context keys: #{unknown.inspect}. " \
+                 "Known keys: #{KNOWN_CONTEXT_KEYS.inspect}"
+          end
+
+          def resolve_adapter(context)
+            adapter = context[:adapter] || RubyLLM::Contract.configuration.default_adapter
+            return adapter if adapter
+
+            raise RubyLLM::Contract::Error, "No adapter configured. Set one with RubyLLM::Contract.configure " \
+                                            "{ |c| c.default_adapter = ... } or pass context: { adapter: ... }"
+          end
+
+          def run_once(input, adapter:, model:)
+            Runner.new(
+              input_type: input_type, output_type: output_type,
+              prompt_block: prompt, contract_definition: effective_contract,
+              adapter: adapter, model: model, output_schema: output_schema,
+              max_output: max_output, max_input: max_input, max_cost: max_cost
+            ).call(input)
+          rescue ArgumentError => e
+            Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
+                       validation_errors: [e.message])
+          end
+
+          def effective_contract
+            base = contract
+            extra = class_validates
+            inferred_parse = json_compatible_type?(output_type) ? :json : nil
+
+            return base if extra.empty? && inferred_parse.nil?
+
+            has_own_contract = defined?(@contract_definition) && @contract_definition
+            Definition.merge(
+              base,
+              extra_invariants: extra,
+              parse_override: inferred_parse && !has_own_contract ? inferred_parse : nil
+            )
+          end
+
+          def build_messages(input)
+            dynamic = prompt.arity >= 1
+            ast = Prompt::Builder.build(input: dynamic ? input : nil, &prompt)
+            variables = dynamic ? {} : { input: input }
+            variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
+            Prompt::Renderer.render(ast, variables: variables)
+          end
+
+          def json_compatible_type?(type)
+            type == RubyLLM::Contract::Types::Hash || type == Hash ||
+              type == RubyLLM::Contract::Types::Array || type == Array ||
+              (type.respond_to?(:name) && type.name&.match?(/Hash|Array/))
+          end
+        end
+      end
+    end
+  end
+end