ruby_llm-contract 0.2.0

data/lib/ruby_llm/contract/eval/runner.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class Runner
+        include TraitEvaluator
+        include ContractDetailBuilder
+
+        def self.run(step:, dataset:, context: {})
+          new(step: step, dataset: dataset, context: context).run
+        end
+
+        def initialize(step:, dataset:, context: {})
+          @step = step
+          @dataset = dataset
+          @context = context
+        end
+
+        def run
+          results = @dataset.cases.map { |test_case| evaluate_case(test_case) }
+          Report.new(dataset_name: @dataset.name, results: results)
+        end
+
+        private
+
+        def evaluate_case(test_case)
+          run_result = @step.run(test_case.input, context: @context)
+          step_result = normalize_result(run_result)
+          eval_result = dispatch_evaluation(step_result, test_case)
+
+          build_case_result(test_case, step_result, eval_result)
+        rescue RubyLLM::Contract::Error => e
+          # No adapter configured — skip this case (offline mode without sample_response)
+          skipped_result(test_case, e.message)
+        end
+
+        def build_case_result(test_case, step_result, eval_result)
+          trace = step_result.respond_to?(:trace) ? step_result.trace : nil
+          CaseResult.new(
+            name: test_case.name,
+            input: test_case.input,
+            output: step_result.parsed_output,
+            expected: test_case.expected,
+            step_status: step_result.status,
+            score: eval_result.score,
+            passed: eval_result.passed,
+            label: eval_result.label,
+            details: eval_result.details,
+            duration_ms: extract_latency(trace),
+            cost: extract_cost(trace)
+          )
+        end
+
+        def extract_latency(trace)
+          return nil unless trace
+
+          # Pipeline::Trace uses total_latency_ms, Step::Trace uses latency_ms
+          if trace.respond_to?(:total_latency_ms)
+            trace.total_latency_ms
+          else
+            trace[:latency_ms]
+          end
+        end
+
+        def extract_cost(trace)
+          return nil unless trace
+
+          # Pipeline::Trace uses total_cost, Step::Trace uses cost
+          if trace.respond_to?(:total_cost)
+            trace.total_cost
+          else
+            trace[:cost]
+          end
+        end
+
+        def dispatch_evaluation(step_result, test_case)
+          return contract_failure(step_result) unless step_result.ok?
+
+          if test_case.evaluator
+            evaluate_with_custom(step_result, test_case)
+          elsif test_case.expected_traits
+            evaluate_traits(step_result, test_case)
+          elsif test_case.expected
+            evaluate_expected(step_result, test_case)
+          else
+            evaluate_contract_only
+          end
+        end
+
+        def normalize_result(result)
+          return result if result.respond_to?(:parsed_output)
+
+          normalize_pipeline_result(result)
+        end
+
+        def normalize_pipeline_result(result)
+          last_result = result.step_results&.last&.dig(:result)
+          is_ok = result.ok?
+          pipeline_trace = result.respond_to?(:trace) ? result.trace : nil
+
+          PipelineResultAdapter.new(
+            status: result.status,
+            ok_flag: is_ok,
+            parsed_output: is_ok ? result.outputs_by_step.values.last : nil,
+            validation_errors: last_result.respond_to?(:validation_errors) ? last_result.validation_errors : [],
+            trace: pipeline_trace || (last_result.respond_to?(:trace) ? last_result.trace : {})
+          )
+        end
+
+        def evaluate_expected(step_result, test_case)
+          dispatch_expected_evaluator(
+            output: step_result.parsed_output,
+            expected: test_case.expected,
+            input: test_case.input
+          )
+        end
+
+        def dispatch_expected_evaluator(output:, expected:, input:)
+          if expected.is_a?(Hash)
+            Evaluator::JsonIncludes.new.call(output: output, expected: expected, input: input)
+          elsif expected.is_a?(::Regexp)
+            Evaluator::Regex.new(expected).call(output: output, input: input)
+          else
+            Evaluator::Exact.new.call(output: output, expected: expected, input: input)
+          end
+        end
+
+        def evaluate_with_custom(step_result, test_case)
+          evaluator = test_case.evaluator
+          evaluator = Evaluator::ProcEvaluator.new(evaluator) if evaluator.is_a?(::Proc)
+          evaluator.call(output: step_result.parsed_output, expected: test_case.expected, input: test_case.input)
+        end
+
+        def evaluate_contract_only
+          EvaluationResult.new(score: 1.0, passed: true, details: build_contract_details)
+        end
+
+        def contract_failure(step_result)
+          EvaluationResult.new(
+            score: 0.0, passed: false,
+            details: "step failed: #{step_result.status} — #{step_result.validation_errors.join(", ")}"
+          )
+        end
+
+        def skipped_result(test_case, reason)
+          CaseResult.new(
+            name: test_case.name,
+            input: test_case.input,
+            output: nil,
+            expected: test_case.expected,
+            step_status: :skipped,
+            score: 0.0,
+            passed: false,
+            label: "SKIP",
+            details: "skipped: #{reason}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/trait_evaluator.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Extracted from Runner to reduce class length.
+      # Evaluates expected_traits against parsed output.
+      module TraitEvaluator
+        private
+
+        def evaluate_traits(step_result, test_case)
+          output = step_result.parsed_output
+          traits = test_case.expected_traits
+          errors = traits.each_with_object([]) do |(key, expectation), errs|
+            check_trait(output, key, expectation, errs)
+          end
+
+          build_trait_result(errors, traits.length)
+        end
+
+        def check_trait(output, key, expectation, errors)
+          value = output.is_a?(Hash) ? output[key] : nil
+          error_msg = trait_error(key, value, expectation)
+          errors << error_msg if error_msg
+        end
+
+        def trait_error(key, value, expectation)
+          case expectation
+          when ::Regexp
+            trait_regexp_error(key, value, expectation)
+          when Range
+            trait_range_error(key, value, expectation)
+          when true
+            trait_truthy_error(key, value)
+          when false
+            trait_falsy_error(key, value)
+          else
+            trait_equality_error(key, value, expectation)
+          end
+        end
+
+        def trait_regexp_error(key, value, expectation)
+          "#{key}: does not match #{expectation.inspect}" unless value.to_s.match?(expectation)
+        end
+
+        def trait_range_error(key, value, expectation)
+          comparable = value.is_a?(Numeric) ? value : value.to_s.length
+          "#{key}: #{value.inspect} not in #{expectation}" unless expectation.include?(comparable)
+        end
+
+        def trait_truthy_error(key, value)
+          "#{key}: expected truthy, got #{value.inspect}" unless value
+        end
+
+        def trait_falsy_error(key, value)
+          "#{key}: expected falsy, got #{value.inspect}" if value
+        end
+
+        def trait_equality_error(key, value, expectation)
+          "#{key}: expected #{expectation.inspect}, got #{value.inspect}" unless value == expectation
+        end
+
+        def build_trait_result(errors, trait_count)
+          if errors.empty?
+            EvaluationResult.new(score: 1.0, passed: true, details: "all traits match")
+          else
+            matched = trait_count - errors.length
+            score = trait_count.zero? ? 0.0 : matched.to_f / trait_count
+            EvaluationResult.new(score: score, passed: false, details: errors.join("; "))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "eval/evaluation_result"
+require_relative "eval/case_result"
+require_relative "eval/evaluator/exact"
+require_relative "eval/evaluator/regex"
+require_relative "eval/evaluator/json_includes"
+require_relative "eval/evaluator/proc_evaluator"
+require_relative "eval/dataset"
+require_relative "eval/pipeline_result_adapter"
+require_relative "eval/trait_evaluator"
+require_relative "eval/contract_detail_builder"
+require_relative "eval/runner"
+require_relative "eval/report"
+require_relative "eval/eval_definition"
+require_relative "eval/model_comparison"

data/lib/ruby_llm/contract/pipeline/base.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Pipeline
+      class Base
+        def self.inherited(subclass)
+          super
+          Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
+        end
+
+        class << self
+          include Concerns::EvalHost
+
+          # depends_on is accepted for forward compatibility with DAG pipelines (v0.3).
+          # Currently, execution is always linear in declaration order.
+          def step(step_class, as:, depends_on: nil, model: nil)
+            validate_dependency!(depends_on) if depends_on
+            steps_registry << { step_class: step_class, alias: as, depends_on: depends_on, model: model }
+          end
+
+          def steps
+            steps_registry.dup.freeze
+          end
+
+          # Internal mutable steps list for registration
+          def steps_registry
+            @steps_registry ||= []
+          end
+
+          def token_budget(limit = nil)
+            return @token_budget = limit if limit
+
+            @token_budget
+          end
+
+          def run(input, context: {}, timeout_ms: nil)
+            Runner.new(steps: steps, context: context, timeout_ms: timeout_ms, token_budget: token_budget).call(input)
+          end
+
+          def test(input, responses: {}, timeout_ms: nil)
+            ordered_responses = steps.map { |step_entry| responses.fetch(step_entry[:alias], "") }
+            adapter = Adapters::Test.new(responses: ordered_responses)
+            run(input, context: { adapter: adapter }, timeout_ms: timeout_ms)
+          end
+
+          private
+
+          def known_step_aliases
+            steps_registry.map { |step_entry| step_entry[:alias] }
+          end
+
+          def validate_dependency!(dep)
+            return if known_step_aliases.include?(dep)
+
+            raise ArgumentError, "Unknown dependency: #{dep.inspect}. Known steps: #{known_step_aliases.inspect}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/pipeline/result.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Pipeline
+      class Result
+        attr_reader :status, :step_results, :outputs_by_step, :failed_step, :trace
+
+        # Column widths for pretty_print table
+        COL1 = 14   # step name
+        COL2 = 10   # status
+        COL3 = 50   # output
+
+        TOP_BORDER = "+#{"-" * (COL1 + COL2 + COL3 + 8)}+".freeze
+        MID_BORDER = "+-#{"-" * COL1}-+-#{"-" * COL2}-+-#{"-" * COL3}-+".freeze
+
+        def initialize(status:, step_results:, outputs_by_step:, failed_step: nil, trace: Trace.new)
+          @status = status
+          @step_results = step_results.each(&:freeze).freeze
+          @outputs_by_step = outputs_by_step.freeze
+          @failed_step = failed_step
+          @trace = trace
+          freeze
+        end
+
+        def ok?
+          @status == :ok
+        end
+
+        def failed?
+          @status != :ok
+        end
+
+        def to_s
+          lines = [header_line]
+          @step_results.each { |sr| lines << step_line(sr) }
+          lines.join("\n")
+        end
+
+        def pretty_print(io = $stdout)
+          build_table.each { |line| io.puts line }
+        end
+
+        private
+
+        def build_table
+          header_width = COL1 + COL2 + COL3 + 2
+          [TOP_BORDER,
+           "| #{header_line.ljust(header_width)} |",
+           MID_BORDER,
+           "| #{"Step".ljust(COL1)} | #{"Status".ljust(COL2)} | #{"Output".ljust(COL3)} |",
+           MID_BORDER,
+           *build_step_rows,
+           TOP_BORDER]
+        end
+
+        def build_step_rows
+          rows = []
+          @step_results.each_with_index do |sr, idx|
+            rows.concat(build_single_step_rows(sr))
+            rows << MID_BORDER if idx < @step_results.size - 1
+          end
+          rows
+        end
+
+        def build_single_step_rows(step_record)
+          step_alias = step_record[:alias].to_s
+          status_str = step_status(step_record[:result])
+          output_lines = format_output(@outputs_by_step[step_record[:alias]])
+          first_row = build_first_step_row(step_alias, status_str, output_lines.first || "")
+          continuation_rows = build_continuation_rows(output_lines.drop(1))
+
+          [first_row, *continuation_rows]
+        end
+
+        def build_first_step_row(step_alias, status_str, first_line)
+          "| #{step_alias.ljust(COL1)} | #{status_str.ljust(COL2)} | #{first_line.ljust(COL3)} |"
+        end
+
+        def build_continuation_rows(lines)
+          blank_prefix = "| #{" " * COL1} | #{" " * COL2} | "
+          lines.map { |line| "#{blank_prefix}#{line.ljust(COL3)} |" }
+        end
+
+        def header_line
+          parts = ["Pipeline: #{@status}"]
+          append_trace_details(parts) if @trace
+          parts.join("  ")
+        end
+
+        def append_trace_details(parts)
+          parts << "#{@step_results.size} steps"
+          parts << "#{@trace.total_latency_ms}ms" if @trace.total_latency_ms
+          append_usage_details(parts)
+          parts << "$#{format("%.6f", @trace.total_cost)}" if @trace.total_cost
+          parts << "trace=#{@trace.trace_id&.slice(0, 8)}" if @trace.trace_id
+        end
+
+        def append_usage_details(parts)
+          usage = @trace.total_usage
+          return unless usage.is_a?(Hash)
+
+          parts << "#{usage[:input_tokens]}+#{usage[:output_tokens]} tokens"
+        end
+
+        def step_line(step_record)
+          step_result = step_record[:result]
+          trace = step_result.trace
+          status = step_status(step_result)
+          trace_str = trace.respond_to?(:to_s) ? trace.to_s : ""
+          "  #{step_record[:alias].to_s.ljust(14)} #{status.ljust(10)} #{trace_str}"
+        end
+
+        def step_status(step_result)
+          step_result.ok? ? "ok" : step_result.status.to_s
+        end
+
+        def format_output(output)
+          return ["(no output)"] unless output
+
+          pairs = output.is_a?(Hash) ? output : { value: output }
+          pairs.map do |key, val|
+            str = val.is_a?(String) ? val : val.inspect
+            line = "#{key}: #{str}"
+            line.size > COL3 ? "#{line[0, COL3 - 3]}..." : line
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/pipeline/runner.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module RubyLLM
+  module Contract
+    module Pipeline
+      class Runner
+        include Concerns::UsageAggregator
+
+        def initialize(steps:, context:, timeout_ms: nil, token_budget: nil)
+          raise ArgumentError, "timeout_ms must be positive (got #{timeout_ms})" if timeout_ms && timeout_ms <= 0
+          raise ArgumentError, "Pipeline has no steps defined" if steps.empty?
+
+          @steps = steps
+          @context = context
+          @timeout_ms = timeout_ms
+          @token_budget = token_budget
+        end
+
+        def call(input)
+          execution = ExecutionState.new(input)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          run_steps(execution, start_time)
+          finalize_result(execution, start_time)
+        end
+
+        def run_steps(execution, start_time)
+          @steps.each_with_index do |step_def, index|
+            execute_step(step_def, execution)
+            break if execution.failed?
+            break if check_limits(index, step_def, execution, start_time)
+          end
+        end
+
+        private
+
+        def execute_step(step_def, execution)
+          step_context = build_step_context(step_def)
+          result = step_def[:step_class].run(execution.current_input, context: step_context)
+
+          execution.record_step(step_def[:alias], result)
+        end
+
+        def build_step_context(step_def)
+          model = step_def[:model]
+          model ? @context.merge(model: model) : @context
+        end
+
+        def check_limits(index, step_def, execution, start_time)
+          limit_status = detect_limit_violation(execution, start_time)
+          return unless limit_status
+
+          failing_alias = next_step_alias(index, step_def)
+          execution.mark_limit_failure(limit_status, failing_alias)
+          true
+        end
+
+        # NOTE: This is a cooperative timeout, not a hard deadline. The timeout is
+        # checked between steps, after each step completes. A slow step (e.g. long
+        # LLM call or multi-attempt retry) can exceed the deadline before the check
+        # runs. This is a known architectural limitation -- safely interrupting a
+        # running HTTP call in Ruby requires threads/fibers, which adds significant
+        # complexity. For most pipelines this cooperative approach is sufficient;
+        # set timeout_ms with enough headroom for your slowest expected step.
+        def detect_limit_violation(execution, start_time)
+          if @timeout_ms && elapsed_ms(start_time) >= @timeout_ms
+            :timeout
+          elsif @token_budget && sum_tokens(execution.step_traces) > @token_budget
+            :budget_exceeded
+          end
+        end
+
+        def next_step_alias(index, step_def)
+          @steps[index + 1]&.dig(:alias) || step_def[:alias]
+        end
+
+        def finalize_result(execution, start_time)
+          traces = execution.step_traces
+          trace = Trace.new(
+            trace_id: execution.trace_id,
+            total_latency_ms: elapsed_ms(start_time),
+            total_usage: aggregate_usage(traces),
+            step_traces: traces
+          )
+
+          Result.new(
+            status: execution.status, step_results: execution.step_results,
+            outputs_by_step: execution.outputs_by_step, failed_step: execution.failed_step,
+            trace: trace
+          )
+        end
+
+        def elapsed_ms(start_time)
+          ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+        end
+
+        # Encapsulates mutable state during pipeline execution
+        class ExecutionState
+          attr_reader :trace_id, :step_results, :step_traces, :outputs_by_step,
+                      :current_input, :status, :failed_step
+
+          def initialize(input)
+            @trace_id = SecureRandom.uuid
+            @step_results = []
+            @step_traces = []
+            @outputs_by_step = {}
+            @current_input = input
+            @status = :ok
+            @failed_step = nil
+          end
+
+          def record_step(step_alias, result)
+            @step_results << { alias: step_alias, result: result }
+            @step_traces << result.trace
+
+            if result.ok?
+              output = result.parsed_output
+              @outputs_by_step[step_alias] = output
+              @current_input = output
+            else
+              @status = result.status
+              @failed_step = step_alias
+            end
+          end
+
+          def mark_limit_failure(status, failed_alias)
+            @status = status
+            @failed_step = failed_alias
+          end
+
+          def failed?
+            @status != :ok
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/pipeline/trace.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Pipeline
+      class Trace
+        include Concerns::TraceEquality
+
+        attr_reader :trace_id, :total_latency_ms, :total_usage, :step_traces, :total_cost
+
+        def initialize(trace_id: nil, total_latency_ms: nil, total_usage: nil, step_traces: nil)
+          @trace_id = trace_id
+          @total_latency_ms = total_latency_ms
+          @total_usage = total_usage
+          @step_traces = step_traces
+          @total_cost = calculate_total_cost
+          freeze
+        end
+
+        KNOWN_KEYS = %i[trace_id total_latency_ms total_usage step_traces total_cost].freeze
+
+        def [](key)
+          return nil unless KNOWN_KEYS.include?(key.to_sym)
+
+          public_send(key)
+        end
+
+        def to_h
+          { trace_id: @trace_id, total_latency_ms: @total_latency_ms,
+            total_usage: @total_usage, step_traces: @step_traces,
+            total_cost: @total_cost }.compact
+        end
+
+        def to_s
+          build_summary_parts.join(" ")
+        end
+
+        private
+
+        def build_summary_parts
+          parts = ["trace=#{@trace_id&.slice(0, 8)}"]
+          parts << "#{@total_latency_ms}ms" if @total_latency_ms
+          parts << format_token_usage if @total_usage.is_a?(Hash)
+          parts << "$#{format("%.6f", @total_cost)}" if @total_cost
+          parts << "(#{step_count} steps)"
+          parts
+        end
+
+        def format_token_usage
+          "#{@total_usage[:input_tokens] || 0}+#{@total_usage[:output_tokens] || 0} tokens"
+        end
+
+        def step_count
+          @step_traces.is_a?(Array) ? @step_traces.size : 0
+        end
+
+        def calculate_total_cost
+          return nil unless @step_traces.is_a?(Array)
+
+          costs = collect_step_costs
+          return nil if costs.empty?
+
+          costs.sum.round(6)
+        end
+
+        def collect_step_costs
+          @step_traces.filter_map { |step_trace| step_trace.respond_to?(:cost) ? step_trace.cost : nil }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/pipeline.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require_relative "pipeline/trace"
+require_relative "pipeline/result"
+require_relative "pipeline/runner"
+require_relative "pipeline/base"