ruby_llm-agent-evaluations 0.2.0

data/lib/ruby_llm/evaluations/report.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "json"
+
+module RubyLLM
+  module Evaluations
+    class Report
+      def initialize(run, baseline: nil, tolerance: 0.0)
+        unless tolerance.is_a?(Numeric) && tolerance.finite? && tolerance >= 0
+          raise ArgumentError, "tolerance must be a finite nonnegative number"
+        end
+        raise ArgumentError, "run must be a RubyLLM::Evaluations::Run" unless run.is_a?(Run)
+        raise ArgumentError, "baseline must be a RubyLLM::Evaluations::Run" if baseline && !baseline.is_a?(Run)
+        @run = run.snapshot
+        @baseline = baseline&.snapshot
+        @tolerance = tolerance.to_f
+        if @baseline && %w[suite agent suite_digest].any? { |key| @run.fetch(key) != @baseline.fetch(key) }
+          raise ArgumentError, "baseline must have the same suite, agent, and suite digest"
+        end
+        if baseline && ([baseline.started_at, baseline.sequence] <=> [run.started_at, run.sequence]) >= 0
+          raise ArgumentError, "baseline must be earlier than the current run"
+        end
+        @comparison = compare if @baseline
+      end
+
+      def comparison
+        @comparison
+      end
+
+      attr_reader :run
+      def export(path, format: nil)
+        path = File.path(path)
+        format ||= { ".html" => "html", ".htm" => "html", ".json" => "json", ".md" => "markdown",
+                     ".markdown" => "markdown", ".txt" => "text", ".text" => "text" }[File.extname(path).downcase]
+        unless %w[text markdown html json].include?(format.to_s)
+          raise ArgumentError, "report format must be text, markdown, html, or json"
+        end
+        File.write(path, public_send("to_#{format}"))
+        path
+      end
+
+      def to_json(*args)
+        @run.merge("comparison" => comparison).to_json(*args)
+      end
+
+      def to_text
+        sections.map do |title, values|
+          ([title, "=" * title.length] + values.map do |label, value|
+            "#{label}: #{display_value(value).gsub("\n", "\n  ")}"
+          end).join("\n")
+        end.join("\n\n") + "\n"
+      end
+
+      def to_markdown
+        sections.map.with_index do |(title, values), index|
+          heading = "#{index.zero? ? '#' : '##'} #{markdown_escape(title)}"
+          body = values.map do |label, value|
+            text = display_value(value)
+            if value.is_a?(Hash) || value.is_a?(Array) || text.include?("\n")
+              fence = "`" * [3, (text.scan(/`+/).map(&:length).max || 0) + 1].max
+              "**#{markdown_escape(label)}**\n\n#{fence}\n#{text}\n#{fence}"
+            else
+              "**#{markdown_escape(label)}:** #{markdown_escape(text)}"
+            end
+          end.join("\n\n")
+          "#{heading}\n\n#{body}"
+        end.join("\n\n") + "\n"
+      end
+
+      def to_html
+        body = sections.map.with_index do |(title, values), index|
+          level = index.zero? ? 1 : 2
+          entries = values.map do |label, value|
+            "<dt>#{html_escape(label)}</dt><dd><pre>#{html_escape(display_value(value))}</pre></dd>"
+          end.join
+          "<section><h#{level}>#{html_escape(title)}</h#{level}><dl>#{entries}</dl></section>"
+        end.join("\n")
+        <<~HTML
+          <!doctype html>
+          <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+          <title>#{html_escape(@run.fetch("suite"))} — Evaluation report</title>
+          <style>
+            :root { color-scheme: light dark; font-family: system-ui, sans-serif; }
+            body { max-width: 960px; margin: 2rem auto; padding: 0 1rem; line-height: 1.5; }
+            section { border-bottom: 1px solid #888; padding-bottom: 1rem; }
+            dt { font-weight: 650; margin-top: 1rem; }
+            dd { margin: .25rem 0 0; }
+            pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0; font: inherit; }
+          </style></head><body>#{body}</body></html>
+        HTML
+      end
+
+      private
+
+      def compare
+        regressions = []
+        improvements = []
+        previous = @baseline.fetch("cases").to_h { |item| [item.fetch("id"), item] }
+        current = @run.fetch("cases").to_h { |item| [item.fetch("id"), item] }
+        (previous.keys | current.keys).each do |id|
+          before = previous[id]
+          after = current[id]
+          if after.nil?
+            regressions << change(before, nil, nil, case_score(before), nil, "Case result is missing")
+          elsif before.nil?
+            improvements << change(nil, after, nil, nil, case_score(after), "Case result was added")
+          else
+            compare_case(before, after, regressions, improvements)
+          end
+        end
+        baseline_summary = @baseline.fetch("summary")
+        current_summary = @run.fetch("summary")
+        {
+          "baseline_id" => @baseline.fetch("id"), "current_id" => @run.fetch("id"),
+          "compatible" => true, "regressed" => !regressions.empty?,
+          "pass_rate_delta" => score_delta(baseline_summary["pass_rate"], current_summary["pass_rate"]),
+          "score_delta" => score_delta(baseline_summary["score"], current_summary["score"]),
+          "regressions" => regressions, "improvements" => improvements
+        }
+      end
+
+      def compare_case(before, after, regressions, improvements)
+        if agent_error?(after) && !agent_error?(before)
+          regressions << change(before, after, nil, case_score(before), 0.0, "Agent execution began raising an error")
+          return
+        elsif agent_error?(before) && !agent_error?(after)
+          improvements << change(before, after, nil, 0.0, case_score(after), "Agent execution recovered from an error")
+          return
+        elsif agent_error?(before) && agent_error?(after)
+          return
+        end
+
+        old_evaluations = before.fetch("metrics").to_h { |item| [item.fetch("name"), item] }
+        new_evaluations = after.fetch("metrics").to_h { |item| [item.fetch("name"), item] }
+        regression_count = regressions.length
+        improvement_count = improvements.length
+        (old_evaluations.keys | new_evaluations.keys).each do |name|
+          old_result = old_evaluations[name]
+          new_result = new_evaluations[name]
+          if new_result.nil?
+            regressions << change(before, after, name, evaluation_score(old_result), nil, "Evaluator result is missing")
+            next
+          elsif old_result.nil?
+            target = new_result["passed"] ? improvements : regressions
+            target << change(before, after, name, nil, evaluation_score(new_result), "Evaluator result was added #{new_result['passed'] ? 'and passed' : 'without passing'}")
+            next
+          end
+
+          old_score = evaluation_score(old_result)
+          new_score = evaluation_score(new_result)
+          reason = nil
+          target = nil
+          if new_result["error"] && !old_result["error"]
+            reason, target = "Evaluator began raising an error", regressions
+          elsif old_result["error"] && !new_result["error"]
+            reason, target = "Evaluator recovered from an error", improvements
+          elsif old_result["passed"] && !new_result["passed"]
+            reason, target = "Evaluator changed from passing to failing", regressions
+          elsif !old_result["passed"] && new_result["passed"]
+            reason, target = "Evaluator changed from failing to passing", improvements
+          elsif old_score && new_score && old_score - new_score > @tolerance
+            reason, target = "Evaluator score decreased beyond tolerance", regressions
+          elsif old_score && new_score && new_score - old_score > @tolerance
+            reason, target = "Evaluator score increased beyond tolerance", improvements
+          end
+          target << change(before, after, name, old_score, new_score, reason) if target
+        end
+        # A case outcome without a changed evaluator is still evidence; do not duplicate evaluator failures.
+        if before["status"] == "passed" && after["status"] != "passed" && regressions.length == regression_count
+          regressions << change(before, after, nil, case_score(before), case_score(after), "Case changed from passing to #{after['status']}")
+        elsif before["status"] != "passed" && after["status"] == "passed" && improvements.length == improvement_count
+          improvements << change(before, after, nil, case_score(before), case_score(after), "Case changed from #{before['status']} to passing")
+        end
+      end
+
+      def agent_error?(item)
+        !item["error"].nil?
+      end
+
+      def evaluation_score(item)
+        item["error"] ? 0.0 : item["score"]
+      end
+
+      def case_score(item)
+        return 0.0 if agent_error?(item)
+        return nil if item["status"] == "not_evaluated"
+
+        scores = item.fetch("metrics").map { |evaluation| evaluation_score(evaluation) || 0.0 }
+        scores.empty? ? (item["status"] == "passed" ? 1.0 : 0.0) : scores.sum.to_f / scores.length
+      end
+      def score_delta(before, after)
+        after - before unless before.nil? || after.nil?
+      end
+
+      def change(before, after, evaluator, old_score, new_score, reason)
+        item = after || before
+        {
+          "case_id" => item.fetch("id"), "case_name" => item.fetch("name"), "evaluator" => evaluator,
+          "baseline_score" => old_score, "current_score" => new_score,
+          "delta" => old_score && new_score ? new_score - old_score : nil, "reason" => reason
+        }
+      end
+
+      def sections
+        result = [["#{@run.fetch('suite')} — Evaluation report", {
+          "Run" => @run.fetch("id"), "Agent" => @run.fetch("agent"),
+          "Suite digest" => @run.fetch("suite_digest"), "Status" => @run.fetch("status"),
+          "Started at" => @run.fetch("started_at"), "Finished at" => @run.fetch("finished_at"),
+          "Metadata" => @run.fetch("metadata")
+        }], ["Summary", @run.fetch("summary")]]
+        if comparison
+          result << ["Baseline comparison", {
+            "Baseline run" => comparison.fetch("baseline_id"), "Regressed" => comparison.fetch("regressed"),
+            "Pass rate delta" => comparison.fetch("pass_rate_delta"), "Score delta" => comparison.fetch("score_delta"),
+            "Regressions" => comparison.fetch("regressions"), "Improvements" => comparison.fetch("improvements")
+          }]
+        end
+        @run.fetch("cases").each do |item|
+          result << ["Case: #{item.fetch('name')}", {
+            "ID" => item.fetch("id"), "Status" => item.fetch("status"),
+            "Question" => item["question"], "Expected answer" => item["expected_answer"], "Output" => item["output"],
+            "Minimum passing scores" => item["minimum_scores"],
+            "Expected actions (ordered)" => item["expected_actions"] || "Unavailable — expected actions not specified",
+            "Observed actions (ordered)" => item["actions"] || "Unavailable — action instrumentation missing",
+            "Response metadata" => item["response_metadata"].nil? ? "Unavailable — response metadata not reported" : item["response_metadata"],
+            "Metadata" => item["metadata"], "Duration (ms)" => item["duration_ms"],
+            "Input tokens" => item["input_tokens"], "Output tokens" => item["output_tokens"],
+            "Cost" => item["cost"], "Error" => item["error"], "Metrics" => item.fetch("metrics")
+          }]
+        end
+        result
+      end
+
+      def display_value(value)
+        value.is_a?(String) ? value : JSON.pretty_generate(value)
+      end
+
+      def html_escape(value)
+        CGI.escapeHTML(value.to_s)
+      end
+
+      def markdown_escape(value)
+        html_escape(value).gsub(/([\\`*_{}\[\]()#+.!|>~-])/) { |character| "\\#{character}" }
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluations/response.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Evaluations
+    class Response
+      attr_reader :output, :input_tokens, :output_tokens, :cost, :metadata, :actions
+
+      def initialize(output:, input_tokens: nil, output_tokens: nil, cost: nil, metadata: {}, actions: nil)
+        @output = output
+        @input_tokens = tokens(input_tokens)
+        @output_tokens = tokens(output_tokens)
+        unless cost.nil? || (cost.is_a?(Numeric) && cost.finite? && cost >= 0)
+          raise ArgumentError, "cost must be a finite nonnegative number"
+        end
+        raise ArgumentError, "response metadata must be a Hash" unless metadata.is_a?(Hash)
+        @cost = cost&.to_f
+        @metadata = Data.json_value(metadata)
+        @actions = actions.nil? ? nil : Actions.normalize(actions)
+      end
+
+      def self.normalize(value)
+        return value if value.is_a?(self)
+        return new(output: value) unless value.respond_to?(:content)
+
+        new(output: value.content,
+            input_tokens: value.respond_to?(:input_tokens) ? value.input_tokens : nil,
+            output_tokens: value.respond_to?(:output_tokens) ? value.output_tokens : nil,
+            metadata: message_metadata(value))
+      end
+
+      def self.message_metadata(value)
+        %i[model_id role cached_tokens cache_creation_tokens thinking_tokens].each_with_object({}) do |field, metadata|
+          next unless value.respond_to?(field)
+          item = value.public_send(field)
+          metadata[field] = item unless item.nil?
+        end
+      end
+
+      private
+
+      def tokens(value)
+        unless value.nil? || (value.is_a?(Integer) && value >= 0)
+          raise ArgumentError, "token counts must be nonnegative integers"
+        end
+        value
+      end
+    end
+
+  end
+end

data/lib/ruby_llm/evaluations/runner.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Evaluations
+    class Runner
+      def run(suite, metadata: {})
+        suite.validate!
+        Run.require_postgresql!
+        raise Error, "Apply evaluation migrations before running a suite" unless Run.table_exists?
+        raise ArgumentError, "run metadata must be a Hash" unless metadata.is_a?(Hash)
+        run_metadata = Data.json_value(metadata)
+        digest = suite.digest
+        started_at = Time.now.utc
+        started = clock
+        # Finish all agent and evaluator calls before building/saving the database graph.
+        results = suite.cases.map { |question| run_case(suite, question) }
+        run = Run.new(suite: suite.name, agent: suite.agent_name, version: suite.version,
+                      suite_digest: digest, started_at: started_at, finished_at: Time.now.utc,
+                      status: "completed", metadata: run_metadata,
+                      summary: summarize(results, milliseconds(started)))
+        results.each_with_index do |result, position|
+          attributes = result.except("id", "metrics", "error")
+          record = run.case_results.build(attributes.merge("case_id" => result.fetch("id"),
+                                                           "error_details" => result["error"],
+                                                           "position" => position))
+          result.fetch("metrics").each_with_index do |metric, metric_position|
+            record.metric_results.build(metric.except("error").merge("error_details" => metric["error"],
+                                                                     "position" => metric_position))
+          end
+        end
+        # ActiveRecord save! atomically saves the autosaved associations in its short transaction.
+        run.save!
+        run.reload
+      end
+
+      private
+
+      def run_case(suite, question)
+        result = Data.json_value(question.to_h).merge(
+          "output" => nil, "status" => "error", "input_tokens" => nil,
+          "output_tokens" => nil, "cost" => nil, "actions" => nil, "error" => nil, "metrics" => []
+        )
+        started = clock
+        replay = ApiReplay.new(question.api_interactions) if question.api_interactions
+        begin
+          invoke = -> { suite.agent.call(mutable_copy(question.question)) }
+          response = Response.normalize(replay ? replay.call(&invoke) : invoke.call)
+          result.merge!("output" => Data.json_value(response.output),
+                        "input_tokens" => response.input_tokens, "output_tokens" => response.output_tokens,
+                        "cost" => response.cost, "response_metadata" => response.metadata,
+                        "actions" => Data.json_value(response.actions))
+        rescue StandardError => exception
+          result["error"] = Data.error(exception)
+          result["duration_ms"] = milliseconds(started)
+          return result
+        ensure
+          if replay
+            result["response_metadata"] = (result["response_metadata"] || {}).merge("api_replay" => replay.evidence)
+          end
+        end
+        result["duration_ms"] = milliseconds(started)
+        context = Context.new(question, response, judge: suite.judge)
+        result["metrics"] = question.evaluators.map { |evaluator| assess(evaluator, response.output, context) }
+        result["status"] = if result["metrics"].empty?
+                             "not_evaluated"
+                           elsif result["metrics"].any? { |item| item["error"] }
+                             "error"
+                           elsif result["metrics"].all? { |item| item["passed"] }
+                             "passed"
+                           else
+                             "failed"
+                           end
+        result
+      end
+
+      def assess(evaluator, output, context)
+        result = { "name" => evaluator.name, "threshold" => evaluator.threshold,
+                   "score" => nil, "passed" => false, "reason" => nil, "error" => nil }
+        begin
+          value = evaluator.callable.call(output, context)
+          assessment = value.is_a?(Assessment) ? value : Assessment.new(score: value)
+          score = case assessment.score
+                  when true then 1.0
+                  when false then 0.0
+                  else assessment.score
+                  end
+          unless score.is_a?(Numeric) && score.finite? && score.between?(0, 1)
+            raise ArgumentError, "evaluator must return a boolean or a finite score between 0 and 1"
+          end
+          unless assessment.reason.nil? || assessment.reason.is_a?(String)
+            raise ArgumentError, "assessment reason must be a string"
+          end
+          result.merge!("score" => score.to_f, "passed" => score >= evaluator.threshold,
+                        "reason" => assessment.reason)
+        rescue StandardError => exception
+          result["error"] = Data.error(exception)
+        end
+        result
+      end
+
+      def summarize(results, duration)
+        counts = results.group_by { |item| item["status"] }.transform_values(&:size)
+        evaluated = results.reject { |item| item["status"] == "not_evaluated" }
+        # An agent error has no assessments; its case quality is zero, not missing data.
+        scores = evaluated.map do |item|
+          metrics = item["metrics"]
+          metrics.empty? ? 0.0 : metrics.sum { |evaluation| evaluation["score"] || 0.0 } / metrics.size
+        end
+        {
+          "cases" => results.size, "passed" => counts.fetch("passed", 0),
+          "failed" => counts.fetch("failed", 0), "errors" => counts.fetch("error", 0),
+          "not_evaluated" => counts.fetch("not_evaluated", 0), "evaluated_cases" => evaluated.size,
+          "pass_rate" => evaluated.empty? ? nil : counts.fetch("passed", 0).fdiv(evaluated.size),
+          "score" => scores.empty? ? nil : scores.sum / scores.size, "duration_ms" => duration,
+          "input_tokens" => results.sum { |item| item["input_tokens"] || 0 },
+          "output_tokens" => results.sum { |item| item["output_tokens"] || 0 },
+          "cost" => results.sum { |item| item["cost"] || 0.0 }
+        }
+      end
+
+      def mutable_copy(value)
+        case value
+        when Hash then value.to_h { |key, item| [key, mutable_copy(item)] }
+        when Array then value.map { |item| mutable_copy(item) }
+        when String then value.dup
+        else value
+        end
+      end
+
+      def clock
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def milliseconds(started)
+        (clock - started) * 1000.0
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluations/suite.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module RubyLLM
+  module Evaluations
+    Question = Struct.new(:id, :name, :question, :expected_answer, :expected_actions, :minimum_scores, :metadata,
+                          :evaluators, :api_interactions, keyword_init: true) do
+      def to_h
+        super.reject { |key, _value| %i[evaluators api_interactions].include?(key) }
+      end
+    end
+    Evaluator = Struct.new(:name, :callable, :threshold, keyword_init: true)
+
+    class Suite
+      SOURCE_DIRECTORY = Dir.pwd.freeze
+      private_constant :SOURCE_DIRECTORY
+
+      attr_reader :name, :agent, :agent_name, :version, :cases, :evaluators, :judge
+
+      def initialize(name, agent:, judge: nil, agent_name: nil, version: "1")
+        @name = nonempty(name, "suite name")
+        @agent_name = nonempty(agent_name || name, "agent name")
+        @version = nonempty(version, "version")
+        raise DefinitionError, "agent must respond to call" unless agent.respond_to?(:call)
+        unless judge.nil? || (judge.is_a?(Class) && defined?(::RubyLLM::Agent) && judge <= ::RubyLLM::Agent)
+          raise DefinitionError, "judge must be a RubyLLM::Agent class"
+        end
+        @agent = agent
+        @judge = judge
+        @cases = []
+        @evaluators = []
+      end
+
+      def question(value, expected_answer: nil, minimum_scores: nil, metadata: {}, name: nil, id: nil, expected_actions: nil, &block)
+        name = nonempty(name || (value.is_a?(String) ? value : "Question #{cases.size + 1}"), "question name")
+        id = nonempty(id || name, "question id")
+        raise DefinitionError, "duplicate question id: #{id}" if cases.any? { |item| item.id == id }
+        raise DefinitionError, "question metadata must be a Hash" unless metadata.is_a?(Hash)
+        [value, expected_answer, metadata].each { |item| Data.json_value(item) }
+        minimum_scores = normalize_minimum_scores(minimum_scores) unless minimum_scores.nil?
+        unless expected_actions.nil?
+          expected_actions = Actions.normalize(expected_actions)
+          if expected_actions.any? { |action| action["group"].nil? }
+            raise DefinitionError, "expected actions must specify an action group"
+          end
+        end
+        if block
+          source = block.source_location.first
+          # Ruby main scripts can retain relative source locations after Dir.chdir.
+          source = File.expand_path(source, File.file?(source) ? Dir.pwd : SOURCE_DIRECTORY)
+          replay = ApiReplay::Definition.new(base_dir: File.dirname(source))
+          replay.instance_eval(&block)
+          replay.validate!
+        end
+        cases << Question.new(id: id, name: name, question: immutable_copy(value),
+                              expected_answer: immutable_copy(expected_answer),
+                              expected_actions: immutable_copy(expected_actions), api_interactions: replay,
+                              minimum_scores: minimum_scores, metadata: immutable_copy(metadata)).freeze
+      end
+
+      def evaluate(name, using: nil, threshold: 1.0, &block)
+        name = nonempty(name, "evaluator name")
+        raise DefinitionError, "duplicate evaluator: #{name}" if evaluators.any? { |item| item.name == name }
+        raise DefinitionError, "choose using: or a block, not both" if using && block
+        callable = using || block
+        raise DefinitionError, "evaluator must respond to call" unless callable.respond_to?(:call)
+        validate_threshold(threshold)
+        evaluators << Evaluator.new(name: name, callable: callable, threshold: threshold.to_f).freeze
+      end
+
+      def validate!
+        raise DefinitionError, "suite must contain at least one question" if cases.empty?
+        cases.map! do |item|
+          minimum_scores = item.minimum_scores || if evaluators.empty?
+                                                   { "semantic_accuracy" => 0.9, "answer_relevancy" => 0.9 }
+                                                 else
+                                                   evaluators.to_h { |definition| [definition.name, definition.threshold] }
+                                                 end
+          definitions = minimum_scores.filter_map do |metric_name, threshold|
+            custom = evaluators.find { |definition| definition.name == metric_name }
+            callable = custom ? custom.callable : standard_metric(metric_name)
+            next if judge.nil? && callable.is_a?(JudgeMetric)
+            Evaluator.new(name: metric_name, callable: callable, threshold: threshold).freeze
+          end.freeze
+          Question.new(**item.to_h, minimum_scores: immutable_copy(minimum_scores),
+                       api_interactions: item.api_interactions, evaluators: definitions).freeze
+        end
+        self
+      end
+
+      def digest
+        validate!
+        definition = { "name" => name, "version" => version,
+                       "cases" => cases.map do |item|
+                         # Persistent fingerprint labels are stable across public API renames.
+                         data = item.to_h.transform_keys { |key| key == :minimum_scores ? :criteria : key }
+                         data = data.merge(api_interactions: item.api_interactions.to_h) if item.api_interactions
+                         skipped = item.minimum_scores.keys - item.evaluators.map(&:name)
+                         data = data.merge(skipped_metrics: skipped) unless skipped.empty?
+                         definition_value(data)
+                       end }
+        Digest::SHA256.hexdigest(JSON.generate(Data.canonical(definition)))
+      end
+
+      private
+
+      def standard_metric(name)
+        case name
+        when "semantic_accuracy", "answer_relevancy"
+          Evaluators.public_send(name)
+        when "action_group"
+          Evaluators.action_group
+        else
+          raise DefinitionError, "unknown metric: #{name}; register it with evaluate"
+        end
+      end
+
+      def normalize_minimum_scores(minimum_scores)
+        unless minimum_scores.is_a?(Hash) && !minimum_scores.empty?
+          raise DefinitionError, "minimum_scores must be a nonempty Hash of metric names and minimum passing scores"
+        end
+        minimum_scores.each_with_object({}) do |(name, threshold), normalized|
+          name = nonempty(name, "metric name")
+          raise DefinitionError, "duplicate metric: #{name}" if normalized.key?(name)
+          validate_threshold(threshold)
+          normalized[name] = threshold.to_f
+        end.freeze
+      end
+
+      def validate_threshold(threshold)
+        unless threshold.is_a?(Numeric) && threshold.finite? && threshold.between?(0, 1)
+          raise DefinitionError, "threshold must be a finite number between 0 and 1"
+        end
+      end
+
+      # JSON snapshots normalize symbols, but Ruby agents distinguish key/value types.
+      def definition_value(value)
+        case value
+        when Hash
+          entries = value.map { |key, item| [key.class.name, key.to_s, definition_value(item)] }
+          { "hash" => entries.sort_by { |type, name, _item| [type, name] } }
+        when Array
+          value.map { |item| definition_value(item) }
+        when Symbol
+          { "symbol" => value.to_s }
+        else
+          value
+        end
+      end
+
+      def nonempty(value, label)
+        raise DefinitionError, "#{label} must not be empty" if value.to_s.strip.empty?
+        value.to_s.dup.freeze
+      end
+
+      def immutable_copy(value)
+        case value
+        when Hash
+          value.to_h { |key, item| [immutable_copy(key), immutable_copy(item)] }.freeze
+        when Array
+          value.map { |item| immutable_copy(item) }.freeze
+        when String
+          value.dup.freeze
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluations/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Evaluations
+    VERSION = "0.2.0"
+  end
+end

data/lib/ruby_llm/evaluations.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative "evaluations/version"
+require_relative "evaluations/data"
+require_relative "evaluations/actions"
+require_relative "evaluations/judge_metric"
+require_relative "evaluations/evaluators"
+require_relative "evaluations/api_replay"
+require_relative "evaluations/suite"
+require_relative "evaluations/response"
+require_relative "evaluations/adapters"
+require_relative "evaluations/context"
+require_relative "evaluations/action_group"
+require_relative "evaluations/runner"
+require_relative "../../app/models/ruby_llm/evaluations/application_record"
+require_relative "../../app/models/ruby_llm/evaluations/run"
+require_relative "../../app/models/ruby_llm/evaluations/case_result"
+require_relative "../../app/models/ruby_llm/evaluations/metric_result"
+require_relative "evaluations/report"
+require_relative "evaluations/dashboard"
+
+module RubyLLM
+  module Evaluations
+    DEFAULT_DATABASE_URL = "postgresql://localhost/ruby_llm_evaluations"
+    MIGRATIONS_PATH = File.expand_path("../../db/migrate", __dir__).freeze
+    autoload :Engine, File.expand_path("evaluations/engine", __dir__)
+
+    def self.define(name, agent:, judge: nil, agent_name: nil, version: "1", &block)
+      Suite.new(name, agent: agent, judge: judge, agent_name: agent_name, version: version).tap do |suite|
+        suite.instance_eval(&block) if block
+        suite.validate!
+      end
+    end
+
+    def self.run(suite, metadata: {})
+      Runner.new.run(suite, metadata: metadata)
+    end
+
+    # Standalone setup uses Rails migrations; a host application uses db:migrate.
+    def self.migrate!
+      ApplicationRecord.require_postgresql!
+      ::ActiveRecord::MigrationContext.new([MIGRATIONS_PATH]).migrate
+      [Run, CaseResult, MetricResult].each(&:reset_column_information)
+    end
+  end
+end
+
+require_relative "evaluations/engine" if defined?(::Rails::Engine)