ruby_llm-tribunal 0.1.0

data/lib/ruby_llm/tribunal/assertions/deterministic.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tribunal
+    module Assertions
+      # Deterministic assertions that don't require LLM calls.
+      #
+      # Fast, free, and should run first before expensive LLM-based checks.
+      module Deterministic
+        class << self
+          # Evaluates a deterministic assertion.
+          #
+          # @param type [Symbol] The assertion type
+          # @param output [String] The output to evaluate
+          # @param opts [Hash] Options for the assertion
+          # @return [Array] [:pass, details] or [:fail, details]
+          #
+          # @example
+          #   evaluate(:contains, "Hello world", value: "world")
+          #   #=> [:pass, { matched: ["world"] }]
+          def evaluate(type, output, opts = {})
+            send(:"evaluate_#{type}", output, opts)
+          end
+
+          private
+
+          def evaluate_contains(output, opts)
+            values = Array(opts[:value] || opts[:values])
+            results = values.map { |v| [v, output.include?(v)] }
+
+            if results.all? { |_, matched| matched }
+              [:pass, { matched: values }]
+            else
+              missing = results.reject { |_, m| m }.map(&:first)
+              [:fail, { missing:, reason: "Output missing: #{missing.inspect}" }]
+            end
+          end
+
+          def evaluate_not_contains(output, opts)
+            values = Array(opts[:value] || opts[:values])
+            found = values.select { |v| output.include?(v) }
+
+            if found.empty?
+              [:pass, { checked: values }]
+            else
+              [:fail, { found:, reason: "Output contains forbidden: #{found.inspect}" }]
+            end
+          end
+
+          def evaluate_contains_any(output, opts)
+            values = Array(opts[:value] || opts[:values])
+            found = values.find { |v| output.include?(v) }
+
+            if found
+              [:pass, { matched: found }]
+            else
+              [:fail, { expected_any: values, reason: "Output contains none of: #{values.inspect}" }]
+            end
+          end
+
+          def evaluate_contains_all(output, opts)
+            values = Array(opts[:value] || opts[:values])
+            results = values.map { |v| [v, output.include?(v)] }
+
+            if results.all? { |_, matched| matched }
+              [:pass, { matched: values }]
+            else
+              missing = results.reject { |_, m| m }.map(&:first)
+              [:fail, { missing:, reason: "Output missing: #{missing.inspect}" }]
+            end
+          end
+
+          def evaluate_regex(output, opts)
+            pattern = opts[:value] || opts[:pattern]
+            regex = pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern)
+
+            match = output.match(regex)
+            if match
+              [:pass, { matched: match[0], pattern: regex.source }]
+            else
+              [:fail, { pattern: regex.source, reason: "Pattern not found: #{regex.source}" }]
+            end
+          end
+
+          def evaluate_is_json(output, _opts)
+            parsed = JSON.parse(output)
+            [:pass, { parsed: }]
+          rescue JSON::ParserError
+            [:fail, { reason: 'Invalid JSON' }]
+          end
+
+          def evaluate_max_tokens(output, opts)
+            max = opts[:value] || opts[:max] || 500
+
+            # Approximate: 1 token ~= 0.75 words ~= 4 chars
+            # Using word count as a reasonable approximation
+            word_count = output.split(/\s+/).length
+            approx_tokens = (word_count / 0.75).ceil
+
+            if approx_tokens <= max
+              [:pass, { approx_tokens:, max: }]
+            else
+              [:fail, {
+                approx_tokens:,
+                max:,
+                reason: "Output ~#{approx_tokens} tokens exceeds max #{max}"
+              }]
+            end
+          end
+
+          def evaluate_latency_ms(_output, opts)
+            max = opts[:value] || opts[:max] || 5000
+            actual = opts[:actual] || opts[:latency]
+
+            if actual.nil?
+              [:fail, { reason: 'No latency value provided in opts[:actual]' }]
+            elsif actual <= max
+              [:pass, { latency_ms: actual, max: }]
+            else
+              [:fail, { latency_ms: actual, max:, reason: "Latency #{actual}ms exceeds max #{max}ms" }]
+            end
+          end
+
+          def evaluate_starts_with(output, opts)
+            prefix = opts[:value]
+
+            if output.start_with?(prefix)
+              [:pass, { prefix: }]
+            else
+              [:fail, { expected: prefix, reason: "Output does not start with: #{prefix}" }]
+            end
+          end
+
+          def evaluate_ends_with(output, opts)
+            suffix = opts[:value]
+
+            if output.end_with?(suffix)
+              [:pass, { suffix: }]
+            else
+              [:fail, { expected: suffix, reason: "Output does not end with: #{suffix}" }]
+            end
+          end
+
+          def evaluate_equals(output, opts)
+            expected = opts[:value]
+
+            if output == expected
+              [:pass, {}]
+            else
+              [:fail, { expected:, actual: output, reason: 'Output does not match expected' }]
+            end
+          end
+
+          def evaluate_min_length(output, opts)
+            min = opts[:value] || opts[:min]
+            length = output.length
+
+            if length >= min
+              [:pass, { length:, min: }]
+            else
+              [:fail, { length:, min:, reason: "Output length #{length} below minimum #{min}" }]
+            end
+          end
+
+          def evaluate_max_length(output, opts)
+            max = opts[:value] || opts[:max]
+            length = output.length
+
+            if length <= max
+              [:pass, { length:, max: }]
+            else
+              [:fail, { length:, max:, reason: "Output length #{length} exceeds maximum #{max}" }]
+            end
+          end
+
+          def evaluate_word_count(output, opts)
+            min = opts[:min] || 0
+            max = opts[:max]
+
+            words = output.split(/\s+/).reject(&:empty?)
+            count = words.length
+
+            if count < min
+              [:fail, { word_count: count, min:, reason: "Word count #{count} below minimum #{min}" }]
+            elsif max && count > max
+              [:fail, { word_count: count, max:, reason: "Word count #{count} exceeds maximum #{max}" }]
+            else
+              [:pass, { word_count: count }]
+            end
+          end
+
+          def evaluate_is_url(output, _opts)
+            url = output.strip
+
+            if url.match?(%r{^https?://[^\s]+$})
+              [:pass, { url: }]
+            else
+              [:fail, { reason: 'Output is not a valid URL' }]
+            end
+          end
+
+          def evaluate_is_email(output, _opts)
+            email = output.strip
+
+            if email.match?(/^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/)
+              [:pass, { email: }]
+            else
+              [:fail, { reason: 'Output is not a valid email' }]
+            end
+          end
+
+          def evaluate_levenshtein(output, opts)
+            target = opts[:value]
+            max_distance = opts[:max_distance] || 3
+
+            distance = levenshtein_distance(output, target)
+
+            if distance <= max_distance
+              [:pass, { distance:, max_distance: }]
+            else
+              [:fail, {
+                distance:,
+                max_distance:,
+                reason: "Edit distance #{distance} exceeds max #{max_distance}"
+              }]
+            end
+          end
+
+          # Levenshtein distance algorithm
+          def levenshtein_distance(s1, s2)
+            s1_chars = s1.chars
+            s2_chars = s2.chars
+
+            return s2_chars.length if s1_chars.empty?
+            return s1_chars.length if s2_chars.empty?
+
+            row = (0..s2_chars.length).to_a
+
+            s1_chars.each_with_index do |c1, i|
+              prev_row = row
+              row = [i + 1]
+
+              s2_chars.each_with_index do |c2, j|
+                cost = c1 == c2 ? 0 : 1
+                row << [
+                  row[j] + 1,           # deletion
+                  prev_row[j + 1] + 1,  # insertion
+                  prev_row[j] + cost    # substitution
+                ].min
+              end
+            end
+
+            row.last
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/tribunal/assertions/embedding.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tribunal
+    module Assertions
+      # Embedding-based semantic similarity assertions.
+      #
+      # Uses sentence embeddings to determine if two texts are semantically similar.
+      module Embedding
+        DEFAULT_THRESHOLD = 0.7
+
+        class << self
+          # Returns list of available embedding assertion types.
+          #
+          # @return [Array<Symbol>]
+          def available
+            [:similar]
+          end
+
+          # Evaluates semantic similarity between actual and expected output.
+          #
+          # @param test_case [TestCase] The test case
+          # @param opts [Hash] Options
+          # @option opts [Float] :threshold Similarity threshold (0.0 to 1.0). Default: 0.7
+          # @option opts [Proc] :similarity_fn Custom similarity function for testing
+          # @return [Array] [:pass, details], [:fail, details], or [:error, message]
+          #
+          # @example
+          #   test_case = TestCase.new(
+          #     actual_output: "The cat is sleeping",
+          #     expected_output: "A feline is resting"
+          #   )
+          #
+          #   Embedding.evaluate(test_case, threshold: 0.8)
+          #   #=> [:pass, { similarity: 0.85, threshold: 0.8 }]
+          def evaluate(test_case, opts = {})
+            if test_case.expected_output.nil?
+              return [:error,
+                      'Similar assertion requires expected_output to be provided']
+            end
+
+            threshold = opts[:threshold] || DEFAULT_THRESHOLD
+            similarity_fn = opts[:similarity_fn] || method(:default_similarity)
+
+            result = similarity_fn.call(test_case.actual_output, test_case.expected_output, opts)
+
+            case result
+            in [:ok, similarity] if similarity >= threshold
+              [:pass, { similarity:, threshold: }]
+            in [:ok, similarity]
+              [:fail, {
+                similarity:,
+                threshold:,
+                reason: "Output is not semantically similar to expected (#{similarity.round(2)} < #{threshold})"
+              }]
+            in [:error, reason]
+              [:error, "Failed to compute similarity: #{reason}"]
+            end
+          end
+
+          private
+
+          def default_similarity(text1, text2, opts)
+            # Use RubyLLM embeddings
+            model = opts[:embedding_model] || 'text-embedding-3-small'
+
+            embedding1 = RubyLLM.embed(text1, model:).vectors.first
+            embedding2 = RubyLLM.embed(text2, model:).vectors.first
+
+            # Compute cosine similarity
+            similarity = cosine_similarity(embedding1, embedding2)
+            [:ok, similarity]
+          rescue StandardError => e
+            [:error, e.message]
+          end
+
+          def cosine_similarity(vec1, vec2)
+            dot_product = vec1.zip(vec2).sum { |a, b| a * b }
+            magnitude1 = Math.sqrt(vec1.sum { |x| x**2 })
+            magnitude2 = Math.sqrt(vec2.sum { |x| x**2 })
+
+            return 0.0 if magnitude1.zero? || magnitude2.zero?
+
+            dot_product / (magnitude1 * magnitude2)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/tribunal/assertions/judge.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tribunal
+    module Assertions
+      # LLM-as-judge assertions for evaluating LLM outputs.
+      #
+      # Uses RubyLLM to get structured verdicts from the judge model.
+      module Judge
+        DEFAULT_MODEL = 'anthropic:claude-3-5-haiku-latest'
+        DEFAULT_THRESHOLD = 0.8
+
+        SYSTEM_PROMPT = <<~PROMPT
+          You are a precise evaluator of LLM outputs. Your task is to assess outputs
+          based on specific criteria and provide structured verdicts.
+
+          Always respond with valid JSON containing:
+          - verdict: "yes", "no", or "partial"
+          - reason: A brief explanation
+          - score: A float from 0.0 to 1.0
+
+          Be objective and consistent in your evaluations.
+        PROMPT
+
+        class << self
+          # Returns list of available judge assertion types.
+          #
+          # @return [Array<Symbol>]
+          def available
+            Tribunal::Judge.all_judge_names
+          end
+
+          # Evaluates a judge assertion against a test case.
+          #
+          # @param type [Symbol] The judge type
+          # @param test_case [TestCase] The test case
+          # @param opts [Hash] Options
+          # @return [Array] [:pass, details], [:fail, details], or [:error, message]
+          def evaluate(type, test_case, opts)
+            judge_class = Tribunal::Judge.find(type)
+            return [:error, "Unknown judge assertion: #{type}"] unless judge_class
+
+            # Validate test case
+            error = judge_class.validate(test_case) if judge_class.respond_to?(:validate)
+            return [:error, error] if error
+
+            run_judge(judge_class, test_case, opts)
+          end
+
+          private
+
+          def run_judge(judge_class, test_case, opts)
+            model = opts[:model] || Tribunal.configuration.default_model || DEFAULT_MODEL
+            threshold = opts[:threshold] || Tribunal.configuration.default_threshold || DEFAULT_THRESHOLD
+            prompt = judge_class.prompt(test_case, opts)
+
+            response = call_llm(model, prompt, opts)
+
+            case response
+            in [:ok, result]
+              # Check if judge has custom evaluation
+              if judge_class.respond_to?(:evaluate_result)
+                custom_result = judge_class.evaluate_result(result, opts)
+                return custom_result if custom_result
+              end
+
+              negative_metric = judge_class.respond_to?(:negative_metric?) && judge_class.negative_metric?
+              interpret_response(result, threshold, negative_metric)
+            in [:error, reason]
+              [:error, reason.to_s]
+            end
+          end
+
+          def call_llm(model, prompt, opts)
+            # Allow injecting custom LLM for tests via opts[:llm]
+            return opts[:llm].call(model, build_messages(prompt), opts) if opts[:llm]
+
+            call_ruby_llm(model, prompt, opts)
+          end
+
+          def build_messages(prompt)
+            [
+              { role: 'system', content: SYSTEM_PROMPT },
+              { role: 'user', content: prompt }
+            ]
+          end
+
+          def call_ruby_llm(model, prompt, _opts)
+            # Parse model string (e.g., "anthropic:claude-3-5-haiku-latest")
+            _provider, model_name = parse_model(model)
+
+            chat = RubyLLM.chat(model: model_name)
+
+            # Add system message
+            chat.with_instructions(SYSTEM_PROMPT)
+
+            # Get response
+            response = chat.ask(prompt)
+            content = response.content
+
+            # Parse JSON response
+            parsed = JSON.parse(content)
+            [:ok, parsed]
+          rescue JSON::ParserError => e
+            # Try to extract JSON from response
+            if (match = content&.match(/\{[\s\S]*\}/))
+              begin
+                parsed = JSON.parse(match[0])
+                return [:ok, parsed]
+              rescue JSON::ParserError
+                # Fall through to error
+              end
+            end
+            [:error, "Failed to parse LLM response as JSON: #{e.message}"]
+          rescue StandardError => e
+            [:error, e.message]
+          end
+
+          def parse_model(model_string)
+            if model_string.include?(':')
+              model_string.split(':', 2)
+            else
+              [nil, model_string]
+            end
+          end
+
+          def interpret_response(response, threshold, negative_metric)
+            details = {
+              verdict: response['verdict'],
+              reason: response['reason'],
+              score: response['score']
+            }
+
+            verdict_result(response['verdict'], response['score'], threshold, negative_metric, details)
+          end
+
+          def verdict_result(verdict, score, threshold, negative_metric, details)
+            return [:error, "Unexpected verdict: #{verdict}"] unless %w[yes no partial].include?(verdict)
+
+            passed = case verdict
+                     when 'yes' then !negative_metric
+                     when 'no' then negative_metric
+                     when 'partial' then score.is_a?(Numeric) && score >= threshold
+                     end
+
+            passed ? [:pass, details] : [:fail, details]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/tribunal/assertions.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tribunal
+    # Assertion evaluation engine.
+    #
+    # Routes assertions to the appropriate implementation:
+    # - Deterministic: `contains`, `regex`, `is_json`, etc.
+    # - Judge (requires ruby_llm): `faithful`, `relevant`, etc.
+    # - Embedding (requires neighbor): `similar`
+    module Assertions
+      DETERMINISTIC_ASSERTIONS = %i[
+        contains
+        not_contains
+        contains_any
+        contains_all
+        regex
+        is_json
+        max_tokens
+        latency_ms
+        starts_with
+        ends_with
+        equals
+        min_length
+        max_length
+        word_count
+        is_url
+        is_email
+        levenshtein
+      ].freeze
+
+      EMBEDDING_ASSERTIONS = %i[similar].freeze
+
+      class << self
+        # Evaluates a single assertion against a test case.
+        #
+        # @param assertion_type [Symbol] The type of assertion
+        # @param test_case [TestCase] The test case to evaluate
+        # @param opts [Hash] Options for the assertion
+        # @return [Array] [:pass, details] or [:fail, details]
+        def evaluate(assertion_type, test_case, opts = {})
+          if DETERMINISTIC_ASSERTIONS.include?(assertion_type)
+            Deterministic.evaluate(assertion_type, test_case.actual_output, opts)
+          elsif Tribunal::Judge.builtin_judge?(assertion_type) || Tribunal::Judge.custom_judge?(assertion_type)
+            evaluate_judge(assertion_type, test_case, opts)
+          elsif EMBEDDING_ASSERTIONS.include?(assertion_type)
+            evaluate_embedding(assertion_type, test_case, opts)
+          else
+            [:error, "Unknown assertion type: #{assertion_type}"]
+          end
+        end
+
+        # Evaluates multiple assertions against a test case.
+        #
+        # @param assertions [Array, Hash] Assertions to evaluate
+        # @param test_case [TestCase] The test case to evaluate
+        # @return [Hash] Map of assertion_type => result
+        def evaluate_all(assertions, test_case)
+          normalized = normalize_assertions(assertions)
+          normalized.each_with_object({}) do |(type, opts), results|
+            results[type] = evaluate(type, test_case, opts)
+          end
+        end
+
+        # Checks if all assertions passed.
+        #
+        # @param results [Hash] Results from evaluate_all
+        # @return [Boolean] True if all passed
+        def all_passed?(results)
+          results.all? { |_type, result| result.first == :pass }
+        end
+
+        # Returns list of available assertion types based on loaded dependencies.
+        #
+        # @return [Array<Symbol>] Available assertion types
+        def available
+          base = DETERMINISTIC_ASSERTIONS.dup
+
+          # Add judge assertions
+          base.concat(Tribunal::Judge.all_judge_names)
+
+          # Add embedding assertions if neighbor is available
+          begin
+            require 'neighbor'
+            base.concat(EMBEDDING_ASSERTIONS)
+          rescue LoadError
+            # neighbor not available
+          end
+
+          base
+        end
+
+        private
+
+        def normalize_assertions(assertions)
+          case assertions
+          when Hash
+            assertions.map do |type, opts|
+              opts = { value: opts } unless opts.is_a?(Hash)
+              [type.to_sym, opts]
+            end
+          when Array
+            assertions.map do |item|
+              case item
+              when Symbol, String
+                [item.to_sym, {}]
+              when Array
+                type, opts = item
+                opts = { value: opts } unless opts.is_a?(Hash)
+                [type.to_sym, opts]
+              else
+                raise ArgumentError, "Invalid assertion format: #{item.inspect}"
+              end
+            end
+          else
+            raise ArgumentError, 'Assertions must be a Hash or Array'
+          end
+        end
+
+        def evaluate_judge(type, test_case, opts)
+          Assertions::Judge.evaluate(type, test_case, opts)
+        end
+
+        def evaluate_embedding(_type, test_case, opts)
+          begin
+            require 'neighbor'
+          rescue LoadError
+            raise Error, <<~MSG
+              Embedding similarity requires the 'neighbor' gem.
+
+              Add to your Gemfile:
+                gem 'neighbor', '~> 0.4'
+            MSG
+          end
+
+          Embedding.evaluate(test_case, opts)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/tribunal/configuration.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tribunal
+    # Configuration for Tribunal.
+    #
+    # @example
+    #   RubyLLM::Tribunal.configure do |config|
+    #     config.default_model = "anthropic:claude-3-5-haiku-latest"
+    #     config.default_threshold = 0.8
+    #     config.verbose = true
+    #   end
+    class Configuration
+      # Default LLM model for judge assertions
+      # @return [String]
+      attr_accessor :default_model
+
+      # Default threshold for judge assertions (0.0-1.0)
+      # @return [Float]
+      attr_accessor :default_threshold
+
+      # Whether to print verbose output
+      # @return [Boolean]
+      attr_accessor :verbose
+
+      # Custom judges to register
+      # @return [Array<Class>]
+      attr_accessor :custom_judges
+
+      def initialize
+        @default_model = 'anthropic:claude-3-5-haiku-latest'
+        @default_threshold = 0.8
+        @verbose = false
+        @custom_judges = []
+      end
+    end
+  end
+end