raif 1.2.1 → 1.3.0

data/lib/raif/evals/llm_judges/binary.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module LlmJudges
+      class Binary < Raif::Evals::LlmJudge
+        task_run_arg :criteria
+        task_run_arg :examples
+        task_run_arg :strict_mode
+
+        json_response_schema do
+          boolean :passes, description: "Whether the content passes the criteria"
+          string :reasoning, description: "Detailed explanation of the judgment"
+          number :confidence, description: "Confidence level from 0.0 to 1.0", minimum: 0, maximum: 1
+        end
+
+        def build_system_prompt
+          <<~PROMPT.strip
+            You are an expert evaluator assessing whether content meets specific criteria.
+            Your task is to make binary pass/fail judgments with clear reasoning.
+
+            First, provide detailed reasoning/explanation of your evaluation. Then, provide a precise pass/fail judgment.
+
+            Respond with JSON matching this schema:
+            {
+              "passes": boolean,
+              "reasoning": "detailed explanation",
+              "confidence": 0.0-1.0
+            }
+          PROMPT
+        end
+
+        def build_prompt
+          prompt = <<~PROMPT
+            Evaluation criteria: #{criteria}
+
+            #{strict_mode ? "Apply the criteria strictly without any leniency." : "Apply reasonable judgment while adhering to the criteria."}
+          PROMPT
+
+          if examples.present?
+            prompt += "\nHere are examples of how to evaluate:"
+            examples.each do |example|
+              prompt += format_example(example)
+            end
+          end
+
+          prompt += additional_context_prompt if additional_context.present?
+
+          prompt += <<~PROMPT.rstrip
+
+            Now evaluate this content:
+            #{content_to_judge}
+
+            Does this content meet the evaluation criteria?
+          PROMPT
+
+          prompt
+        end
+
+        # Judgment accessor methods
+        def passes?
+          parsed_response["passes"] if completed?
+        end
+
+      private
+
+        def additional_context_prompt
+          <<~PROMPT
+
+            Additional context:
+            #{additional_context}
+          PROMPT
+        end
+
+        def format_example(example)
+          if example.key?(:output)
+            content_label = "Output"
+            content_value = example[:output]
+          else
+            content_label = "Content"
+            content_value = example[:content]
+          end
+
+          <<~EXAMPLE
+
+            #{content_label}: #{content_value}
+            Reasoning: #{example[:reasoning]}
+            Judgment: #{example[:passes] ? "PASS" : "FAIL"}
+          EXAMPLE
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals/llm_judges/comparative.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module LlmJudges
+      class Comparative < Raif::Evals::LlmJudge
+        task_run_arg :over_content # the content to compare against
+        task_run_arg :comparison_criteria # the criteria to use when comparing content_to_judge to over_content
+        task_run_arg :allow_ties # whether to allow ties in the comparison
+
+        attr_accessor :content_a, :content_b, :expected_winner
+
+        before_create do
+          self.expected_winner = ["A", "B"].sample
+
+          if expected_winner == "A"
+            self.content_a = content_to_judge
+            self.content_b = over_content
+          else
+            self.content_a = over_content
+            self.content_b = content_to_judge
+          end
+        end
+
+        json_response_schema do
+          string :winner, description: "Which content is better (A, B, or tie)", enum: ["A", "B", "tie"]
+          string :reasoning, description: "Detailed explanation of the judgment"
+          number :confidence, description: "Confidence level from 0.0 to 1.0", minimum: 0, maximum: 1
+        end
+
+        def build_system_prompt
+          <<~PROMPT.strip
+            You are an expert evaluator comparing two pieces of content to determine which better meets specified criteria.
+
+            #{allow_ties ? "You may declare a tie if both pieces of content are equally good." : "You must choose a winner even if the difference is minimal."}
+
+            First, provide detailed reasoning for your choice. Then, provide a precise winner #{allow_ties ? "(A, B, or tie)" : "(A or B)"}.
+
+            Respond with JSON matching the required schema.
+          PROMPT
+        end
+
+        def build_prompt
+          <<~PROMPT.strip
+            Comparison criteria: #{comparison_criteria}
+            #{additional_context_prompt}
+            Compare the following two pieces of content:
+
+            CONTENT A:
+            #{content_a}
+
+            CONTENT B:
+            #{content_b}
+
+            Which content better meets the comparison criteria?
+          PROMPT
+        end
+
+        def winner
+          parsed_response["winner"] if completed?
+        end
+
+        def tie?
+          return unless completed? # rubocop:disable Style/ReturnNilInPredicateMethodDefinition
+
+          parsed_response["winner"] == "tie"
+        end
+
+        def correct_expected_winner?
+          return unless completed? # rubocop:disable Style/ReturnNilInPredicateMethodDefinition
+
+          parsed_response["winner"] == expected_winner
+        end
+
+      private
+
+        def additional_context_prompt
+          return if additional_context.blank?
+
+          <<~PROMPT
+
+            Additional context:
+            #{additional_context}
+          PROMPT
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals/llm_judges/scored.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module LlmJudges
+      class Scored < Raif::Evals::LlmJudge
+        task_run_arg :scoring_rubric # the scoring rubric to use when evaluating the content
+
+        json_response_schema do
+          number :score, description: "Numerical score based on the rubric"
+          string :reasoning, description: "Detailed explanation of the score"
+          number :confidence, description: "Confidence level from 0.0 to 1.0", minimum: 0, maximum: 1
+        end
+
+        def build_system_prompt
+          <<~PROMPT.strip
+            You are an expert evaluator providing numerical scores based on a detailed rubric.
+
+            First, provide detailed reasoning/explanation of your evaluation. Then, provide a precise score according to the provided rubric.
+
+            Respond with JSON matching this schema:
+            {
+              "score": number,
+              "reasoning": "detailed explanation",
+              "confidence": 0.0-1.0
+            }
+          PROMPT
+        end
+
+        def build_prompt
+          <<~PROMPT.strip
+            Scoring rubric:
+            #{format_rubric(scoring_rubric)}
+            #{additional_context_prompt}
+            Evaluate the following content according to the scoring rubric:
+            #{content_to_judge}
+
+            Provide your score and detailed reasoning.
+          PROMPT
+        end
+
+        def judgment_score
+          parsed_response["score"] if completed?
+        end
+
+      private
+
+        def additional_context_prompt
+          return if additional_context.blank?
+
+          <<~PROMPT
+            \nAdditional context:
+            #{additional_context}
+          PROMPT
+        end
+
+        def format_rubric(rubric)
+          rubric.is_a?(ScoringRubric) ? rubric.to_prompt : rubric.to_s
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals/llm_judges/summarization.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module LlmJudges
+      class Summarization < Raif::Evals::LlmJudge
+        task_run_arg :original_content # the original content to evaluate the summary against
+        task_run_arg :summary # the summary to evaluate against the original content
+
+        json_response_schema do
+          object :coverage do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :accuracy do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :clarity do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :conciseness do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :overall do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+        end
+
+        def build_system_prompt
+          <<~PROMPT.strip
+            You are an impartial expert judge of summary quality. You'll be provided a original piece of content and its summary. Your job is to evaluate the summary against the original content based on the following criteria, and assign a score from 1 to 5 for each (5 = excellent, 1 = very poor):
+
+            **Coverage (Relevance & Completeness):** Does the summary capture all the important points of the original content?
+            - 5 = Excellent Coverage - Nearly all key points and essential details from the content are present in the summary, with no major omissions.
+            - 4 = Good Coverage - Most important points are included, but a minor detail or two might be missing.
+            - 3 = Fair Coverage - Some main points appear, but the summary misses or glosses over other important information.
+            - 2 = Poor Coverage - Many critical points from the content are missing; the summary is incomplete.
+            - 1 = Very Poor - The summary fails to include most of the content's main points (highly incomplete).
+
+            **Accuracy (Faithfulness to the Source):** Is the summary factually correct and free of hallucinations or misrepresentations of the content?
+            - 5 = Fully Accurate - All statements in the summary are correct and directly supported by the content. No errors or invented information.
+            - 4 = Mostly Accurate - The summary is generally accurate with perhaps one minor error or slight ambiguity, but no significant falsehoods.
+            - 3 = Some Inaccuracies - Contains a few errors or unsupported claims from the content, but overall captures the gist correctly.
+            - 2 = Mostly Inaccurate - Multiple statements in the summary are incorrect or not supported by the content.
+            - 1 = Completely Inaccurate - The summary seriously distorts or contradicts the content; many claims are false or not in the source.
+
+            **Clarity and Coherence:** Is the summary well-written and easy to understand? (Consider organization, flow, and whether it would make sense to a reader.)
+            - 5 = Very Clear & Coherent - The summary is logically organized, flows well, and would be easily understood by the target reader. No confusion or ambiguity.
+            - 4 = Mostly Clear - Readable and mostly well-structured, though a sentence or transition could be smoother.
+            - 3 = Somewhat Clear - The summary makes sense overall but might be disjointed or awkward in places, requiring effort to follow.
+            - 2 = Generally Unclear - Lacks coherence or has poor phrasing that makes it hard to follow the ideas.
+            - 1 = Very Poor Clarity - The summary is very confusing or poorly structured, making it hard to understand.
+
+            **Conciseness:** Is the summary succinct while still informative? (It should omit unnecessary detail but not at the expense of coverage.)
+            - 5 = Highly Concise - The summary is brief yet covers all important information (no fluff or redundancy).
+            - 4 = Concise - Generally to-the-point, with only minor redundancy or superfluous content.
+            - 3 = Moderately Concise - Some excess detail or repetition that could be trimmed, but not egregious.
+            - 2 = Verbose - Contains a lot of unnecessary detail or repeats points, making it longer than needed.
+            - 1 = Excessively Verbose - The summary is overly long or wordy, with much content that doesn't add value.
+          PROMPT
+        end
+
+        def build_prompt
+          <<~PROMPT.strip
+            # Instructions
+            Below is an original piece of content and its summary. Evaluate the summary against the original content based on our 4 criteria. For each, you should provide:
+            - A brief justification (1-3 sentences) noting any relevant observations (e.g. what was missing, incorrect, unclear, or well-done).
+            - A score from 1 to 5 (5 = excellent, 1 = very poor).
+
+            Finally, provide an **overall evaluation** of the summary, consisting of a brief justification (1-3 sentences) and a score from 1 to 5 (5 = excellent, 1 = very poor).
+
+            # Output Format
+            Format your output as a JSON object with the following keys:
+            {
+              "coverage": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "accuracy": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "clarity": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "conciseness": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "overall": {
+                "justification": "...",
+                "score": 1-5
+              }
+            }
+            #{additional_context_prompt}
+            # Original Article/Document
+            #{original_content}
+
+            # Summary to Evaluate
+            #{summary}
+          PROMPT
+        end
+
+        def overall_score
+          parsed_response["overall"]["score"] if completed?
+        end
+
+        def overall_justification
+          parsed_response["overall"]["justification"] if completed?
+        end
+
+        def coverage_score
+          parsed_response["coverage"]["score"] if completed?
+        end
+
+        def coverage_justification
+          parsed_response["coverage"]["justification"] if completed?
+        end
+
+        def accuracy_score
+          parsed_response["accuracy"]["score"] if completed?
+        end
+
+        def accuracy_justification
+          parsed_response["accuracy"]["justification"] if completed?
+        end
+
+        def clarity_score
+          parsed_response["clarity"]["score"] if completed?
+        end
+
+        def clarity_justification
+          parsed_response["clarity"]["justification"] if completed?
+        end
+
+        def conciseness_score
+          parsed_response["conciseness"]["score"] if completed?
+        end
+
+        def conciseness_justification
+          parsed_response["conciseness"]["justification"] if completed?
+        end
+
+      private
+
+        def additional_context_prompt
+          return if additional_context.blank?
+
+          <<~PROMPT
+            \n# Additional context:
+            #{additional_context}
+          PROMPT
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals/run.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Raif
+  module Evals
+    class Run
+      attr_reader :eval_sets, :results, :output
+
+      def initialize(file_paths: nil, output: $stdout)
+        @output = output
+        @results = {}
+
+        @eval_sets = if file_paths&.any?
+          load_eval_sets_from_files(file_paths)
+        else
+          discover_eval_sets
+        end
+      end
+
+      def execute
+        # Load setup file if it exists
+        setup_file = Rails.root.join("raif_evals", "setup.rb")
+        if File.exist?(setup_file)
+          require setup_file
+        else
+          output.puts Raif::Utils::Colors.red("\n\nNo setup file found. To set up Raif evals, run:\n")
+          output.puts Raif::Utils::Colors.red("bundle exec raif evals:setup\n")
+          exit 1
+        end
+
+        output.puts "\nStarting Raif Eval Run"
+        output.puts ""
+        output.puts "Raif.config.default_llm_model_key: #{Raif.config.default_llm_model_key}"
+        output.puts ""
+        output.puts "=" * 50
+
+        @eval_sets.each do |eval_set_entry|
+          eval_set_class, file_path, line_number = if eval_set_entry.is_a?(Hash)
+            [eval_set_entry[:class], eval_set_entry[:file_path], eval_set_entry[:line_number]]
+          else
+            [eval_set_entry, nil, nil]
+          end
+
+          if line_number
+            # Running specific eval by line number
+            output.puts "\nRunning #{eval_set_class.name} at line #{line_number}"
+            output.puts "-" * 50
+
+            eval_results = run_eval_at_line(eval_set_class, file_path, line_number)
+          else
+            # Running all evals in the set
+            output.puts "\nRunning #{eval_set_class.name}"
+            output.puts "-" * 50
+
+            eval_results = eval_set_class.run(output: output)
+          end
+
+          @results[eval_set_class.name] = eval_results.map(&:to_h)
+          passed_count = eval_results.count(&:passed?)
+          total_count = eval_results.count
+
+          output.puts "-" * 50
+          output.puts "#{eval_set_class.name}: #{passed_count}/#{total_count} evals passed"
+        end
+
+        export_results
+        print_summary
+      end
+
+    private
+
+      def load_eval_sets_from_files(file_paths)
+        eval_sets = []
+
+        file_paths.each do |f|
+          file_path = f[:file_path]
+          line_number = f[:line_number]
+
+          # Convert relative path to absolute
+          absolute_path = File.expand_path(file_path)
+
+          unless File.exist?(absolute_path)
+            output.puts Raif::Utils::Colors.red("Error: File not found: #{file_path}")
+            exit 1
+          end
+
+          subclasses_before = Raif::Evals::EvalSet.subclasses
+
+          require absolute_path
+
+          loaded_eval_sets = Raif::Evals::EvalSet.subclasses - subclasses_before
+          eval_set_class = loaded_eval_sets.first
+
+          eval_set_entry = { class: eval_set_class, file_path: absolute_path }
+          eval_set_entry[:line_number] = line_number if line_number
+
+          eval_sets << eval_set_entry
+        end
+
+        eval_sets
+      end
+
+      def run_eval_at_line(eval_set_class, file_path, line_number)
+        target_eval = eval_set_class.evals.find{|e| e[:definition_line_number] == line_number }
+
+        if target_eval.nil?
+          output.puts Raif::Utils::Colors.red("Error: No eval block found at line #{line_number}")
+          return []
+        end
+
+        instance = eval_set_class.new(output: output)
+        [instance.run_eval(target_eval)]
+      end
+
+      def discover_eval_sets
+        eval_sets_dir = Rails.root.join("raif_evals", "eval_sets")
+        return [] unless eval_sets_dir.exist?
+
+        Dir.glob(eval_sets_dir.join("**", "*_eval_set.rb")).map do |file|
+          relative_path = Pathname.new(file).relative_path_from(Rails.root)
+          require Rails.root.join(relative_path)
+
+          # Extract the path components after raif_evals/eval_sets
+          path_from_eval_sets = Pathname.new(file).relative_path_from(eval_sets_dir)
+          path_parts = path_from_eval_sets.dirname.to_s.split("/")
+
+          # Remove "." if it's the only element (meaning file is in eval_sets root)
+          path_parts = [] if path_parts == ["."]
+
+          # Build the full class name
+          class_name = File.basename(file, ".rb").camelize
+          namespace_parts = ["Raif", "Evals"] + path_parts.map(&:camelize)
+          full_class_name = (namespace_parts + [class_name]).join("::")
+
+          full_class_name.constantize
+        end.select { |klass| klass < Raif::Evals::EvalSet }
+      end
+
+      def export_results
+        results_dir = Rails.root.join("raif_evals", "results")
+        FileUtils.mkdir_p(results_dir)
+
+        timestamp = Time.current.strftime("%Y%m%d_%H%M%S")
+        filename = results_dir.join("eval_run_#{timestamp}.json")
+
+        File.write(filename, JSON.pretty_generate({
+          run_at: Time.current.iso8601,
+          results: @results,
+          summary: summary_data
+        }))
+
+        output.puts "\nResults exported to: #{filename}"
+      end
+
+      def summary_data
+        total_eval_sets = @results.count
+        total_evals = @results.values.sum(&:count)
+        passed_evals = @results.values.sum { |evals| evals.count { |e| e[:passed] } }
+
+        total_expectations = @results.values.sum do |evals|
+          evals.sum { |e| e[:expectation_results].count }
+        end
+
+        passed_expectations = @results.values.sum do |evals|
+          evals.sum { |e| e[:expectation_results].count { |r| r[:status] == :passed } }
+        end
+
+        {
+          total_eval_sets: total_eval_sets,
+          total_evals: total_evals,
+          passed_evals: passed_evals,
+          total_expectations: total_expectations,
+          passed_expectations: passed_expectations
+        }
+      end
+
+      def print_summary
+        data = summary_data
+
+        output.puts ""
+        output.puts "\n" + "=" * 50
+        output.puts "SUMMARY"
+        output.puts "=" * 50
+        output.puts "Eval Sets: #{data[:total_eval_sets]}"
+        output.puts ""
+        output.puts "Evals:"
+        output.puts "  #{data[:total_evals]} total"
+        output.puts Raif::Utils::Colors.green("  #{data[:passed_evals]} passed")
+        output.puts Raif::Utils::Colors.red("  #{data[:total_evals] - data[:passed_evals]} failed")
+        output.puts ""
+        output.puts "Expectations:"
+        output.puts "  #{data[:total_expectations]} total"
+        output.puts Raif::Utils::Colors.green("  #{data[:passed_expectations]} passed")
+        output.puts Raif::Utils::Colors.red("  #{data[:total_expectations] - data[:passed_expectations]} failed")
+        output.puts ""
+      end
+    end
+  end
+end