raif 1.2.2 → 1.4.0

data/lib/raif/evals/llm_judges/summarization.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module LlmJudges
+      class Summarization < Raif::Evals::LlmJudge
+        run_with :original_content # the original content to evaluate the summary against
+        run_with :summary # the summary to evaluate against the original content
+
+        json_response_schema do
+          object :coverage do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :accuracy do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :clarity do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :conciseness do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+
+          object :overall do
+            string :justification, description: "Justification for the score"
+            number :score, description: "Score from 1 to 5", enum: [1, 2, 3, 4, 5]
+          end
+        end
+
+        def build_system_prompt
+          <<~PROMPT.strip
+            You are an impartial expert judge of summary quality. You'll be provided a original piece of content and its summary. Your job is to evaluate the summary against the original content based on the following criteria, and assign a score from 1 to 5 for each (5 = excellent, 1 = very poor):
+
+            **Coverage (Relevance & Completeness):** Does the summary capture all the important points of the original content?
+            - 5 = Excellent Coverage - Nearly all key points and essential details from the content are present in the summary, with no major omissions.
+            - 4 = Good Coverage - Most important points are included, but a minor detail or two might be missing.
+            - 3 = Fair Coverage - Some main points appear, but the summary misses or glosses over other important information.
+            - 2 = Poor Coverage - Many critical points from the content are missing; the summary is incomplete.
+            - 1 = Very Poor - The summary fails to include most of the content's main points (highly incomplete).
+
+            **Accuracy (Faithfulness to the Source):** Is the summary factually correct and free of hallucinations or misrepresentations of the content?
+            - 5 = Fully Accurate - All statements in the summary are correct and directly supported by the content. No errors or invented information.
+            - 4 = Mostly Accurate - The summary is generally accurate with perhaps one minor error or slight ambiguity, but no significant falsehoods.
+            - 3 = Some Inaccuracies - Contains a few errors or unsupported claims from the content, but overall captures the gist correctly.
+            - 2 = Mostly Inaccurate - Multiple statements in the summary are incorrect or not supported by the content.
+            - 1 = Completely Inaccurate - The summary seriously distorts or contradicts the content; many claims are false or not in the source.
+
+            **Clarity and Coherence:** Is the summary well-written and easy to understand? (Consider organization, flow, and whether it would make sense to a reader.)
+            - 5 = Very Clear & Coherent - The summary is logically organized, flows well, and would be easily understood by the target reader. No confusion or ambiguity.
+            - 4 = Mostly Clear - Readable and mostly well-structured, though a sentence or transition could be smoother.
+            - 3 = Somewhat Clear - The summary makes sense overall but might be disjointed or awkward in places, requiring effort to follow.
+            - 2 = Generally Unclear - Lacks coherence or has poor phrasing that makes it hard to follow the ideas.
+            - 1 = Very Poor Clarity - The summary is very confusing or poorly structured, making it hard to understand.
+
+            **Conciseness:** Is the summary succinct while still informative? (It should omit unnecessary detail but not at the expense of coverage.)
+            - 5 = Highly Concise - The summary is brief yet covers all important information (no fluff or redundancy).
+            - 4 = Concise - Generally to-the-point, with only minor redundancy or superfluous content.
+            - 3 = Moderately Concise - Some excess detail or repetition that could be trimmed, but not egregious.
+            - 2 = Verbose - Contains a lot of unnecessary detail or repeats points, making it longer than needed.
+            - 1 = Excessively Verbose - The summary is overly long or wordy, with much content that doesn't add value.
+          PROMPT
+        end
+
+        def build_prompt
+          <<~PROMPT.strip
+            # Instructions
+            Below is an original piece of content and its summary. Evaluate the summary against the original content based on our 4 criteria. For each, you should provide:
+            - A brief justification (1-3 sentences) noting any relevant observations (e.g. what was missing, incorrect, unclear, or well-done).
+            - A score from 1 to 5 (5 = excellent, 1 = very poor).
+
+            Finally, provide an **overall evaluation** of the summary, consisting of a brief justification (1-3 sentences) and a score from 1 to 5 (5 = excellent, 1 = very poor).
+
+            # Output Format
+            Format your output as a JSON object with the following keys:
+            {
+              "coverage": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "accuracy": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "clarity": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "conciseness": {
+                "justification": "...",
+                "score": 1-5
+              },
+              "overall": {
+                "justification": "...",
+                "score": 1-5
+              }
+            }
+            #{additional_context_prompt}
+            # Original Article/Document
+            #{original_content}
+
+            # Summary to Evaluate
+            #{summary}
+          PROMPT
+        end
+
+        def overall_score
+          parsed_response["overall"]["score"] if completed?
+        end
+
+        def overall_justification
+          parsed_response["overall"]["justification"] if completed?
+        end
+
+        def coverage_score
+          parsed_response["coverage"]["score"] if completed?
+        end
+
+        def coverage_justification
+          parsed_response["coverage"]["justification"] if completed?
+        end
+
+        def accuracy_score
+          parsed_response["accuracy"]["score"] if completed?
+        end
+
+        def accuracy_justification
+          parsed_response["accuracy"]["justification"] if completed?
+        end
+
+        def clarity_score
+          parsed_response["clarity"]["score"] if completed?
+        end
+
+        def clarity_justification
+          parsed_response["clarity"]["justification"] if completed?
+        end
+
+        def conciseness_score
+          parsed_response["conciseness"]["score"] if completed?
+        end
+
+        def conciseness_justification
+          parsed_response["conciseness"]["justification"] if completed?
+        end
+
+      private
+
+        def additional_context_prompt
+          return if additional_context.blank?
+
+          <<~PROMPT
+            \n# Additional context:
+            #{additional_context}
+          PROMPT
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals/run.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Raif
+  module Evals
+    class Run
+      attr_reader :eval_sets, :results, :output
+
+      def initialize(file_paths: nil, output: $stdout)
+        @output = output
+        @results = {}
+
+        @eval_sets = if file_paths&.any?
+          load_eval_sets_from_files(file_paths)
+        else
+          discover_eval_sets
+        end
+      end
+
+      def execute
+        # Load setup file if it exists
+        setup_file = Rails.root.join("raif_evals", "setup.rb")
+        if File.exist?(setup_file)
+          require setup_file
+        else
+          output.puts Raif::Utils::Colors.red("\n\nNo setup file found. To set up Raif evals, run:\n")
+          output.puts Raif::Utils::Colors.red("bundle exec raif evals:setup\n")
+          exit 1
+        end
+
+        output.puts "\nStarting Raif Eval Run"
+        output.puts ""
+        output.puts "Raif.config.default_llm_model_key: #{Raif.config.default_llm_model_key}"
+        output.puts "Raif.config.evals_default_llm_judge_model_key: #{Raif.config.evals_default_llm_judge_model_key}"
+        output.puts ""
+        output.puts "=" * 50
+
+        @eval_sets.each do |eval_set_entry|
+          eval_set_class, file_path, line_number = if eval_set_entry.is_a?(Hash)
+            [eval_set_entry[:class], eval_set_entry[:file_path], eval_set_entry[:line_number]]
+          else
+            [eval_set_entry, nil, nil]
+          end
+
+          if line_number
+            # Running specific eval by line number
+            output.puts "\nRunning #{eval_set_class.name} at line #{line_number}"
+            output.puts "-" * 50
+
+            eval_results = run_eval_at_line(eval_set_class, file_path, line_number)
+          else
+            # Running all evals in the set
+            output.puts "\nRunning #{eval_set_class.name}"
+            output.puts "-" * 50
+
+            eval_results = eval_set_class.run(output: output)
+          end
+
+          @results[eval_set_class.name] = eval_results.map(&:to_h)
+          passed_count = eval_results.count(&:passed?)
+          total_count = eval_results.count
+
+          output.puts "-" * 50
+          output.puts "#{eval_set_class.name}: #{passed_count}/#{total_count} evals passed"
+        end
+
+        export_results
+        print_summary
+      end
+
+    private
+
+      def load_eval_sets_from_files(file_paths)
+        eval_sets = []
+
+        file_paths.each do |f|
+          file_path = f[:file_path]
+          line_number = f[:line_number]
+
+          # Convert relative path to absolute
+          absolute_path = File.expand_path(file_path)
+
+          unless File.exist?(absolute_path)
+            output.puts Raif::Utils::Colors.red("Error: File not found: #{file_path}")
+            exit 1
+          end
+
+          subclasses_before = Raif::Evals::EvalSet.subclasses
+
+          require absolute_path
+
+          loaded_eval_sets = Raif::Evals::EvalSet.subclasses - subclasses_before
+          eval_set_class = loaded_eval_sets.first
+
+          eval_set_entry = { class: eval_set_class, file_path: absolute_path }
+          eval_set_entry[:line_number] = line_number if line_number
+
+          eval_sets << eval_set_entry
+        end
+
+        eval_sets
+      end
+
+      def run_eval_at_line(eval_set_class, file_path, line_number)
+        target_eval = eval_set_class.evals.find{|e| e[:definition_line_number] == line_number }
+
+        if target_eval.nil?
+          output.puts Raif::Utils::Colors.red("Error: No eval block found at line #{line_number}")
+          return []
+        end
+
+        instance = eval_set_class.new(output: output)
+        [instance.run_eval(target_eval)]
+      end
+
+      def discover_eval_sets
+        eval_sets_dir = Rails.root.join("raif_evals", "eval_sets")
+        return [] unless eval_sets_dir.exist?
+
+        Dir.glob(eval_sets_dir.join("**", "*_eval_set.rb")).map do |file|
+          relative_path = Pathname.new(file).relative_path_from(Rails.root)
+          require Rails.root.join(relative_path)
+
+          # Extract the path components after raif_evals/eval_sets
+          path_from_eval_sets = Pathname.new(file).relative_path_from(eval_sets_dir)
+          path_parts = path_from_eval_sets.dirname.to_s.split("/")
+
+          # Remove "." if it's the only element (meaning file is in eval_sets root)
+          path_parts = [] if path_parts == ["."]
+
+          # Build the full class name
+          class_name = File.basename(file, ".rb").camelize
+          namespace_parts = ["Raif", "Evals"] + path_parts.map(&:camelize)
+          full_class_name = (namespace_parts + [class_name]).join("::")
+
+          full_class_name.constantize
+        end.select { |klass| klass < Raif::Evals::EvalSet }
+      end
+
+      def export_results
+        results_dir = Rails.root.join("raif_evals", "results")
+        FileUtils.mkdir_p(results_dir)
+
+        timestamp = Time.current.strftime("%Y%m%d_%H%M%S")
+        filename = results_dir.join("eval_run_#{timestamp}.json")
+
+        File.write(filename, JSON.pretty_generate({
+          run_at: Time.current.iso8601,
+          results: @results,
+          summary: summary_data
+        }))
+
+        output.puts "\nResults exported to: #{filename}"
+      end
+
+      def summary_data
+        total_eval_sets = @results.count
+        total_evals = @results.values.sum(&:count)
+        passed_evals = @results.values.sum { |evals| evals.count { |e| e[:passed] } }
+
+        total_expectations = @results.values.sum do |evals|
+          evals.sum { |e| e[:expectation_results].count }
+        end
+
+        passed_expectations = @results.values.sum do |evals|
+          evals.sum { |e| e[:expectation_results].count { |r| r[:status] == :passed } }
+        end
+
+        {
+          total_eval_sets: total_eval_sets,
+          total_evals: total_evals,
+          passed_evals: passed_evals,
+          total_expectations: total_expectations,
+          passed_expectations: passed_expectations
+        }
+      end
+
+      def print_summary
+        data = summary_data
+
+        output.puts ""
+        output.puts "\n" + "=" * 50
+        output.puts "SUMMARY"
+        output.puts "=" * 50
+        output.puts "Eval Sets: #{data[:total_eval_sets]}"
+        output.puts ""
+        output.puts "Evals:"
+        output.puts "  #{data[:total_evals]} total"
+        output.puts Raif::Utils::Colors.green("  #{data[:passed_evals]} passed")
+        output.puts Raif::Utils::Colors.red("  #{data[:total_evals] - data[:passed_evals]} failed")
+        output.puts ""
+        output.puts "Expectations:"
+        output.puts "  #{data[:total_expectations]} total"
+        output.puts Raif::Utils::Colors.green("  #{data[:passed_expectations]} passed")
+        output.puts Raif::Utils::Colors.red("  #{data[:total_expectations] - data[:passed_expectations]} failed")
+        output.puts ""
+      end
+    end
+  end
+end

data/lib/raif/evals/scoring_rubric.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    # ScoringRubric provides a standardized way to define evaluation criteria with
+    # multiple scoring levels. Each level can define either a score range or a single
+    # score value, along with descriptive text explaining what qualifies for that score.
+    #
+    # @example Creating a custom rubric
+    #   rubric = ScoringRubric.new(
+    #     name: :technical_accuracy,
+    #     description: "Evaluates technical correctness and precision",
+    #     levels: [
+    #       { score_range: (9..10), description: "Technically perfect with no errors" },
+    #       { score_range: (7..8), description: "Mostly correct with minor technical issues" },
+    #       { score_range: (5..6), description: "Generally correct but some technical problems" },
+    #       { score_range: (3..4), description: "Significant technical errors present" },
+    #       { score_range: (0..2), description: "Technically incorrect or misleading" }
+    #     ]
+    #   )
+    #
+    # @example Integer scoring levels
+    #   rubric = ScoringRubric.new(
+    #     name: :technical_accuracy ,
+    #     description: "Evaluates technical correctness and precision",
+    #     levels: [
+    #       { score: 5, description: "Technically perfect with no errors" },
+    #       { score: 4, description: "Mostly correct with minor technical issues" },
+    #       { score: 3, description: "Generally correct but some technical problems" },
+    #       { score: 2, description: "Significant technical errors present" },
+    #       { score: 1, description: "Mostly incorrect or misleading" },
+    #       { score: 0, description: "Completely incorrect or misleading" }
+    #     ]
+    #   )
+    #
+    # @example Using built-in rubrics
+    #   accuracy_rubric = ScoringRubric.accuracy
+    #   helpfulness_rubric = ScoringRubric.helpfulness
+    #   clarity_rubric = ScoringRubric.clarity
+    #
+    class ScoringRubric
+      # @return [Symbol] The rubric's identifier name
+      attr_reader :name
+      # @return [String] Human-readable description of what this rubric evaluates
+      attr_reader :description
+      # @return [Array<Hash>] Array of scoring level definitions
+      attr_reader :levels
+
+      # Creates a new ScoringRubric with the specified criteria.
+      #
+      # @param name [Symbol] Identifier for this rubric (e.g., :accuracy, :helpfulness)
+      # @param description [String] Human-readable description of what this rubric evaluates
+      # @param levels [Array<Hash>] Array of scoring level definitions. Each level must contain
+      #   either :score (Integer) or :score_range (Range), plus :description (String)
+      def initialize(name:, description:, levels:)
+        @name = name
+        @description = description
+        @levels = levels
+      end
+
+      # Converts the rubric into a formatted string suitable for LLM prompts.
+      #
+      # The output includes the rubric description followed by a detailed breakdown
+      # of all scoring levels with their criteria.
+      #
+      # @return [String] Formatted rubric text ready for inclusion in prompts
+      #
+      # @example Output format
+      #   "Evaluates factual correctness and precision
+      #
+      #   Scoring levels:
+      #   - 9-10: Completely accurate with no errors
+      #   - 7-8: Mostly accurate with minor imprecisions
+      #   - 5-6: Generally accurate but some notable errors"
+      #
+      # @raise [ArgumentError] If a level doesn't contain :score or :score_range
+      def to_prompt
+        prompt = "#{description}\n\nScoring levels:\n"
+
+        levels.each do |level|
+          if level.key?(:score)
+            score = level[:score]
+            prompt += "- #{score}: #{level[:description]}\n"
+          else
+            range = level[:score_range]
+            min, max = case range
+            when Range
+              [range.begin, range.exclude_end? ? range.end - 1 : range.end]
+            else
+              raise ArgumentError, "level must include :score or :score_range (Range)"
+            end
+            prompt += "- #{min}-#{max}: #{level[:description]}\n"
+          end
+        end
+
+        prompt.strip
+      end
+
+      class << self
+        # Creates a rubric for evaluating factual accuracy and correctness.
+        #
+        # This rubric focuses on whether information is factually correct,
+        # precise, and free from errors or misconceptions.
+        #
+        # @return [ScoringRubric] Pre-configured accuracy rubric (1-5 scale)
+        #
+        # @example
+        #   rubric = ScoringRubric.accuracy
+        #   expect_llm_judge_score(response, scoring_rubric: rubric, min_passing_score: 4)
+        def accuracy
+          new(
+            name: :accuracy,
+            description: "Evaluates factual correctness and precision",
+            levels: [
+              { score: 5, description: "Completely accurate with no errors" },
+              { score: 4, description: "Mostly accurate with minor imprecisions" },
+              { score: 3, description: "Generally accurate but some notable errors" },
+              { score: 2, description: "Significant inaccuracies present" },
+              { score: 1, description: "Mostly or entirely inaccurate" }
+            ]
+          )
+        end
+
+        # Creates a rubric for evaluating how well content addresses user needs.
+        #
+        # This rubric assesses whether the response is useful, relevant, and
+        # effectively helps the user accomplish their goals.
+        #
+        # @return [ScoringRubric] Pre-configured helpfulness rubric (1-5 scale)
+        #
+        # @example
+        #   rubric = ScoringRubric.helpfulness
+        #   expect_llm_judge_score(response, scoring_rubric: rubric, min_passing_score: 4)
+        def helpfulness
+          new(
+            name: :helpfulness,
+            description: "Evaluates how well the response addresses user needs",
+            levels: [
+              { score: 5, description: "Extremely helpful, fully addresses the need" },
+              { score: 4, description: "Very helpful with good coverage" },
+              { score: 3, description: "Moderately helpful but missing some aspects" },
+              { score: 2, description: "Somewhat helpful but significant gaps" },
+              { score: 1, description: "Not helpful or misleading" }
+            ]
+          )
+        end
+
+        # Creates a rubric for evaluating clarity and comprehensibility.
+        #
+        # This rubric focuses on how easy content is to understand, whether
+        # it's well-organized, and if the language is appropriate for the audience.
+        #
+        # @return [ScoringRubric] Pre-configured clarity rubric (1-5 scale)
+        #
+        # @example
+        #   rubric = ScoringRubric.clarity
+        #   expect_llm_judge_score(response, scoring_rubric: rubric, min_passing_score: 4)
+        def clarity
+          new(
+            name: :clarity,
+            description: "Evaluates clarity and comprehensibility",
+            levels: [
+              { score: 5, description: "Crystal clear and easy to understand" },
+              { score: 4, description: "Clear with minor ambiguities" },
+              { score: 3, description: "Generally clear but some confusion" },
+              { score: 2, description: "Unclear in significant ways" },
+              { score: 1, description: "Very unclear or incomprehensible" }
+            ]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/raif/evals.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "raif/evals/expectation_result"
+require "raif/evals/eval"
+require "raif/evals/eval_set"
+require "raif/evals/run"
+require "raif/evals/llm_judge"
+require "raif/evals/llm_judges/binary"
+require "raif/evals/llm_judges/comparative"
+require "raif/evals/llm_judges/scored"
+require "raif/evals/llm_judges/summarization"
+require "raif/evals/scoring_rubric"
+
+module Raif
+  module Evals
+    # Namespace modules for organizing eval sets
+    module Tasks
+    end
+
+    module Conversations
+    end
+
+    module Agents
+    end
+  end
+end

data/lib/raif/json_schema_builder.rb

@@ -10,6 +10,20 @@ module Raif
       @items_schema = nil
     end
 
+    # Build schema with instance context for instance-dependent schemas
+    # The block receives the instance as a parameter and has access to the builder methods
+    #
+    # @param instance [Object] The instance to use as context
+    # @param block [Proc] The block to evaluate with instance context
+    # @return [JsonSchemaBuilder] self for chaining
+    def build_with_instance(instance, &block)
+      # Evaluate the block in the context of the builder, passing the instance as parameter
+      # This allows the block to use both builder methods (string, integer, etc.)
+      # and access the instance parameter for conditional logic
+      instance_exec(instance, &block)
+      self
+    end
+
     def string(name, options = {})
       add_property(name, "string", options)
     end