active_genie 0.0.10 → 0.0.18

data/lib/active_genie/ranking/player.rb

@@ -0,0 +1,92 @@
+require 'digest'
+
+module ActiveGenie::Ranking
+  class Player
+    def initialize(params)
+      params = { content: params } if params.is_a?(String)
+
+      @content = params.dig(:content) || params
+      @id = params.dig(:id) || Digest::MD5.hexdigest(@content)
+      @score = params.dig(:score) || nil
+      @elo = params.dig(:elo) || nil
+      @ffa_win_count = params.dig(:ffa_win_count) || 0
+      @ffa_lose_count = params.dig(:ffa_lose_count) || 0
+      @ffa_draw_count = params.dig(:ffa_draw_count) || 0
+      @eliminated = params.dig(:eliminated) || nil
+    end
+
+    attr_reader :id, :content, :score, :elo, 
+      :ffa_win_count, :ffa_lose_count, :ffa_draw_count,
+      :eliminated
+    attr_accessor :rank
+
+    def score=(value)
+      ActiveGenie::Logger.debug({ code: :new_score, player_id: id, score: value }) if value != @score
+      @score = value
+      @elo = generate_elo_by_score
+    end
+
+    def elo=(value)
+      ActiveGenie::Logger.debug({ code: :new_elo, player_id: id, elo: value }) if value != @elo
+      @elo = value
+    end
+
+    def eliminated=(value)
+      ActiveGenie::Logger.debug({ code: :new_eliminated, player_id: id, eliminated: value }) if value != @eliminated
+      @eliminated = value
+    end
+
+    def draw!
+      @ffa_draw_count += 1
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'draw', ffa_score: })
+    end
+
+    def win!
+      @ffa_win_count += 1
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'win', ffa_score: })
+    end
+
+    def lose!
+      @ffa_lose_count += 1
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'lose', ffa_score: })
+    end
+
+    def ffa_score
+      @ffa_win_count * 3 + @ffa_draw_count
+    end
+
+    def to_h
+      {
+        id:, content:, score:, elo:,
+        ffa_win_count:, ffa_lose_count:, ffa_draw_count:,
+        eliminated:, ffa_score:
+      }
+    end
+
+    def method_missing(method_name, *args, &block)
+      if method_name == :[] && args.size == 1
+        attr_name = args.first.to_sym
+
+        if respond_to?(attr_name) 
+          return send(attr_name)
+        else
+          return nil
+        end
+      end
+
+      super
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      method_name == :[] || super
+    end
+
+    def generate_elo_by_score
+      BASE_ELO + ((@score || 0) - 50)
+    end
+
+    private
+    
+    BASE_ELO = 1000
+  end
+end

data/lib/active_genie/{league → ranking}/players_collection.rb

@@ -1,7 +1,6 @@
-require_relative '../utils/math'
 require_relative './player'
 
-module ActiveGenie::Leaderboard
+module ActiveGenie::Ranking
   class PlayersCollection
     def initialize(param_players)
       @players = build(param_players)
@@ -9,9 +8,11 @@ module ActiveGenie::Leaderboard
     attr_reader :players
 
     def coefficient_of_variation
-      score_list = eligible.map(&:score)
+      score_list = eligible.map(&:score).compact
+      return nil if score_list.empty?
+
       mean = score_list.sum.to_f / score_list.size
-      return nil if mean == 0  # To avoid division by zero
+      return nil if mean == 0
 
       variance = score_list.map { |num| (num - mean) ** 2 }.sum / score_list.size
       standard_deviation = Math.sqrt(variance)
@@ -19,11 +20,11 @@ module ActiveGenie::Leaderboard
       (standard_deviation / mean) * 100
     end
 
-    def tier_relegation
+    def calc_relegation_tier
       eligible[(tier_size*-1)..-1]
     end
 
-    def tier_defense
+    def calc_defender_tier
       eligible[(tier_size*-2)...(tier_size*-1)]
     end
 
@@ -35,22 +36,28 @@ module ActiveGenie::Leaderboard
       @players.reject(&:eliminated).size
     end
 
+    def elo_eligible?
+      eligible.size > 15
+    end
+
+    def sorted
+      sorted_players = @players.sort_by { |p| [-p.ffa_score, -(p.elo || 0), -(p.score || 0)] }
+      sorted_players.each_with_index { |p, i| p.rank = i + 1 }
+      sorted_players
+    end
+
     def to_h
-      sorted.map(&:to_h)
+      sorted.map { |p| p.to_h }
     end
 
     def method_missing(...)
       @players.send(...)
     end
 
-    def sorted
-      @players.sort_by { |p| [-p.league_score, -(p.elo || 0), -p.score] }
-    end
-
     private
 
     def build(param_players)
-      param_players.map { |player| Player.new(player) }
+      param_players.map { |p| Player.new(p) }
     end
 
     # Returns the number of players to battle in each round

data/lib/active_genie/ranking/ranking.rb

@@ -0,0 +1,153 @@
+require_relative '../concerns/loggable'
+require_relative './players_collection'
+require_relative './free_for_all'
+require_relative './elo_round'
+require_relative './ranking_scoring'
+
+# This class orchestrates player ranking through multiple evaluation stages
+# using Elo ranking and free-for-all match simulations.
+# 1. Sets initial scores
+# 2. Eliminates low performers
+# 3. Runs Elo ranking (for large groups)
+# 4. Conducts free-for-all matches
+#
+# @example Basic usage
+#   Ranking.call(players, criteria)
+#
+# @param param_players [Array<Hash|String>] Collection of player objects to evaluate
+#   Example: ["Circle", "Triangle", "Square"]
+#            or
+#   [
+#     { content: "Circle", score: 10 },
+#     { content: "Triangle", score: 7 },
+#     { content: "Square", score: 5 }
+#   ]
+# @param criteria [String] Evaluation criteria configuration
+#   Example: "What is more similar to the letter 'O'?"
+# @param config [Hash] Additional configuration config
+#   Example: { model: "gpt-4o", api_key: ENV['OPENAI_API_KEY'] }
+# @return [Hash] Final ranked player results
+module ActiveGenie::Ranking
+  class Ranking
+    include ActiveGenie::Concerns::Loggable
+
+    def self.call(...)
+      new(...).call
+    end
+
+    def initialize(param_players, criteria, reviewers: [], config: {})
+      @criteria = criteria
+      @reviewers = Array(reviewers).compact.uniq
+      @config = ActiveGenie::Configuration.to_h(config)
+      @players = PlayersCollection.new(param_players)
+      @elo_rounds_played = 0
+      @elo_round_battle_count = 0
+      @free_for_all_battle_count = 0
+      @total_tokens = 0
+      @start_time = Time.now
+    end
+
+    def call
+      initial_log
+
+      set_initial_player_scores!
+      eliminate_obvious_bad_players!
+
+      while @players.elo_eligible?
+        elo_report = run_elo_round!
+        eliminate_relegation_players!
+        rebalance_players!(elo_report)
+      end
+
+      run_free_for_all!
+      final_logs
+
+      @players.sorted
+    end
+
+    private
+
+    SCORE_VARIATION_THRESHOLD = 15
+    ELIMINATION_VARIATION = 'variation_too_high'
+    ELIMINATION_RELEGATION = 'relegation_tier'
+    
+    with_logging_context :log_context, ->(log) { 
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    }
+
+    def initial_log
+      @players.each { |p| ActiveGenie::Logger.debug({ code: :new_player, player: p.to_h }) }
+    end
+
+    def set_initial_player_scores!
+      RankingScoring.call(@players, @criteria, reviewers: @reviewers, config: @config)
+    end
+
+    def eliminate_obvious_bad_players!
+      while @players.coefficient_of_variation >= SCORE_VARIATION_THRESHOLD
+        @players.eligible.last.eliminated = ELIMINATION_VARIATION
+      end
+    end
+
+    def run_elo_round!
+      @elo_rounds_played += 1
+
+      elo_report = EloRound.call(@players, @criteria, config: @config)
+
+      @elo_round_battle_count += elo_report[:battles_count]
+
+      elo_report
+    end
+
+    def eliminate_relegation_players!
+      @players.calc_relegation_tier.each { |player| player.eliminated = ELIMINATION_RELEGATION }
+    end
+
+    def rebalance_players!(elo_report)
+      return if elo_report[:highest_elo_diff].negative?
+
+      @players.eligible.each do |player|
+        next if elo_report[:players_in_round].include?(player.id)
+
+        player.elo += elo_report[:highest_elo_diff]
+      end
+    end
+
+    def run_free_for_all!
+      ffa_report = FreeForAll.call(@players, @criteria, config: @config)
+
+      @free_for_all_battle_count += ffa_report[:battles_count]
+    end
+
+    def report
+      {
+        ranking_id: ranking_id,
+        players_count: @players.size,
+        variation_too_high: @players.select { |player| player.eliminated == ELIMINATION_VARIATION }.size,
+        elo_rounds_played: @elo_rounds_played,
+        elo_round_battle_count: @elo_round_battle_count,
+        relegation_tier: @players.select { |player| player.eliminated == ELIMINATION_RELEGATION }.size,
+        ffa_round_battle_count: @free_for_all_battle_count,
+        top3: @players.eligible[0..2].map(&:id),
+        total_tokens: @total_tokens,
+        duration_seconds: Time.now - @start_time,
+      }
+    end
+    
+    def final_logs
+      ActiveGenie::Logger.debug({ code: :ranking_final, players: @players.sorted.map(&:to_h) })
+      ActiveGenie::Logger.info({ code: :ranking, **report })
+    end
+
+    def log_context
+      { config: @config[:log], ranking_id: }
+    end
+
+    def ranking_id
+      player_ids = @players.map(&:id).join(',')
+      ranking_unique_key = [player_ids, @criteria, @config.to_json].join('-')
+
+      Digest::MD5.hexdigest(ranking_unique_key)
+    end
+  end
+end

data/lib/active_genie/ranking/ranking_scoring.rb

@@ -0,0 +1,71 @@
+require_relative '../scoring/recommended_reviewers'
+
+module ActiveGenie::Ranking
+  class RankingScoring
+    def self.call(...)
+      new(...).call
+    end
+
+    def initialize(players, criteria, reviewers: [], config: {})
+      @players = players
+      @criteria = criteria
+      @config = ActiveGenie::Configuration.to_h(config)
+      @reviewers = Array(reviewers).compact.uniq
+    end
+
+    def call
+      ActiveGenie::Logger.with_context(log_context) do
+        @reviewers = generate_reviewers
+
+        players_without_score.each do |player|
+          # TODO: This can take a while, can be parallelized
+          player.score = generate_score(player)
+        end
+      end
+    end
+
+    private
+
+    def players_without_score
+      @players_without_score ||= @players.select { |player| player.score.nil? }
+    end
+
+    def generate_score(player)
+      score, reasoning = ActiveGenie::Scoring.call(
+        player.content,
+        @criteria,
+        @reviewers,
+        config: @config
+      ).values_at('final_score', 'final_reasoning')
+
+      ActiveGenie::Logger.debug({ code: :new_score, player_id: player.id, score:, reasoning: })
+
+      score
+    end
+
+    def generate_reviewers
+      return @reviewers if @reviewers.size > 0
+
+      reviewer1, reviewer2, reviewer3 = ActiveGenie::Scoring::RecommendedReviewers.call(
+        [@players.sample.content, @players.sample.content].join("\n\n"),
+        @criteria,
+        config: @config
+      ).values_at('reviewer1', 'reviewer2', 'reviewer3')
+      
+      ActiveGenie::Logger.debug({ code: :new_reviewers, reviewers: [reviewer1, reviewer2, reviewer3] })
+
+      [reviewer1, reviewer2, reviewer3]
+    end
+
+    def log_context
+      { ranking_scoring_id: }
+    end
+
+    def ranking_scoring_id
+      player_ids = players_without_score.map(&:id).join(',')
+      ranking_unique_key = [player_ids, @criteria, @config.to_json].join('-')
+
+      Digest::MD5.hexdigest(ranking_unique_key)
+    end
+  end
+end

data/lib/active_genie/ranking.rb

@@ -0,0 +1,12 @@
+require_relative 'ranking/ranking'
+
+module ActiveGenie
+  # See the [ranking README](lib/active_genie/ranking/README.md) for more information.
+  module Ranking
+    module_function
+
+    def call(...)
+      Ranking.call(...)
+    end
+  end
+end

data/lib/active_genie/scoring/README.md

@@ -58,7 +58,7 @@ Main interface for scoring text content.
 - `reviewers` [Array<String>] - Optional list of specific reviewers
 - `config` [Hash] - Additional configuration config
 
-### `RecommendedReviews.call(text, criteria, config: {})`
+### `RecommendedReviewers.call(text, criteria, config: {})`
 Recommends appropriate reviewers based on content and criteria.
 
 #### Parameters

data/lib/active_genie/scoring/basic.rb

@@ -25,15 +25,15 @@ module ActiveGenie::Scoring
     # @return [Hash] The evaluation result containing the scores and reasoning
     #   @return [Number] :final_score The final score of the text based on the criteria and reviewers
     #   @return [String] :final_reasoning Detailed explanation of why the final score was reached
-    def self.call(text, criteria, reviewers = [], config: {})
-      new(text, criteria, reviewers, config:).call
+    def self.call(...)
+      new(...).call
     end
 
     def initialize(text, criteria, reviewers = [], config: {})
       @text = text
       @criteria = criteria
       @reviewers = Array(reviewers).compact.uniq
-      @config = config
+      @config = ActiveGenie::Configuration.to_h(config)
     end
 
     def call
@@ -43,6 +43,42 @@ module ActiveGenie::Scoring
         {  role: 'user', content: "Text to score: #{@text}" }, 
       ]
 
+      properties = build_properties
+
+      function = {
+        name: 'scoring',
+        description: 'Score the text based on the given criteria.',
+        schema: {
+          type: "object",
+          properties:,
+          required: properties.keys
+        }
+      }
+
+      result = ::ActiveGenie::Clients::UnifiedClient.function_calling(
+        messages,
+        function,
+        model_tier: 'lower_tier',
+        config: @config
+      )
+
+      result['final_score'] = 0 if result['final_score'].nil?
+
+      ActiveGenie::Logger.debug({
+        code: :scoring,
+        text: @text[0..30],
+        criteria: @criteria[0..30],
+        reviewers: get_or_recommend_reviewers,
+        score: result['final_score'],
+        reasoning: result['final_reasoning']
+      })
+
+      result
+    end
+
+    private
+    
+    def build_properties
       properties = {}
       get_or_recommend_reviewers.each do |reviewer|
         properties["#{reviewer}_reasoning"] = {
@@ -57,70 +93,78 @@ module ActiveGenie::Scoring
         }
       end
 
-      function = {
-        name: 'scoring',
-        description: 'Score the text based on the given criteria.',
-        schema: {
-          type: "object",
-          properties: {
-            **properties,
-            final_score: {
-              type: 'number',
-              description: 'The final score based on the previous reviewers',
-            },
-            final_reasoning: {
-              type: 'string',
-              description: 'The final reasoning based on the previous reviewers',
-            }
-          }
-        }
+      properties[:final_score] = {
+        type: 'number',
+        description: 'The final score based on the previous reviewers',
+      }
+      properties[:final_reasoning] = {
+        type: 'string',
+        description: 'The final reasoning based on the previous reviewers',
       }
 
-      ::ActiveGenie::Clients::UnifiedClient.function_calling(messages, function, config:)
+      properties
     end
 
-    private
-
     def get_or_recommend_reviewers
       @get_or_recommend_reviewers ||= if @reviewers.count > 0 
         @reviewers
       else
-        recommended_reviews = RecommendedReviews.call(@text, @criteria, config:)
+        result = RecommendedReviewers.call(@text, @criteria, config: @config)
 
-        [recommended_reviews[:reviewer1], recommended_reviews[:reviewer2], recommended_reviews[:reviewer3]]
+        [result['reviewer1'], result['reviewer2'], result['reviewer3']]
       end
     end
 
     PROMPT = <<~PROMPT
-    Evaluate and score the provided text based on predefined criteria, which may include rules, keywords, or patterns. Use a scoring range of 0 to 100, with 100 representing the highest possible score. Follow the instructions below to ensure an accurate and objective assessment.
+    Evaluate and score the provided text based on predefined criteria, using a scoring range of 0 to 100 with 100 representing the highest possible score.
+
+    Follow the instructions below to ensure a comprehensive and objective assessment. 
 
     # Evaluation Process
-    1. **Analysis**: Thoroughly compare the text against each criterion to ensure comprehensive evaluation.
-    2. **Document Deviations**: Clearly identify and document any areas where the content does not align with the specified criteria.
-    3. **Highlight Strengths**: Emphasize notable features or elements that enhance the overall quality or effectiveness of the content.
-    4. **Identify Weaknesses**: Specify areas where the content fails to meet the criteria or where improvements could be made.
 
-    # Output Requirements
-    Provide a detailed review, including:
-      - A final score (0-100)
-      - Specific reasoning for the assigned score, covering all evaluated criteria.
-      - Ensure the reasoning includes both positive aspects and suggested improvements.
+    1. **Analysis**: 
+      - Thoroughly compare the text against each criterion for a comprehensive evaluation.
+      
+    2. **Document Deviations**:
+      - Identify and document areas where the content does not align with the specified criteria.
+
+    3. **Highlight Strengths**:
+      - Note notable features or elements that enhance the quality or effectiveness of the content.
+
+    4. **Identify Weaknesses**:
+      - Specify areas where the content fails to meet the criteria or where improvements could be made.
+
+    # Scoring Fairness
+
+    - Ensure the assigned score reflects both the alignment with the criteria and the content's effectiveness.
+    - Consider if the fulfillment of other criteria compensates for areas lacking extreme details.
+
+    # Scoring Range
+
+    Segment scores into five parts before assigning a final score:
+    - **Terrible**: 0-20 - Content does not meet the criteria.
+    - **Bad**: 21-40 - Content is substandard but meets some criteria.
+    - **Average**: 41-60 - Content meets criteria with room for improvement.
+    - **Good**: 61-80 - Content exceeds criteria and is above average.
+    - **Great**: 81-100 - Content exceeds all expectations.
 
     # Guidelines
-    - Maintain objectivity, avoiding biases or preconceived notions.
-    - Deconstruct each criterion into actionable components for a systematic evaluation.
-    - If the text lacks information, apply reasonable judgment to assign a score while clearly explaining the rationale.
-    PROMPT
 
-    def config
-      {
-        all_providers: { model_tier: 'lower_tier' },
-        log: {
-          **(@config.dig(:log) || {}),
-          trace: self.class.name,
-        },
-        **@config
-      }
-    end
+    - Maintain objectivity and avoid biases.
+    - Deconstruct each criterion into actionable components for systematic evaluation.
+    - Apply reasonable judgment in assigning a score, justifying your rationale clearly.
+
+    # Output Format
+
+    - Provide a detailed review including:
+      - A final score (0-100)
+      - Specific reasoning for the assigned score, detailing all evaluated criteria
+      - Include both positive aspects and suggested improvements
+
+    # Notes
+
+    - Consider edge cases where the text may partially align with criteria.
+    - If lacking information, reasonably judge and explain your scoring approach.
+    PROMPT
   end
 end

data/lib/active_genie/scoring/{recommended_reviews.rb → recommended_reviewers.rb}

@@ -3,7 +3,7 @@
 require_relative '../clients/unified_client'
 
 module ActiveGenie::Scoring
-  # The RecommendedReviews class intelligently suggests appropriate reviewer roles
+  # The RecommendedReviewers class intelligently suggests appropriate reviewer roles
   # for evaluating text content based on specific criteria. It uses AI to analyze
   # the content and criteria to identify the most suitable subject matter experts.
   #
@@ -11,14 +11,14 @@ module ActiveGenie::Scoring
   # three distinct reviewer roles with complementary expertise and perspectives.
   #
   # @example Getting recommended reviewers for technical content
-  #   RecommendedReviews.call("Technical documentation about API design", 
+  #   RecommendedReviewers.call("Technical documentation about API design", 
   #                           "Evaluate technical accuracy and clarity")
   #   # => { reviewer1: "API Architect", reviewer2: "Technical Writer", 
   #   #      reviewer3: "Developer Advocate", reasoning: "..." }
   #
-  class RecommendedReviews
-    def self.call(text, criteria, config: {})
-      new(text, criteria, config:).call
+  class RecommendedReviewers
+    def self.call(...)
+      new(...).call
     end
 
     # Initializes a new reviewer recommendation instance
@@ -29,7 +29,7 @@ module ActiveGenie::Scoring
     def initialize(text, criteria, config: {})
       @text = text
       @criteria = criteria
-      @config = config
+      @config = ActiveGenie::Configuration.to_h(config)
     end
 
     def call
@@ -53,7 +53,14 @@ module ActiveGenie::Scoring
         }
       }
 
-      ::ActiveGenie::Clients::UnifiedClient.function_calling(messages, function, config: @config)
+      result = client.function_calling(
+        messages,
+        function,
+        model_tier: 'lower_tier',
+        config: @config
+      )
+
+      result
     end
 
     private
@@ -72,5 +79,9 @@ module ActiveGenie::Scoring
     - Include reasoning for how each choice supports a thorough and insightful review.
     - Avoid redundant or overly similar titles/roles to maintain diversity.
     PROMPT
+
+    def client
+      ::ActiveGenie::Clients::UnifiedClient
+    end
   end
 end

data/lib/active_genie/scoring.rb

@@ -1,5 +1,5 @@
 require_relative 'scoring/basic'
-require_relative 'scoring/recommended_reviews'
+require_relative 'scoring/recommended_reviewers'
 
 module ActiveGenie
   # See the [Scoring README](lib/active_genie/scoring/README.md) for more information.
@@ -9,9 +9,12 @@ module ActiveGenie
     def basic(...)
       Basic.call(...)
     end
+    def call(...)
+      Basic.call(...)
+    end
 
-    def recommended_reviews(...)
-      RecommendedReviews.call(...)
+    def recommended_reviewers(...)
+      RecommendedReviewers.call(...)
     end
   end
 end

data/lib/active_genie.rb

@@ -5,7 +5,7 @@ module ActiveGenie
   autoload :DataExtractor, File.join(__dir__, 'active_genie/data_extractor')
   autoload :Battle, File.join(__dir__, 'active_genie/battle')
   autoload :Scoring, File.join(__dir__, 'active_genie/scoring')
-  autoload :Leaderboard, File.join(__dir__, 'active_genie/leaderboard')
+  autoload :Ranking, File.join(__dir__, 'active_genie/ranking')
 
   class << self
     def configure