active_genie 0.0.12 → 0.0.19

data/lib/active_genie/ranking/elo_round.rb

@@ -13,32 +13,39 @@ module ActiveGenie::Ranking
       @criteria = criteria
       @config = config
       @tmp_defenders = []
+      @start_time = Time.now
+      @total_tokens = 0
+      @previous_elo = {}
+      @previous_highest_elo = @defender_tier.max_by(&:elo).elo
     end
 
     def call
       ActiveGenie::Logger.with_context(log_context) do
-        matches.each do |player_a, player_b|
+        save_previous_elo
+        matches.each do |player_1, player_2|
           # TODO: battle can take a while, can be parallelized
-          winner, loser = battle(player_a, player_b)
-
+          winner, loser = battle(player_1, player_2)
           next if winner.nil? || loser.nil?
 
-          new_winner_elo, new_loser_elo = calculate_new_elo(winner.elo, loser.elo)
-
-          winner.elo = new_winner_elo
-          loser.elo = new_loser_elo
+          winner.elo = calculate_new_elo(winner.elo, loser.elo, 1)
+          loser.elo = calculate_new_elo(loser.elo, winner.elo, 0)
         end
-
-        # TODO: add a round report. Duration, Elo changes, etc.
       end
+
+      ActiveGenie::Logger.info({ code: :elo_round_report, **report })
+
+      report
     end
 
     private
 
     BATTLE_PER_PLAYER = 3
-    LOSE_PENALTY = 15
     K = 32
 
+    def save_previous_elo
+      @previous_elo = @players.map { |player| [player.id, player.elo] }.to_h
+    end
+
     def matches
       @relegation_tier.reduce([]) do |matches, attack_player|
         BATTLE_PER_PLAYER.times do
@@ -49,53 +56,35 @@ module ActiveGenie::Ranking
     end
 
     def next_defense_player
-      @tmp_defenders = @defender_tier if @tmp_defenders.size.zero?
+      @tmp_defenders = @defender_tier.shuffle if @tmp_defenders.size.zero?
 
-      @tmp_defenders.shuffle.pop!
+      @tmp_defenders.pop
     end
 
-    def battle(player_a, player_b)
-      result = ActiveGenie::Battle.basic(
-        player_a,
-        player_b,
-        @criteria,
-        config: @config
-      )
-      
-      winner, loser = case result['winner']
-        when 'player_a' then [player_a, player_b]
-        when 'player_b' then [player_b, player_a]
-        when 'draw' then [nil, nil]
-      end
-
-      ActiveGenie::Logger.debug({
-        step: :elo_round_battle,
-        player_ids: [player_a.id, player_b.id],
-        winner_id: winner&.id,
-        loser_id: loser&.id,
-        reasoning: result['reasoning']
-      })
+    def battle(player_1, player_2)
+      ActiveGenie::Logger.with_context({ player_1_id: player_1.id, player_2_id: player_2.id }) do
+        result = ActiveGenie::Battle.basic(
+          player_1.content,
+          player_2.content,
+          @criteria,
+          config: @config
+        )
+
+        winner, loser = case result['winner']
+          when 'player_1' then [player_1, player_2]
+          when 'player_2' then [player_2, player_1]
+          when 'draw' then [nil, nil]
+        end
 
-      [winner, loser]
+        [winner, loser]
+      end
     end
 
     # INFO: Read more about the Elo rating system on https://en.wikipedia.org/wiki/Elo_rating_system
-    def calculate_new_elo(winner_elo, loser_elo)
-      expected_score_a = 1 / (1 + 10**((loser_elo - winner_elo) / 400))
-      expected_score_b = 1 - expected_score_a
-
-      new_winner_elo = [winner_elo + K * (1 - expected_score_a), max_defense_elo].min
-      new_loser_elo = [loser_elo + K * (1 - expected_score_b) - LOSE_PENALTY, min_relegation_elo].max
-
-      [new_winner_elo, new_loser_elo]
-    end
-
-    def max_defense_elo
-      @defender_tier.max_by(&:elo).elo
-    end
-
-    def min_relegation_elo
-      @relegation_tier.min_by(&:elo).elo
+    def calculate_new_elo(player_rating, opponent_rating, score)
+      expected_score = 1.0 / (1.0 + 10.0 ** ((opponent_rating - player_rating) / 400.0))
+      
+      player_rating + (K * (score - expected_score)).round
     end
 
     def log_context
@@ -109,5 +98,37 @@ module ActiveGenie::Ranking
       ranking_unique_key = [relegation_tier_ids, defender_tier_ids, @criteria, @config.to_json].join('-')
       Digest::MD5.hexdigest(ranking_unique_key)
     end
+
+    def report
+      {
+        elo_round_id:,
+        players_in_round: players_in_round.map(&:id),
+        battles_count: matches.size,
+        duration_seconds: Time.now - @start_time,
+        total_tokens: @total_tokens,
+        previous_highest_elo: @previous_highest_elo,
+        highest_elo:,
+        highest_elo_diff: highest_elo - @previous_highest_elo,
+        players_elo_diff:,
+      }
+    end
+
+    def players_in_round
+      @defender_tier + @relegation_tier
+    end
+
+    def highest_elo
+      players_in_round.max_by(&:elo).elo
+    end
+
+    def players_elo_diff
+      players_in_round.map do |player|
+        [player.id, player.elo - @previous_elo[player.id]]
+      end.sort_by { |_, diff| -diff }.to_h
+    end
+
+    def log_observer(log)
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    end
   end
 end

data/lib/active_genie/ranking/free_for_all.rb

@@ -10,24 +10,28 @@ module ActiveGenie::Ranking
       @players = players
       @criteria = criteria
       @config = config
+      @start_time = Time.now
+      @total_tokens = 0
     end
 
     def call
-      ActiveGenie::Logger.with_context(log_context) do
-        matches.each do |player_a, player_b|
-          winner, loser = battle(player_a, player_b)
+      ActiveGenie::Logger.with_context(log_context, observer: method(:log_observer)) do
+        matches.each do |player_1, player_2|
+          winner, loser = battle(player_1, player_2)
           
           if winner.nil? || loser.nil?
-            player_a.draw!
-            player_b.draw!
+            player_1.draw!
+            player_2.draw!
           else
             winner.win!
             loser.lose!
           end
         end
-
-        # TODO: add a freeForAll report. Duration, Elo changes, etc.
       end
+
+      ActiveGenie::Logger.info({ code: :free_for_all_report, **report })
+
+      report
     end
 
     private
@@ -38,23 +42,23 @@ module ActiveGenie::Ranking
       @players.eligible.combination(2).to_a
     end
 
-    def battle(player_a, player_b)
+    def battle(player_1, player_2)
       result = ActiveGenie::Battle.basic(
-        player_a,
-        player_b,
+        player_1.content,
+        player_2.content,
         @criteria,
         config: @config
       )
 
       winner, loser = case result['winner']
-        when 'player_a' then [player_a, player_b, result['reasoning']]
-        when 'player_b' then [player_b, player_a, result['reasoning']]
+        when 'player_1' then [player_1, player_2, result['reasoning']]
+        when 'player_2' then [player_2, player_1, result['reasoning']]
         when 'draw' then [nil, nil, result['reasoning']]
       end
 
       ActiveGenie::Logger.debug({
-        step: :free_for_all_battle,
-        player_ids: [player_a.id, player_b.id],
+        code: :free_for_all_battle,
+        player_ids: [player_1.id, player_2.id],
         winner_id: winner&.id,
         loser_id: loser&.id,
         reasoning: result['reasoning']
@@ -72,5 +76,18 @@ module ActiveGenie::Ranking
       ranking_unique_key = [eligible_ids, @criteria, @config.to_json].join('-')
       Digest::MD5.hexdigest(ranking_unique_key)
     end
+
+    def report
+      {
+        free_for_all_id:,
+        battles_count: matches.size,
+        duration_seconds: Time.now - @start_time,
+        total_tokens: @total_tokens,
+      }
+    end
+
+    def log_observer(log)
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    end
   end
 end

data/lib/active_genie/ranking/player.rb

@@ -21,39 +21,34 @@ module ActiveGenie::Ranking
     attr_accessor :rank
 
     def score=(value)
+      ActiveGenie::Logger.debug({ code: :new_score, player_id: id, score: value }) if value != @score
       @score = value
-      ActiveGenie::Logger.debug({ step: :new_score, player_id: id, score: value })
-    end
-
-    def elo
-      generate_elo_by_score if @elo.nil?
-
-      @elo
+      @elo = generate_elo_by_score
     end
 
     def elo=(value)
+      ActiveGenie::Logger.debug({ code: :new_elo, player_id: id, elo: value }) if value != @elo
       @elo = value
-      ActiveGenie::Logger.debug({ step: :new_elo, player_id: id, elo: value })
     end
 
     def eliminated=(value)
+      ActiveGenie::Logger.debug({ code: :new_eliminated, player_id: id, eliminated: value }) if value != @eliminated
       @eliminated = value
-      ActiveGenie::Logger.debug({ step: :new_eliminated, player_id: id, eliminated: value })
     end
 
     def draw!
       @ffa_draw_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'draw', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'draw', ffa_score: })
     end
 
     def win!
       @ffa_win_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'win', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'win', ffa_score: })
     end
 
     def lose!
       @ffa_lose_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'lose', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'lose', ffa_score: })
     end
 
     def ffa_score
@@ -86,12 +81,12 @@ module ActiveGenie::Ranking
       method_name == :[] || super
     end
 
+    def generate_elo_by_score
+      BASE_ELO + ((@score || 0) - 50)
+    end
+
     private
     
     BASE_ELO = 1000
-
-    def generate_elo_by_score
-      @elo = BASE_ELO + ((@score || 0) - 50)
-    end
   end
 end

data/lib/active_genie/ranking/players_collection.rb

@@ -41,9 +41,9 @@ module ActiveGenie::Ranking
     end
 
     def sorted
-      @players.sort_by { |p| [-p.ffa_score, -(p.elo || 0), -(p.score || 0)] }
-      @players.each_with_index { |p, i| p.rank = i + 1 }
-      @players
+      sorted_players = @players.sort_by { |p| [-p.ffa_score, -(p.elo || 0), -(p.score || 0)] }
+      sorted_players.each_with_index { |p, i| p.rank = i + 1 }
+      sorted_players
     end
 
     def to_h
@@ -57,7 +57,7 @@ module ActiveGenie::Ranking
     private
 
     def build(param_players)
-      param_players.map { |player| Player.new(player) }
+      param_players.map { |p| Player.new(p) }
     end
 
     # Returns the number of players to battle in each round

data/lib/active_genie/ranking/ranking.rb

@@ -1,3 +1,4 @@
+require_relative '../concerns/loggable'
 require_relative './players_collection'
 require_relative './free_for_all'
 require_relative './elo_round'
@@ -28,39 +29,55 @@ require_relative './ranking_scoring'
 # @return [Hash] Final ranked player results
 module ActiveGenie::Ranking
   class Ranking
+    include ActiveGenie::Concerns::Loggable
+
     def self.call(...)
       new(...).call
     end
 
     def initialize(param_players, criteria, reviewers: [], config: {})
-      @param_players = param_players
       @criteria = criteria
       @reviewers = Array(reviewers).compact.uniq
       @config = ActiveGenie::Configuration.to_h(config)
-      @players = nil
+      @players = PlayersCollection.new(param_players)
+      @elo_rounds_played = 0
+      @elo_round_battle_count = 0
+      @free_for_all_battle_count = 0
+      @total_tokens = 0
+      @start_time = Time.now
     end
 
     def call
-      @players = PlayersCollection.new(@param_players)
-
-      ActiveGenie::Logger.with_context(log_context) do
-        set_initial_player_scores!
-        eliminate_obvious_bad_players!
-
-        while @players.elo_eligible?
-          run_elo_round!
-          eliminate_relegation_players!
-        end
-  
-        run_free_for_all!
+      initial_log
+
+      set_initial_player_scores!
+      eliminate_obvious_bad_players!
+
+      while @players.elo_eligible?
+        elo_report = run_elo_round!
+        eliminate_relegation_players!
+        rebalance_players!(elo_report)
       end
 
+      run_free_for_all!
+      final_logs
+
       @players.sorted
     end
 
     private
 
-    SCORE_VARIATION_THRESHOLD = 10
+    SCORE_VARIATION_THRESHOLD = 15
+    ELIMINATION_VARIATION = 'variation_too_high'
+    ELIMINATION_RELEGATION = 'relegation_tier'
+    
+    with_logging_context :log_context, ->(log) { 
+      @total_tokens += log[:total_tokens] || 0 if log[:code] == :llm_usage
+    }
+
+    def initial_log
+      @players.each { |p| ActiveGenie::Logger.debug({ code: :new_player, player: p.to_h }) }
+    end
 
     def set_initial_player_scores!
       RankingScoring.call(@players, @criteria, reviewers: @reviewers, config: @config)
@@ -68,20 +85,58 @@ module ActiveGenie::Ranking
 
     def eliminate_obvious_bad_players!
       while @players.coefficient_of_variation >= SCORE_VARIATION_THRESHOLD
-        @players.eligible.last.eliminated = 'variation_too_high'
+        @players.eligible.last.eliminated = ELIMINATION_VARIATION
       end
     end
 
     def run_elo_round!
-      EloRound.call(@players, @criteria, config: @config)
+      @elo_rounds_played += 1
+
+      elo_report = EloRound.call(@players, @criteria, config: @config)
+
+      @elo_round_battle_count += elo_report[:battles_count]
+
+      elo_report
     end
 
     def eliminate_relegation_players!
-      @players.calc_relegation_tier.each { |player| player.eliminated = 'relegation_tier' }
+      @players.calc_relegation_tier.each { |player| player.eliminated = ELIMINATION_RELEGATION }
+    end
+
+    def rebalance_players!(elo_report)
+      return if elo_report[:highest_elo_diff].negative?
+
+      @players.eligible.each do |player|
+        next if elo_report[:players_in_round].include?(player.id)
+
+        player.elo += elo_report[:highest_elo_diff]
+      end
     end
 
     def run_free_for_all!
-      FreeForAll.call(@players, @criteria, config: @config)
+      ffa_report = FreeForAll.call(@players, @criteria, config: @config)
+
+      @free_for_all_battle_count += ffa_report[:battles_count]
+    end
+
+    def report
+      {
+        ranking_id: ranking_id,
+        players_count: @players.size,
+        variation_too_high: @players.select { |player| player.eliminated == ELIMINATION_VARIATION }.size,
+        elo_rounds_played: @elo_rounds_played,
+        elo_round_battle_count: @elo_round_battle_count,
+        relegation_tier: @players.select { |player| player.eliminated == ELIMINATION_RELEGATION }.size,
+        ffa_round_battle_count: @free_for_all_battle_count,
+        top3: @players.eligible[0..2].map(&:id),
+        total_tokens: @total_tokens,
+        duration_seconds: Time.now - @start_time,
+      }
+    end
+    
+    def final_logs
+      ActiveGenie::Logger.debug({ code: :ranking_final, players: @players.sorted.map(&:to_h) })
+      ActiveGenie::Logger.info({ code: :ranking, **report })
     end
 
     def log_context

data/lib/active_genie/ranking/ranking_scoring.rb

@@ -31,14 +31,14 @@ module ActiveGenie::Ranking
     end
 
     def generate_score(player)
-      score, reasoning = ActiveGenie::Scoring::Basic.call(
+      score, reasoning = ActiveGenie::Scoring.call(
         player.content,
         @criteria,
         @reviewers,
         config: @config
       ).values_at('final_score', 'final_reasoning')
 
-      ActiveGenie::Logger.debug({step: :new_score, player_id: player.id, score:, reasoning: })
+      ActiveGenie::Logger.debug({ code: :new_score, player_id: player.id, score:, reasoning: })
 
       score
     end
@@ -52,7 +52,7 @@ module ActiveGenie::Ranking
         config: @config
       ).values_at('reviewer1', 'reviewer2', 'reviewer3')
       
-      ActiveGenie::Logger.debug({step: :new_reviewers, reviewers: [reviewer1, reviewer2, reviewer3] })
+      ActiveGenie::Logger.debug({ code: :new_reviewers, reviewers: [reviewer1, reviewer2, reviewer3] })
 
       [reviewer1, reviewer2, reviewer3]
     end

data/lib/active_genie/scoring/basic.rb

@@ -43,6 +43,42 @@ module ActiveGenie::Scoring
         {  role: 'user', content: "Text to score: #{@text}" }, 
       ]
 
+      properties = build_properties
+
+      function = {
+        name: 'scoring',
+        description: 'Score the text based on the given criteria.',
+        parameters: {
+          type: "object",
+          properties:,
+          required: properties.keys
+        }
+      }
+
+      result = ::ActiveGenie::Clients::UnifiedClient.function_calling(
+        messages,
+        function,
+        model_tier: 'lower_tier',
+        config: @config
+      )
+
+      result['final_score'] = 0 if result['final_score'].nil?
+
+      ActiveGenie::Logger.debug({
+        code: :scoring,
+        text: @text[0..30],
+        criteria: @criteria[0..30],
+        reviewers: get_or_recommend_reviewers,
+        score: result['final_score'],
+        reasoning: result['final_reasoning']
+      })
+
+      result
+    end
+
+    private
+    
+    def build_properties
       properties = {}
       get_or_recommend_reviewers.each do |reviewer|
         properties["#{reviewer}_reasoning"] = {
@@ -57,35 +93,18 @@ module ActiveGenie::Scoring
         }
       end
 
-      function = {
-        name: 'scoring',
-        description: 'Score the text based on the given criteria.',
-        schema: {
-          type: "object",
-          properties: {
-            **properties,
-            final_score: {
-              type: 'number',
-              description: 'The final score based on the previous reviewers',
-            },
-            final_reasoning: {
-              type: 'string',
-              description: 'The final reasoning based on the previous reviewers',
-            }
-          }
-        }
+      properties[:final_score] = {
+        type: 'number',
+        description: 'The final score based on the previous reviewers',
+      }
+      properties[:final_reasoning] = {
+        type: 'string',
+        description: 'The final reasoning based on the previous reviewers',
       }
 
-      ::ActiveGenie::Clients::UnifiedClient.function_calling(
-        messages,
-        function,
-        model_tier: 'lower_tier',
-        config: @config
-      )
+      properties
     end
 
-    private
-
     def get_or_recommend_reviewers
       @get_or_recommend_reviewers ||= if @reviewers.count > 0 
         @reviewers

data/lib/active_genie/scoring/recommended_reviewers.rb

@@ -42,7 +42,7 @@ module ActiveGenie::Scoring
       function = {
         name: 'identify_reviewers',
         description: 'Discover reviewers based on the text and given criteria.',
-        schema: {
+        parameters: {
           type: "object",
           properties: {
             reasoning: { type: 'string' },

data/lib/active_genie/scoring.rb

@@ -9,6 +9,9 @@ module ActiveGenie
     def basic(...)
       Basic.call(...)
     end
+    def call(...)
+      Basic.call(...)
+    end
 
     def recommended_reviewers(...)
       RecommendedReviewers.call(...)

data/lib/tasks/benchmark.rake

@@ -0,0 +1,27 @@
+
+namespace :active_genie do
+  desc "Run benchmarks, optionally for a specific module (e.g., rake active_genie:benchmark[data_extractor])"
+  task :benchmark, [:module_name] do |_, args|
+    Rake::TestTask.new(:run_benchmarks) do |t|
+      t.libs << "benchmark"
+      
+      if args[:module_name]
+        module_name = args[:module_name]
+        module_path = "benchmark/test_cases/#{module_name}/"
+        t.test_files = FileList["#{module_path}**/*_test.rb"]
+        puts "Running benchmarks for module: #{module_name}"
+      else
+        t.test_files = FileList["benchmark/test_cases/**/*_test.rb"]
+        puts "Running all benchmarks"
+      end
+      
+      t.warning = false
+    end
+    
+    begin
+      Rake::Task[:run_benchmarks].invoke
+    rescue => e
+      puts e
+    end
+  end
+end