active_genie 0.0.12 → 0.0.18

data/lib/active_genie/configuration.rb

@@ -1,6 +1,10 @@
 require_relative 'configuration/providers_config'
-require_relative 'configuration/openai_config'
+require_relative 'configuration/providers/openai_config'
+require_relative 'configuration/providers/google_config'
+require_relative 'configuration/providers/anthropic_config'
+require_relative 'configuration/providers/deepseek_config'
 require_relative 'configuration/log_config'
+require_relative 'configuration/runtime_config'
 
 module ActiveGenie
   module Configuration
@@ -9,7 +13,10 @@ module ActiveGenie
     def providers
       @providers ||= begin 
         p = ProvidersConfig.new
-        p.register(:openai, ActiveGenie::Configuration::OpenaiConfig)
+        p.register(ActiveGenie::Configuration::Providers::OpenaiConfig)
+        p.register(ActiveGenie::Configuration::Providers::GoogleConfig)
+        p.register(ActiveGenie::Configuration::Providers::AnthropicConfig)
+        p.register(ActiveGenie::Configuration::Providers::DeepseekConfig)
         p
       end
     end
@@ -18,10 +25,17 @@ module ActiveGenie
       @log ||= LogConfig.new
     end
 
+    def runtime
+      @runtime ||= RuntimeConfig.new
+    end
+
     def to_h(configs = {})
+      normalized_configs = configs.dig(:runtime) ? configs : { runtime: configs }
+
       {
-        providers: providers.to_h(configs.dig(:providers) || {}),
-        log: log.to_h(configs.dig(:log) || {})
+        providers: providers.to_h(normalized_configs.dig(:providers) || {}),
+        log: log.to_h(normalized_configs.dig(:log) || {}),
+        runtime: runtime.to_h(normalized_configs.dig(:runtime) || {})
       }
     end
   end

data/lib/active_genie/data_extractor/basic.rb

@@ -37,21 +37,34 @@ module ActiveGenie::DataExtractor
         {  role: 'system', content: PROMPT },
         {  role: 'user', content: @text }
       ]
+
+      properties = data_to_extract_with_explaination
+
       function = {
         name: 'data_extractor',
         description: 'Extract structured and typed data from user messages.',
         schema: {
           type: "object",
-          properties: data_to_extract_with_explaination
+          properties:,
+          required: properties.keys
         }
       }
 
-      ::ActiveGenie::Clients::UnifiedClient.function_calling(
+      result = ::ActiveGenie::Clients::UnifiedClient.function_calling(
         messages,
         function,
         model_tier: 'lower_tier',
         config: @config
       )
+
+      ActiveGenie::Logger.debug({
+        code: :data_extractor,
+        text: @text[0..30],
+        data_to_extract: @data_to_extract,
+        extracted_data: result
+      })
+
+      result
     end
 
     private

data/lib/active_genie/data_extractor.rb

@@ -10,6 +10,10 @@ module ActiveGenie
       Basic.call(...)
     end
 
+    def call(...)
+      Basic.call(...)
+    end
+
     def from_informal(...)
       FromInformal.call(...)
     end

data/lib/active_genie/logger.rb

@@ -5,56 +5,55 @@ module ActiveGenie
   module Logger
     module_function
 
-    def with_context(context)
+    def with_context(context, observer: nil)
       @context ||= {}
+      @observers ||= []
       begin
         @context = @context.merge(context)
+        @observers << observer if observer
         yield if block_given?
       ensure
         @context.delete_if { |key, _| context.key?(key) }
+        @observers.delete(observer)
       end
     end
 
     def info(log)
-      save(log, level: :info)
+      call(log, level: :info)
     end
 
     def error(log)
-      save(log, level: :error)
+      call(log, level: :error)
     end
 
     def warn(log)
-      save(log, level: :warn)
+      call(log, level: :warn)
     end
 
     def debug(log)
-      save(log, level: :debug)
+      call(log, level: :debug)
     end
 
     def trace(log)
-      save(log, level: :trace)
+      call(log, level: :trace)
     end
 
-    def save(data, level: :info)
-      log = @context.merge(data || {})
-      log[:timestamp] = Time.now
-      log[:level] = level.to_s.upcase
-      log[:process_id] = Process.pid
-      config_log_level = LOG_LEVELS[log.dig(:config, :log_level)] || LOG_LEVELS[:info]
+    def call(data, level: :info)
+      log = {
+        **(@context || {}),
+        **(data || {}),
+        timestamp: Time.now,
+        level: level.to_s.upcase,
+        process_id: Process.pid
+      }
 
-      FileUtils.mkdir_p('logs')
-      File.write('logs/active_genie.log', "#{JSON.generate(log)}\n", mode: 'a')
-      if config_log_level >= LOG_LEVELS[level]
-        $stdout.puts log
-      else
-        $stdout.print '.'
-      end
+      append_to_file(log)
+      output(log, level)
+      call_observers(log)
 
       log
     end
 
-    private
-
     # Log Levels
     # 
     # LOG_LEVELS defines different levels of logging within the application. 
@@ -68,5 +67,25 @@ module ActiveGenie
     LOG_LEVELS = { info: 0, error: 0, warn: 1, debug: 2, trace: 3 }.freeze
 
     attr_accessor :context
+    
+    def append_to_file(log)
+      FileUtils.mkdir_p('logs')
+      File.write('logs/active_genie.log', "#{JSON.generate(log)}\n", mode: 'a')
+    end
+
+    def output(log, level)
+      config_log_level = LOG_LEVELS[log.dig(:config, :log_level)] || LOG_LEVELS[:info]
+      if config_log_level >= LOG_LEVELS[level]
+        $stdout.puts log
+      else
+        $stdout.print '.'
+      end
+    end
+
+    def call_observers(log)
+      return if @observers.nil? || @observers.size.zero?
+
+      @observers.each { |observer| observer.call(log) }
+    end
   end
 end

data/lib/active_genie/ranking/elo_round.rb

@@ -13,32 +13,39 @@ module ActiveGenie::Ranking
       @criteria = criteria
       @config = config
       @tmp_defenders = []
+      @start_time = Time.now
+      @total_tokens = 0
+      @previous_elo = {}
+      @previous_highest_elo = @defender_tier.max_by(&:elo).elo
     end
 
     def call
       ActiveGenie::Logger.with_context(log_context) do
-        matches.each do |player_a, player_b|
+        save_previous_elo
+        matches.each do |player_1, player_2|
           # TODO: battle can take a while, can be parallelized
-          winner, loser = battle(player_a, player_b)
-
+          winner, loser = battle(player_1, player_2)
           next if winner.nil? || loser.nil?
 
-          new_winner_elo, new_loser_elo = calculate_new_elo(winner.elo, loser.elo)
-
-          winner.elo = new_winner_elo
-          loser.elo = new_loser_elo
+          winner.elo = calculate_new_elo(winner.elo, loser.elo, 1)
+          loser.elo = calculate_new_elo(loser.elo, winner.elo, 0)
         end
-
-        # TODO: add a round report. Duration, Elo changes, etc.
       end
+
+      ActiveGenie::Logger.info({ code: :elo_round_report, **report })
+
+      report
     end
 
     private
 
     BATTLE_PER_PLAYER = 3
-    LOSE_PENALTY = 15
     K = 32
 
+    def save_previous_elo
+      @previous_elo = @players.map { |player| [player.id, player.elo] }.to_h
+    end
+
     def matches
       @relegation_tier.reduce([]) do |matches, attack_player|
         BATTLE_PER_PLAYER.times do
@@ -49,53 +56,35 @@ module ActiveGenie::Ranking
     end
 
     def next_defense_player
-      @tmp_defenders = @defender_tier if @tmp_defenders.size.zero?
+      @tmp_defenders = @defender_tier.shuffle if @tmp_defenders.size.zero?
 
-      @tmp_defenders.shuffle.pop!
+      @tmp_defenders.pop
     end
 
-    def battle(player_a, player_b)
-      result = ActiveGenie::Battle.basic(
-        player_a,
-        player_b,
-        @criteria,
-        config: @config
-      )
-      
-      winner, loser = case result['winner']
-        when 'player_a' then [player_a, player_b]
-        when 'player_b' then [player_b, player_a]
-        when 'draw' then [nil, nil]
-      end
-
-      ActiveGenie::Logger.debug({
-        step: :elo_round_battle,
-        player_ids: [player_a.id, player_b.id],
-        winner_id: winner&.id,
-        loser_id: loser&.id,
-        reasoning: result['reasoning']
-      })
+    def battle(player_1, player_2)
+      ActiveGenie::Logger.with_context({ player_1_id: player_1.id, player_2_id: player_2.id }) do
+        result = ActiveGenie::Battle.basic(
+          player_1.content,
+          player_2.content,
+          @criteria,
+          config: @config
+        )
+
+        winner, loser = case result['winner']
+          when 'player_1' then [player_1, player_2]
+          when 'player_2' then [player_2, player_1]
+          when 'draw' then [nil, nil]
+        end
 
-      [winner, loser]
+        [winner, loser]
+      end
     end
 
     # INFO: Read more about the Elo rating system on https://en.wikipedia.org/wiki/Elo_rating_system
-    def calculate_new_elo(winner_elo, loser_elo)
-      expected_score_a = 1 / (1 + 10**((loser_elo - winner_elo) / 400))
-      expected_score_b = 1 - expected_score_a
-
-      new_winner_elo = [winner_elo + K * (1 - expected_score_a), max_defense_elo].min
-      new_loser_elo = [loser_elo + K * (1 - expected_score_b) - LOSE_PENALTY, min_relegation_elo].max
-
-      [new_winner_elo, new_loser_elo]
-    end
-
-    def max_defense_elo
-      @defender_tier.max_by(&:elo).elo
-    end
-
-    def min_relegation_elo
-      @relegation_tier.min_by(&:elo).elo
+    def calculate_new_elo(player_rating, opponent_rating, score)
+      expected_score = 1.0 / (1.0 + 10.0 ** ((opponent_rating - player_rating) / 400.0))
+      
+      player_rating + (K * (score - expected_score)).round
     end
 
     def log_context
@@ -109,5 +98,37 @@ module ActiveGenie::Ranking
       ranking_unique_key = [relegation_tier_ids, defender_tier_ids, @criteria, @config.to_json].join('-')
       Digest::MD5.hexdigest(ranking_unique_key)
     end
+
+    def report
+      {
+        elo_round_id:,
+        players_in_round: players_in_round.map(&:id),
+        battles_count: matches.size,
+        duration_seconds: Time.now - @start_time,
+        total_tokens: @total_tokens,
+        previous_highest_elo: @previous_highest_elo,
+        highest_elo:,
+        highest_elo_diff: highest_elo - @previous_highest_elo,
+        players_elo_diff:,
+      }
+    end
+
+    def players_in_round
+      @defender_tier + @relegation_tier
+    end
+
+    def highest_elo
+      players_in_round.max_by(&:elo).elo
+    end
+
+    def players_elo_diff
+      players_in_round.map do |player|
+        [player.id, player.elo - @previous_elo[player.id]]
+      end.sort_by { |_, diff| -diff }.to_h
+    end
+
+    def log_observer(log)
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    end
   end
 end

data/lib/active_genie/ranking/free_for_all.rb

@@ -10,24 +10,28 @@ module ActiveGenie::Ranking
       @players = players
       @criteria = criteria
       @config = config
+      @start_time = Time.now
+      @total_tokens = 0
     end
 
     def call
-      ActiveGenie::Logger.with_context(log_context) do
-        matches.each do |player_a, player_b|
-          winner, loser = battle(player_a, player_b)
+      ActiveGenie::Logger.with_context(log_context, observer: method(:log_observer)) do
+        matches.each do |player_1, player_2|
+          winner, loser = battle(player_1, player_2)
           
           if winner.nil? || loser.nil?
-            player_a.draw!
-            player_b.draw!
+            player_1.draw!
+            player_2.draw!
           else
             winner.win!
             loser.lose!
           end
         end
-
-        # TODO: add a freeForAll report. Duration, Elo changes, etc.
       end
+
+      ActiveGenie::Logger.info({ code: :free_for_all_report, **report })
+
+      report
     end
 
     private
@@ -38,23 +42,23 @@ module ActiveGenie::Ranking
       @players.eligible.combination(2).to_a
     end
 
-    def battle(player_a, player_b)
+    def battle(player_1, player_2)
       result = ActiveGenie::Battle.basic(
-        player_a,
-        player_b,
+        player_1.content,
+        player_2.content,
         @criteria,
         config: @config
       )
 
       winner, loser = case result['winner']
-        when 'player_a' then [player_a, player_b, result['reasoning']]
-        when 'player_b' then [player_b, player_a, result['reasoning']]
+        when 'player_1' then [player_1, player_2, result['reasoning']]
+        when 'player_2' then [player_2, player_1, result['reasoning']]
         when 'draw' then [nil, nil, result['reasoning']]
       end
 
       ActiveGenie::Logger.debug({
-        step: :free_for_all_battle,
-        player_ids: [player_a.id, player_b.id],
+        code: :free_for_all_battle,
+        player_ids: [player_1.id, player_2.id],
         winner_id: winner&.id,
         loser_id: loser&.id,
         reasoning: result['reasoning']
@@ -72,5 +76,18 @@ module ActiveGenie::Ranking
       ranking_unique_key = [eligible_ids, @criteria, @config.to_json].join('-')
       Digest::MD5.hexdigest(ranking_unique_key)
     end
+
+    def report
+      {
+        free_for_all_id:,
+        battles_count: matches.size,
+        duration_seconds: Time.now - @start_time,
+        total_tokens: @total_tokens,
+      }
+    end
+
+    def log_observer(log)
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    end
   end
 end

data/lib/active_genie/ranking/player.rb

@@ -21,39 +21,34 @@ module ActiveGenie::Ranking
     attr_accessor :rank
 
     def score=(value)
+      ActiveGenie::Logger.debug({ code: :new_score, player_id: id, score: value }) if value != @score
       @score = value
-      ActiveGenie::Logger.debug({ step: :new_score, player_id: id, score: value })
-    end
-
-    def elo
-      generate_elo_by_score if @elo.nil?
-
-      @elo
+      @elo = generate_elo_by_score
     end
 
     def elo=(value)
+      ActiveGenie::Logger.debug({ code: :new_elo, player_id: id, elo: value }) if value != @elo
       @elo = value
-      ActiveGenie::Logger.debug({ step: :new_elo, player_id: id, elo: value })
     end
 
     def eliminated=(value)
+      ActiveGenie::Logger.debug({ code: :new_eliminated, player_id: id, eliminated: value }) if value != @eliminated
       @eliminated = value
-      ActiveGenie::Logger.debug({ step: :new_eliminated, player_id: id, eliminated: value })
     end
 
     def draw!
       @ffa_draw_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'draw', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'draw', ffa_score: })
     end
 
     def win!
       @ffa_win_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'win', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'win', ffa_score: })
     end
 
     def lose!
       @ffa_lose_count += 1
-      ActiveGenie::Logger.debug({ step: :new_ffa_score, player_id: id, result: 'lose', ffa_score: })
+      ActiveGenie::Logger.debug({ code: :new_ffa_score, player_id: id, result: 'lose', ffa_score: })
     end
 
     def ffa_score
@@ -86,12 +81,12 @@ module ActiveGenie::Ranking
       method_name == :[] || super
     end
 
+    def generate_elo_by_score
+      BASE_ELO + ((@score || 0) - 50)
+    end
+
     private
     
     BASE_ELO = 1000
-
-    def generate_elo_by_score
-      @elo = BASE_ELO + ((@score || 0) - 50)
-    end
   end
 end

data/lib/active_genie/ranking/players_collection.rb

@@ -41,9 +41,9 @@ module ActiveGenie::Ranking
     end
 
     def sorted
-      @players.sort_by { |p| [-p.ffa_score, -(p.elo || 0), -(p.score || 0)] }
-      @players.each_with_index { |p, i| p.rank = i + 1 }
-      @players
+      sorted_players = @players.sort_by { |p| [-p.ffa_score, -(p.elo || 0), -(p.score || 0)] }
+      sorted_players.each_with_index { |p, i| p.rank = i + 1 }
+      sorted_players
     end
 
     def to_h
@@ -57,7 +57,7 @@ module ActiveGenie::Ranking
     private
 
     def build(param_players)
-      param_players.map { |player| Player.new(player) }
+      param_players.map { |p| Player.new(p) }
     end
 
     # Returns the number of players to battle in each round

data/lib/active_genie/ranking/ranking.rb

@@ -1,3 +1,4 @@
+require_relative '../concerns/loggable'
 require_relative './players_collection'
 require_relative './free_for_all'
 require_relative './elo_round'
@@ -28,39 +29,55 @@ require_relative './ranking_scoring'
 # @return [Hash] Final ranked player results
 module ActiveGenie::Ranking
   class Ranking
+    include ActiveGenie::Concerns::Loggable
+
     def self.call(...)
       new(...).call
     end
 
     def initialize(param_players, criteria, reviewers: [], config: {})
-      @param_players = param_players
       @criteria = criteria
       @reviewers = Array(reviewers).compact.uniq
       @config = ActiveGenie::Configuration.to_h(config)
-      @players = nil
+      @players = PlayersCollection.new(param_players)
+      @elo_rounds_played = 0
+      @elo_round_battle_count = 0
+      @free_for_all_battle_count = 0
+      @total_tokens = 0
+      @start_time = Time.now
     end
 
     def call
-      @players = PlayersCollection.new(@param_players)
-
-      ActiveGenie::Logger.with_context(log_context) do
-        set_initial_player_scores!
-        eliminate_obvious_bad_players!
-
-        while @players.elo_eligible?
-          run_elo_round!
-          eliminate_relegation_players!
-        end
-  
-        run_free_for_all!
+      initial_log
+
+      set_initial_player_scores!
+      eliminate_obvious_bad_players!
+
+      while @players.elo_eligible?
+        elo_report = run_elo_round!
+        eliminate_relegation_players!
+        rebalance_players!(elo_report)
       end
 
+      run_free_for_all!
+      final_logs
+
       @players.sorted
     end
 
     private
 
-    SCORE_VARIATION_THRESHOLD = 10
+    SCORE_VARIATION_THRESHOLD = 15
+    ELIMINATION_VARIATION = 'variation_too_high'
+    ELIMINATION_RELEGATION = 'relegation_tier'
+    
+    with_logging_context :log_context, ->(log) { 
+      @total_tokens += log[:total_tokens] if log[:code] == :llm_usage
+    }
+
+    def initial_log
+      @players.each { |p| ActiveGenie::Logger.debug({ code: :new_player, player: p.to_h }) }
+    end
 
     def set_initial_player_scores!
       RankingScoring.call(@players, @criteria, reviewers: @reviewers, config: @config)
@@ -68,20 +85,58 @@ module ActiveGenie::Ranking
 
     def eliminate_obvious_bad_players!
       while @players.coefficient_of_variation >= SCORE_VARIATION_THRESHOLD
-        @players.eligible.last.eliminated = 'variation_too_high'
+        @players.eligible.last.eliminated = ELIMINATION_VARIATION
       end
     end
 
     def run_elo_round!
-      EloRound.call(@players, @criteria, config: @config)
+      @elo_rounds_played += 1
+
+      elo_report = EloRound.call(@players, @criteria, config: @config)
+
+      @elo_round_battle_count += elo_report[:battles_count]
+
+      elo_report
     end
 
     def eliminate_relegation_players!
-      @players.calc_relegation_tier.each { |player| player.eliminated = 'relegation_tier' }
+      @players.calc_relegation_tier.each { |player| player.eliminated = ELIMINATION_RELEGATION }
+    end
+
+    def rebalance_players!(elo_report)
+      return if elo_report[:highest_elo_diff].negative?
+
+      @players.eligible.each do |player|
+        next if elo_report[:players_in_round].include?(player.id)
+
+        player.elo += elo_report[:highest_elo_diff]
+      end
     end
 
     def run_free_for_all!
-      FreeForAll.call(@players, @criteria, config: @config)
+      ffa_report = FreeForAll.call(@players, @criteria, config: @config)
+
+      @free_for_all_battle_count += ffa_report[:battles_count]
+    end
+
+    def report
+      {
+        ranking_id: ranking_id,
+        players_count: @players.size,
+        variation_too_high: @players.select { |player| player.eliminated == ELIMINATION_VARIATION }.size,
+        elo_rounds_played: @elo_rounds_played,
+        elo_round_battle_count: @elo_round_battle_count,
+        relegation_tier: @players.select { |player| player.eliminated == ELIMINATION_RELEGATION }.size,
+        ffa_round_battle_count: @free_for_all_battle_count,
+        top3: @players.eligible[0..2].map(&:id),
+        total_tokens: @total_tokens,
+        duration_seconds: Time.now - @start_time,
+      }
+    end
+    
+    def final_logs
+      ActiveGenie::Logger.debug({ code: :ranking_final, players: @players.sorted.map(&:to_h) })
+      ActiveGenie::Logger.info({ code: :ranking, **report })
     end
 
     def log_context