active_genie 0.0.8 → 0.0.10

data/lib/active_genie/league/README.md

@@ -0,0 +1,43 @@
+# league
+
+The `ActiveGenie::League` module organizes players based on scores derived from textual evaluations and then ranks them using a multi-stage process. It leverages the scoring system from the `ActiveGenie::Scoring` module to assign initial scores, and then applies advanced ranking methods to produce a competitive league.
+
+## Overview
+
+The league system performs the following steps:
+
+1. **Initial Scoring**: Each player’s textual content is evaluated using `ActiveGenie::Scoring`. This produces a `score` based on multiple expert reviews.
+
+2. **Elimination of Poor Performers**: Players whose scores show a high coefficient of variation (indicating inconsistency) are progressively eliminated. This ensures that only competitive candidates continue in the ranking process.
+
+3. **ELO Ranking**: If there are more than 10 eligible players, an ELO-based ranking is applied. Battles between players are simulated via `ActiveGenie::Battle`, and scores are updated using an ELO algorithm tailored to the league.
+
+4. **Free for all Matches**: Finally, the remaining players engage in head-to-head matches where each unique pair competes. Match outcomes (wins, losses, draws) are recorded to finalize the rankings.
+
+## Components
+
+- **league**: Orchestrates the entire ranking process. Initializes player scores, eliminates outliers, and coordinates ranking rounds.
+
+- **EloRanking**: Applies an ELO-based system to rank players through simulated battles. It updates players’ ELO scores based on match outcomes and predefined rules (including penalties for losses).
+
+- **Free for all**: Conducts complete pairwise matches among eligible players to record win/loss/draw statistics and refine the final standings.
+
+## Usage
+
+Call the league using:
+
+```ruby
+result = ActiveGenie::League::league.call(players, criteria, config: {})
+```
+
+- `players`: A collection of player instances, each containing textual content to be scored.
+- `criteria`: A string defining the evaluation criteria used by the scoring system.
+- `config`: A hash of additional parameters for customization (e.g., model, api_key).
+
+The method processes the players through scoring, elimination, and ranking phases, then returns a hash containing the player statistics and rankings.
+
+## Possible improvements
+- Adjust initial criteria to ensure consistency
+- Adjust each player's content to ensure consistency
+- Support players with images or audio
+- Parallelize processing battles and scoring

data/lib/active_genie/{leaderboard → league}/elo_ranking.rb

@@ -1,32 +1,36 @@
 require_relative '../battle/basic'
-require_relative '../utils/math'
 
 module ActiveGenie::Leaderboard
   class EloRanking
-    def self.call(players, criteria, options: {})
-      new(players, criteria, options:).call
+    def self.call(players, criteria, config: {})
+      new(players, criteria, config:).call
     end
 
-    def initialize(players, criteria, options: {})
+    def initialize(players, criteria, config: {})
       @players = players
       @criteria = criteria
-      @options = options
+      @config = config
+      @start_time = Time.now
     end
 
     def call
       @players.each(&:generate_elo_by_score)
 
+      round_count = 0
       while @players.eligible_size > MINIMAL_PLAYERS_TO_BATTLE
         round = create_round(@players.tier_relegation, @players.tier_defense)
 
         round.each do |player_a, player_b|
           winner, loser = battle(player_a, player_b) # This can take a while, can be parallelized
           update_elo(winner, loser)
+          ActiveGenie::Logger.trace({ **log, step: :elo_battle, winner_id: winner.id, loser_id: loser.id, winner_elo: winner.elo, loser_elo: loser.elo })
         end
 
-        @players.tier_relegation.each { |player| player.eliminated = "relegation/#{@players.eligible_size}" }
+        eliminate_all_relegation_players
+        round_count += 1
       end
 
+      ActiveGenie::Logger.info({ **log, step: :elo_end, round_count:, eligible_size: @players.eligible_size })
       @players
     end
 
@@ -35,6 +39,7 @@ module ActiveGenie::Leaderboard
     MATCHS_PER_PLAYER = 3
     LOSE_PENALTY = 15
     MINIMAL_PLAYERS_TO_BATTLE = 10
+    K = 32
 
     # Create a round of matches
     # each round is exactly 1 regation player vs 3 defense players for all regation players
@@ -64,14 +69,14 @@ module ActiveGenie::Leaderboard
         player_a,
         player_b,
         @criteria,
-        options: @options
+        config:
       ).values_at('winner', 'loser')
     end
 
     def update_elo(winner, loser)
       return if winner.nil? || loser.nil?
 
-      new_winner_elo, new_loser_elo = ActiveGenie::Utils::Math.calculate_new_elo(winner.elo, loser.elo)
+      new_winner_elo, new_loser_elo = calculate_new_elo(winner.elo, loser.elo)
 
       winner.elo = [new_winner_elo, max_defense_elo].min
       loser.elo = [new_loser_elo - LOSE_PENALTY, min_relegation_elo].max
@@ -84,5 +89,33 @@ module ActiveGenie::Leaderboard
     def min_relegation_elo
       @players.tier_relegation.min_by(&:elo).elo
     end
+
+    # Read more about the formula on https://en.wikipedia.org/wiki/Elo_rating_system
+    def calculate_new_elo(winner_elo, loser_elo)
+      expected_score_a = 1 / (1 + 10**((loser_elo - winner_elo) / 400))
+      expected_score_b = 1 - expected_score_a
+
+      new_elo_winner = winner_elo + K * (1 - expected_score_a)
+      new_elo_loser = loser_elo + K * (1 - expected_score_b)
+
+      [new_elo_winner, new_elo_loser]
+    end
+
+    def eliminate_all_relegation_players
+      eliminations = @players.tier_relegation.size
+      @players.tier_relegation.each { |player| player.eliminated = 'tier_relegation' }
+      ActiveGenie::Logger.trace({ **log, step: :elo_round, eligible_size: @players.eligible_size, eliminations: })
+    end
+
+    def config
+      { **@config }
+    end
+
+    def log
+      {
+        **(@config.dig(:log) || {}),
+        duration: Time.now - @start_time
+      }
+    end
   end
 end

data/lib/active_genie/league/free_for_all.rb

@@ -0,0 +1,62 @@
+require_relative '../battle/basic'
+
+module ActiveGenie::Leaderboard
+  class FreeForAll
+    def self.call(players, criteria, config: {})
+      new(players, criteria, config:).call
+    end
+
+    def initialize(players, criteria, config: {})
+      @players = players
+      @criteria = criteria
+      @config = config
+      @start_time = Time.now
+    end
+
+    def call
+      matches.each do |player_a, player_b|
+        winner, loser = battle(player_a, player_b)
+
+        if winner.nil? || loser.nil?
+          player_a.free_for_all[:draw] += 1
+          player_b.free_for_all[:draw] += 1
+        else
+          winner.free_for_all[:win] += 1
+          loser.free_for_all[:lose] += 1
+        end
+
+        ActiveGenie::Logger.trace({**log, step: :free_for_all_battle, winner_id: winner&.id, player_a_id: player_a.id, player_a_free_for_all_score: player_a.free_for_all_score, player_b_id: player_b.id, player_b_free_for_all_score: player_b.free_for_all_score })
+      end
+
+      @players
+    end
+
+    private
+
+    # TODO: reduce the number of matches based on transitivity.
+    #       For example, if A is better than B, and B is better than C, then A should clearly be better than C
+    def matches
+      @players.eligible.combination(2).to_a
+    end
+
+    def battle(player_a, player_b)
+      result = ActiveGenie::Battle.basic(
+        player_a,
+        player_b,
+        @criteria,
+        config:
+      )
+
+
+      result.values_at('winner', 'loser')
+    end
+
+    def config
+      { **@config }
+    end
+
+    def log
+      { **(@config.dig(:log) || {}), duration: Time.now - @start_time }
+    end
+  end
+end

data/lib/active_genie/league/league.rb

@@ -0,0 +1,120 @@
+require 'securerandom'
+
+require_relative './players_collection'
+require_relative './free_for_all'
+require_relative './elo_ranking'
+require_relative '../scoring/recommended_reviews'
+
+# This class orchestrates player ranking through multiple evaluation stages
+# using Elo ranking and free-for-all match simulations.
+# 1. Sets initial scores
+# 2. Eliminates low performers
+# 3. Runs Elo ranking (for large groups)
+# 4. Conducts free-for-all matches
+#
+# @example Basic usage
+#   League.call(players, criteria)
+#
+# @param param_players [Array] Collection of player objects to evaluate
+#   Example: ["Circle", "Triangle", "Square"]
+#            or
+#   [
+#     { content: "Circle", score: 10 },
+#     { content: "Triangle", score: 7 },
+#     { content: "Square", score: 5 }
+#   ]
+# @param criteria [String] Evaluation criteria configuration
+#   Example: "What is more similar to the letter 'O'?"
+# @param config [Hash] Additional configuration config
+#   Example: { model: "gpt-4o", api_key: ENV['OPENAI_API_KEY'] }
+# @return [Hash] Final ranked player results
+module ActiveGenie::League
+  class League
+    def self.call(param_players, criteria, config: {})
+      new(param_players, criteria, config:).call
+    end
+
+    def initialize(param_players, criteria, config: {})
+      @param_players = param_players
+      @criteria = criteria
+      @config = config
+      @league_id = SecureRandom.uuid
+      @start_time = Time.now
+    end
+
+    def call
+      set_initial_score_players
+      eliminate_obvious_bad_players
+      run_elo_ranking if players.eligible_size > 10
+      run_free_for_all
+
+      ActiveGenie::Logger.info({ **log, step: :league_end, top5: players.first(5).map(&:id) })
+      players.to_h
+    end
+
+    private
+
+    SCORE_VARIATION_THRESHOLD = 10
+
+    def set_initial_score_players
+      players_without_score = players.reject { |player| player.score }
+      players_without_score.each do |player|
+        player.score = generate_score(player.content) # This can take a while, can be parallelized
+        ActiveGenie::Logger.trace({ **log, step: :player_score, player_id: player.id, score: player.score })
+      end
+
+      ActiveGenie::Logger.info({ **log, step: :initial_score, evaluated_players: players_without_score.size })
+    end
+
+    def generate_score(content)
+      ActiveGenie::Scoring::Basic.call(content, @criteria, reviewers, config:)['final_score']
+    end
+
+    def eliminate_obvious_bad_players
+      eliminated_count = 0
+      while players.coefficient_of_variation >= SCORE_VARIATION_THRESHOLD
+        players.eligible.last.eliminated = 'variation_too_high'
+        eliminated_count += 1
+      end
+
+      ActiveGenie::Logger.info({ **log, step: :eliminate_obvious_bad_players, eliminated_count: })
+    end
+
+    def run_elo_ranking
+      EloRanking.call(players, @criteria, config:)
+    end
+
+    def run_free_for_all
+      FreeForAll.call(players, @criteria, config:)
+    end
+
+    def reviewers
+      [recommended_reviews['reviewer1'], recommended_reviews['reviewer2'], recommended_reviews['reviewer3']]
+    end
+
+    def recommended_reviews
+      @recommended_reviews ||= ActiveGenie::Scoring::RecommendedReviews.call(
+        [players.sample.content, players.sample.content].join("\n\n"),
+        @criteria,
+        config:
+      )
+    end
+
+    def players
+      @players ||= PlayersCollection.new(@param_players)
+    end
+
+    def config
+      { log:, **@config }
+    end
+
+    def log
+      {
+        **(@config.dig(:log) || {}),
+        league_id: @league_id,
+        league_start_time: @start_time,
+        duration: Time.now - @start_time
+      }
+    end
+  end
+end

data/lib/active_genie/{leaderboard → league}/player.rb

@@ -9,20 +9,24 @@ module ActiveGenie::Leaderboard
       @content = params.dig(:content) || params
       @score = params.dig(:score) || nil
       @elo = params.dig(:elo) || nil
-      @league = {
-        win: params.dig(:league, :win) || 0,
-        lose: params.dig(:league, :lose) || 0,
-        draw: params.dig(:league, :draw) || 0
+      @free_for_all = {
+        win: params.dig(:free_for_all, :win) || 0,
+        lose: params.dig(:free_for_all, :lose) || 0,
+        draw: params.dig(:free_for_all, :draw) || 0
       }
       @eliminated = params.dig(:eliminated) || nil
     end
 
-    attr_reader :id, :content, :score, :elo, :league, :eliminated
+    attr_reader :id, :content, :score, :elo, :free_for_all, :eliminated
 
     def generate_elo_by_score
-      return if !@elo.nil? || @score.nil?
+      return if !@elo.nil?
 
-      @elo = BASE_ELO + (@score - 50)
+      if @score.nil?
+        @elo = BASE_ELO
+      else
+        @elo = BASE_ELO + (@score - 50)
+      end
     end
 
     def score=(value)
@@ -37,12 +41,15 @@ module ActiveGenie::Leaderboard
       @eliminated = value
     end
 
-    def league_score
-      @league[:win] * 3 + @league[:draw]
+    def free_for_all_score
+      @free_for_all[:win] * 3 + @free_for_all[:draw]
     end
 
     def to_h
-      { id:, content:, score:, elo:, eliminated:, league: }
+      {
+        id:, content:, score:, elo:,
+        eliminated:, free_for_all:, free_for_all_score:
+      }
     end
 
     private

data/lib/active_genie/league.rb

@@ -0,0 +1,12 @@
+require_relative 'league/league'
+
+module ActiveGenie
+  # See the [league README](lib/active_genie/league/README.md) for more information.
+  module League
+    module_function
+
+    def call(...)
+      league.call(...)
+    end
+  end
+end

data/lib/active_genie/logger.rb

@@ -0,0 +1,45 @@
+require 'json'
+require 'fileutils'
+
+module ActiveGenie
+  module Logger
+    module_function
+
+    def info(log)
+      save(log, level: :info)
+    end
+
+    def error(log)
+      save(log, level: :error)
+    end
+
+    def warn(log)
+      save(log, level: :warn)
+    end
+
+    def debug(log)
+      save(log, level: :debug)
+    end
+
+    def trace(log)
+      save(log, level: :trace)
+    end
+
+    LOG_LEVELS = { info: 0, error: 1, warn: 2, debug: 3, trace: 4 }.freeze
+
+    def save(log, level: :info)
+      return if LOG_LEVELS[log.dig(:log, :log_level)] || -1 < LOG_LEVELS[level]
+
+      log[:trace] = log.dig(:trace)&.to_s&.gsub('ActiveGenie::', '')
+      log[:timestamp] = Time.now
+      log[:level] = level.to_s.upcase
+      log[:process_id] = Process.pid
+
+      FileUtils.mkdir_p('logs')
+      File.write('logs/active_genie.log', "#{JSON.generate(log)}\n", mode: 'a')
+      puts log
+
+      log
+    end
+  end
+end

data/lib/active_genie/scoring/README.md

@@ -49,26 +49,22 @@ result = ActiveGenie::Scoring::Basic.call(text, criteria)
 
 ## Interface
 
-### `Basic.call(text, criteria, reviewers = [], options: {})`
+### `Basic.call(text, criteria, reviewers = [], config: {})`
 Main interface for scoring text content.
 
 #### Parameters
 - `text` [String] - The text content to be evaluated
 - `criteria` [String] - The evaluation criteria or rubric to assess against
 - `reviewers` [Array<String>] - Optional list of specific reviewers
-- `options` [Hash] - Additional configuration options
-  - `:detailed_feedback` [Boolean] - Request more detailed feedback (WIP)
-  - `:reviewer_weights` [Hash] - Custom weights for different reviewers (WIP)
+- `config` [Hash] - Additional configuration config
 
-### `RecommendedReviews.call(text, criteria, options: {})`
+### `RecommendedReviews.call(text, criteria, config: {})`
 Recommends appropriate reviewers based on content and criteria.
 
 #### Parameters
 - `text` [String] - The text content to analyze
 - `criteria` [String] - The evaluation criteria
-- `options` [Hash] - Additional configuration options
-  - `:prefer_technical` [Boolean] - Favor technical expertise (WIP)
-  - `:prefer_domain` [Boolean] - Favor domain expertise (WIP)
+- `config` [Hash] - Additional configuration config
 
 ### Usage Notes
 - Best suited for objective evaluation of text content

data/lib/active_genie/scoring/basic.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../clients/router'
+require_relative '../clients/unified_client'
 
 module ActiveGenie::Scoring
   # The Basic class provides a foundation for scoring text content against specified criteria
@@ -21,21 +21,19 @@ module ActiveGenie::Scoring
     # @param criteria [String] The evaluation criteria or rubric to assess against
     # @param reviewers [Array<String>] Optional list of specific reviewers. If empty,
     #   reviewers will be automatically recommended based on the content and criteria
-    # @param options [Hash] Additional configuration options that modify the scoring behavior
-    # @option options [Boolean] :detailed_feedback Request more detailed feedback in the reasoning
-    # @option options [Hash] :reviewer_weights Custom weights for different reviewers
+    # @param config [Hash] Additional configuration config that modify the scoring behavior
     # @return [Hash] The evaluation result containing the scores and reasoning
     #   @return [Number] :final_score The final score of the text based on the criteria and reviewers
     #   @return [String] :final_reasoning Detailed explanation of why the final score was reached
-    def self.call(text, criteria, reviewers = [], options: {})
-      new(text, criteria, reviewers, options:).call
+    def self.call(text, criteria, reviewers = [], config: {})
+      new(text, criteria, reviewers, config:).call
     end
 
-    def initialize(text, criteria, reviewers = [], options: {})
+    def initialize(text, criteria, reviewers = [], config: {})
       @text = text
       @criteria = criteria
       @reviewers = Array(reviewers).compact.uniq
-      @options = options
+      @config = config
     end
 
     def call
@@ -78,7 +76,7 @@ module ActiveGenie::Scoring
         }
       }
 
-      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+      ::ActiveGenie::Clients::UnifiedClient.function_calling(messages, function, config:)
     end
 
     private
@@ -87,7 +85,7 @@ module ActiveGenie::Scoring
       @get_or_recommend_reviewers ||= if @reviewers.count > 0 
         @reviewers
       else
-        recommended_reviews = RecommendedReviews.call(@text, @criteria, options: @options)
+        recommended_reviews = RecommendedReviews.call(@text, @criteria, config:)
 
         [recommended_reviews[:reviewer1], recommended_reviews[:reviewer2], recommended_reviews[:reviewer3]]
       end
@@ -113,5 +111,16 @@ module ActiveGenie::Scoring
     - Deconstruct each criterion into actionable components for a systematic evaluation.
     - If the text lacks information, apply reasonable judgment to assign a score while clearly explaining the rationale.
     PROMPT
+
+    def config
+      {
+        all_providers: { model_tier: 'lower_tier' },
+        log: {
+          **(@config.dig(:log) || {}),
+          trace: self.class.name,
+        },
+        **@config
+      }
+    end
   end
 end

data/lib/active_genie/scoring/recommended_reviews.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../clients/router.rb'
+require_relative '../clients/unified_client'
 
 module ActiveGenie::Scoring
   # The RecommendedReviews class intelligently suggests appropriate reviewer roles
@@ -17,21 +17,19 @@ module ActiveGenie::Scoring
   #   #      reviewer3: "Developer Advocate", reasoning: "..." }
   #
   class RecommendedReviews
-    def self.call(text, criteria, options: {})
-      new(text, criteria, options:).call
+    def self.call(text, criteria, config: {})
+      new(text, criteria, config:).call
     end
 
     # Initializes a new reviewer recommendation instance
     #
     # @param text [String] The text content to analyze for reviewer recommendations
     # @param criteria [String] The evaluation criteria that will guide reviewer selection
-    # @param options [Hash] Additional configuration options that modify the recommendation process
-    # @option options [Boolean] :prefer_technical Whether to favor technical expertise
-    # @option options [Boolean] :prefer_domain Whether to favor domain expertise
-    def initialize(text, criteria, options: {})
+    # @param config [Hash] Additional configuration config that modify the recommendation process
+    def initialize(text, criteria, config: {})
       @text = text
       @criteria = criteria
-      @options = options
+      @config = config
     end
 
     def call
@@ -55,7 +53,7 @@ module ActiveGenie::Scoring
         }
       }
 
-      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+      ::ActiveGenie::Clients::UnifiedClient.function_calling(messages, function, config: @config)
     end
 
     private

data/lib/active_genie/scoring.rb

@@ -2,7 +2,7 @@ require_relative 'scoring/basic'
 require_relative 'scoring/recommended_reviews'
 
 module ActiveGenie
-  # Text evaluation system that provides detailed scoring and feedback using multiple expert reviewers
+  # See the [Scoring README](lib/active_genie/scoring/README.md) for more information.
   module Scoring
     module_function
 

data/lib/active_genie.rb

@@ -1,15 +1,19 @@
-module ActiveGenie
-  autoload :Configuration, File.join(__dir__, 'active_genie/configuration')
+require_relative 'active_genie/logger'
+require_relative 'active_genie/configuration'
 
-  # Modules
+module ActiveGenie
   autoload :DataExtractor, File.join(__dir__, 'active_genie/data_extractor')
   autoload :Battle, File.join(__dir__, 'active_genie/battle')
   autoload :Scoring, File.join(__dir__, 'active_genie/scoring')
   autoload :Leaderboard, File.join(__dir__, 'active_genie/leaderboard')
 
-  class << self    
+  class << self
     def configure
-      yield(config) if block_given?
+      yield(configuration) if block_given?
+    end
+
+    def configuration
+      @configuration ||= Configuration
     end
 
     def load_tasks
@@ -18,17 +22,5 @@ module ActiveGenie
       Rake::Task.define_task(:environment)
       Dir.glob(File.join(__dir__, 'tasks', '*.rake')).each { |r| load r }
     end
-
-    def config
-      @config ||= Configuration.new
-    end
-
-    def [](key)
-      config.values[key.to_s]
-    end
-
-    def config_by_model(model = nil)
-      config.values[model&.to_s&.downcase&.strip] || config.values.values.first || {}
-    end
   end
 end

data/lib/tasks/install.rake

@@ -3,10 +3,10 @@ require 'fileutils'
 namespace :active_genie do
   desc 'Install active_genie configuration file'
   task :install do
-    source = File.join(__dir__, 'templates', 'active_genie.yml')
-    target = File.join('config', 'active_genie.yml')
+    source = File.join(__dir__, 'templates', 'active_genie.rb')
+    target = File.join('config', 'initializers', 'active_genie.rb')
 
     FileUtils.cp(source, target)
-    puts "Successfully installed config/active_genie.yml to #{target}"
+    puts "Successfully installed active_genie!"
   end
 end

data/lib/tasks/templates/active_genie.rb

@@ -0,0 +1,17 @@
+
+ActiveGenie.configure do |config|
+  # example with openai and the current default for each config
+  # config.providers.openai.api_key = ENV['OPENAI_API_KEY']
+  # config.providers.openai.organization = ENV['OPENAI_ORGANIZATION']
+  # config.providers.openai.api_url = 'https://api.openai.com/v1'
+  # config.providers.openai.lower_tier_model = 'gpt-4o-mini'
+  # config.providers.openai.middle_tier_model = 'gpt-4o'
+  # config.providers.openai.upper_tier_model = 'o1-preview'
+  # config.providers.openai.client = ActiveGenie::Providers::Openai::Client.new(config)
+
+  # example how add a new provider
+  # config.providers.register(:internal_company_api, InternalCompanyApi::Configuration)
+
+  # Logs configuration
+  # config.log_level = :debug # default is :info
+end