active_genie 0.0.8 → 0.0.12

data/lib/active_genie/ranking/ranking.rb

@@ -0,0 +1,98 @@
+require_relative './players_collection'
+require_relative './free_for_all'
+require_relative './elo_round'
+require_relative './ranking_scoring'
+
+# This class orchestrates player ranking through multiple evaluation stages
+# using Elo ranking and free-for-all match simulations.
+# 1. Sets initial scores
+# 2. Eliminates low performers
+# 3. Runs Elo ranking (for large groups)
+# 4. Conducts free-for-all matches
+#
+# @example Basic usage
+#   Ranking.call(players, criteria)
+#
+# @param param_players [Array<Hash|String>] Collection of player objects to evaluate
+#   Example: ["Circle", "Triangle", "Square"]
+#            or
+#   [
+#     { content: "Circle", score: 10 },
+#     { content: "Triangle", score: 7 },
+#     { content: "Square", score: 5 }
+#   ]
+# @param criteria [String] Evaluation criteria configuration
+#   Example: "What is more similar to the letter 'O'?"
+# @param config [Hash] Additional configuration config
+#   Example: { model: "gpt-4o", api_key: ENV['OPENAI_API_KEY'] }
+# @return [Hash] Final ranked player results
+module ActiveGenie::Ranking
+  class Ranking
+    def self.call(...)
+      new(...).call
+    end
+
+    def initialize(param_players, criteria, reviewers: [], config: {})
+      @param_players = param_players
+      @criteria = criteria
+      @reviewers = Array(reviewers).compact.uniq
+      @config = ActiveGenie::Configuration.to_h(config)
+      @players = nil
+    end
+
+    def call
+      @players = PlayersCollection.new(@param_players)
+
+      ActiveGenie::Logger.with_context(log_context) do
+        set_initial_player_scores!
+        eliminate_obvious_bad_players!
+
+        while @players.elo_eligible?
+          run_elo_round!
+          eliminate_relegation_players!
+        end
+  
+        run_free_for_all!
+      end
+
+      @players.sorted
+    end
+
+    private
+
+    SCORE_VARIATION_THRESHOLD = 10
+
+    def set_initial_player_scores!
+      RankingScoring.call(@players, @criteria, reviewers: @reviewers, config: @config)
+    end
+
+    def eliminate_obvious_bad_players!
+      while @players.coefficient_of_variation >= SCORE_VARIATION_THRESHOLD
+        @players.eligible.last.eliminated = 'variation_too_high'
+      end
+    end
+
+    def run_elo_round!
+      EloRound.call(@players, @criteria, config: @config)
+    end
+
+    def eliminate_relegation_players!
+      @players.calc_relegation_tier.each { |player| player.eliminated = 'relegation_tier' }
+    end
+
+    def run_free_for_all!
+      FreeForAll.call(@players, @criteria, config: @config)
+    end
+
+    def log_context
+      { config: @config[:log], ranking_id: }
+    end
+
+    def ranking_id
+      player_ids = @players.map(&:id).join(',')
+      ranking_unique_key = [player_ids, @criteria, @config.to_json].join('-')
+
+      Digest::MD5.hexdigest(ranking_unique_key)
+    end
+  end
+end

data/lib/active_genie/ranking/ranking_scoring.rb

@@ -0,0 +1,71 @@
+require_relative '../scoring/recommended_reviewers'
+
+module ActiveGenie::Ranking
+  class RankingScoring
+    def self.call(...)
+      new(...).call
+    end
+
+    def initialize(players, criteria, reviewers: [], config: {})
+      @players = players
+      @criteria = criteria
+      @config = ActiveGenie::Configuration.to_h(config)
+      @reviewers = Array(reviewers).compact.uniq
+    end
+
+    def call
+      ActiveGenie::Logger.with_context(log_context) do
+        @reviewers = generate_reviewers
+
+        players_without_score.each do |player|
+          # TODO: This can take a while, can be parallelized
+          player.score = generate_score(player)
+        end
+      end
+    end
+
+    private
+
+    def players_without_score
+      @players_without_score ||= @players.select { |player| player.score.nil? }
+    end
+
+    def generate_score(player)
+      score, reasoning = ActiveGenie::Scoring::Basic.call(
+        player.content,
+        @criteria,
+        @reviewers,
+        config: @config
+      ).values_at('final_score', 'final_reasoning')
+
+      ActiveGenie::Logger.debug({step: :new_score, player_id: player.id, score:, reasoning: })
+
+      score
+    end
+
+    def generate_reviewers
+      return @reviewers if @reviewers.size > 0
+
+      reviewer1, reviewer2, reviewer3 = ActiveGenie::Scoring::RecommendedReviewers.call(
+        [@players.sample.content, @players.sample.content].join("\n\n"),
+        @criteria,
+        config: @config
+      ).values_at('reviewer1', 'reviewer2', 'reviewer3')
+      
+      ActiveGenie::Logger.debug({step: :new_reviewers, reviewers: [reviewer1, reviewer2, reviewer3] })
+
+      [reviewer1, reviewer2, reviewer3]
+    end
+
+    def log_context
+      { ranking_scoring_id: }
+    end
+
+    def ranking_scoring_id
+      player_ids = players_without_score.map(&:id).join(',')
+      ranking_unique_key = [player_ids, @criteria, @config.to_json].join('-')
+
+      Digest::MD5.hexdigest(ranking_unique_key)
+    end
+  end
+end

data/lib/active_genie/ranking.rb

@@ -0,0 +1,12 @@
+require_relative 'ranking/ranking'
+
+module ActiveGenie
+  # See the [ranking README](lib/active_genie/ranking/README.md) for more information.
+  module Ranking
+    module_function
+
+    def call(...)
+      Ranking.call(...)
+    end
+  end
+end

data/lib/active_genie/scoring/README.md

@@ -49,26 +49,22 @@ result = ActiveGenie::Scoring::Basic.call(text, criteria)
 
 ## Interface
 
-### `Basic.call(text, criteria, reviewers = [], options: {})`
+### `Basic.call(text, criteria, reviewers = [], config: {})`
 Main interface for scoring text content.
 
 #### Parameters
 - `text` [String] - The text content to be evaluated
 - `criteria` [String] - The evaluation criteria or rubric to assess against
 - `reviewers` [Array<String>] - Optional list of specific reviewers
-- `options` [Hash] - Additional configuration options
-  - `:detailed_feedback` [Boolean] - Request more detailed feedback (WIP)
-  - `:reviewer_weights` [Hash] - Custom weights for different reviewers (WIP)
+- `config` [Hash] - Additional configuration config
 
-### `RecommendedReviews.call(text, criteria, options: {})`
+### `RecommendedReviewers.call(text, criteria, config: {})`
 Recommends appropriate reviewers based on content and criteria.
 
 #### Parameters
 - `text` [String] - The text content to analyze
 - `criteria` [String] - The evaluation criteria
-- `options` [Hash] - Additional configuration options
-  - `:prefer_technical` [Boolean] - Favor technical expertise (WIP)
-  - `:prefer_domain` [Boolean] - Favor domain expertise (WIP)
+- `config` [Hash] - Additional configuration config
 
 ### Usage Notes
 - Best suited for objective evaluation of text content

data/lib/active_genie/scoring/basic.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../clients/router'
+require_relative '../clients/unified_client'
 
 module ActiveGenie::Scoring
   # The Basic class provides a foundation for scoring text content against specified criteria
@@ -21,21 +21,19 @@ module ActiveGenie::Scoring
     # @param criteria [String] The evaluation criteria or rubric to assess against
     # @param reviewers [Array<String>] Optional list of specific reviewers. If empty,
     #   reviewers will be automatically recommended based on the content and criteria
-    # @param options [Hash] Additional configuration options that modify the scoring behavior
-    # @option options [Boolean] :detailed_feedback Request more detailed feedback in the reasoning
-    # @option options [Hash] :reviewer_weights Custom weights for different reviewers
+    # @param config [Hash] Additional configuration config that modify the scoring behavior
     # @return [Hash] The evaluation result containing the scores and reasoning
     #   @return [Number] :final_score The final score of the text based on the criteria and reviewers
     #   @return [String] :final_reasoning Detailed explanation of why the final score was reached
-    def self.call(text, criteria, reviewers = [], options: {})
-      new(text, criteria, reviewers, options:).call
+    def self.call(...)
+      new(...).call
     end
 
-    def initialize(text, criteria, reviewers = [], options: {})
+    def initialize(text, criteria, reviewers = [], config: {})
       @text = text
       @criteria = criteria
       @reviewers = Array(reviewers).compact.uniq
-      @options = options
+      @config = ActiveGenie::Configuration.to_h(config)
     end
 
     def call
@@ -78,7 +76,12 @@ module ActiveGenie::Scoring
         }
       }
 
-      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+      ::ActiveGenie::Clients::UnifiedClient.function_calling(
+        messages,
+        function,
+        model_tier: 'lower_tier',
+        config: @config
+      )
     end
 
     private
@@ -87,31 +90,62 @@ module ActiveGenie::Scoring
       @get_or_recommend_reviewers ||= if @reviewers.count > 0 
         @reviewers
       else
-        recommended_reviews = RecommendedReviews.call(@text, @criteria, options: @options)
+        result = RecommendedReviewers.call(@text, @criteria, config: @config)
 
-        [recommended_reviews[:reviewer1], recommended_reviews[:reviewer2], recommended_reviews[:reviewer3]]
+        [result['reviewer1'], result['reviewer2'], result['reviewer3']]
       end
     end
 
     PROMPT = <<~PROMPT
-    Evaluate and score the provided text based on predefined criteria, which may include rules, keywords, or patterns. Use a scoring range of 0 to 100, with 100 representing the highest possible score. Follow the instructions below to ensure an accurate and objective assessment.
+    Evaluate and score the provided text based on predefined criteria, using a scoring range of 0 to 100 with 100 representing the highest possible score.
+
+    Follow the instructions below to ensure a comprehensive and objective assessment. 
 
     # Evaluation Process
-    1. **Analysis**: Thoroughly compare the text against each criterion to ensure comprehensive evaluation.
-    2. **Document Deviations**: Clearly identify and document any areas where the content does not align with the specified criteria.
-    3. **Highlight Strengths**: Emphasize notable features or elements that enhance the overall quality or effectiveness of the content.
-    4. **Identify Weaknesses**: Specify areas where the content fails to meet the criteria or where improvements could be made.
 
-    # Output Requirements
-    Provide a detailed review, including:
-      - A final score (0-100)
-      - Specific reasoning for the assigned score, covering all evaluated criteria.
-      - Ensure the reasoning includes both positive aspects and suggested improvements.
+    1. **Analysis**: 
+      - Thoroughly compare the text against each criterion for a comprehensive evaluation.
+      
+    2. **Document Deviations**:
+      - Identify and document areas where the content does not align with the specified criteria.
+
+    3. **Highlight Strengths**:
+      - Note notable features or elements that enhance the quality or effectiveness of the content.
+
+    4. **Identify Weaknesses**:
+      - Specify areas where the content fails to meet the criteria or where improvements could be made.
+
+    # Scoring Fairness
+
+    - Ensure the assigned score reflects both the alignment with the criteria and the content's effectiveness.
+    - Consider if the fulfillment of other criteria compensates for areas lacking extreme details.
+
+    # Scoring Range
+
+    Segment scores into five parts before assigning a final score:
+    - **Terrible**: 0-20 - Content does not meet the criteria.
+    - **Bad**: 21-40 - Content is substandard but meets some criteria.
+    - **Average**: 41-60 - Content meets criteria with room for improvement.
+    - **Good**: 61-80 - Content exceeds criteria and is above average.
+    - **Great**: 81-100 - Content exceeds all expectations.
 
     # Guidelines
-    - Maintain objectivity, avoiding biases or preconceived notions.
-    - Deconstruct each criterion into actionable components for a systematic evaluation.
-    - If the text lacks information, apply reasonable judgment to assign a score while clearly explaining the rationale.
+
+    - Maintain objectivity and avoid biases.
+    - Deconstruct each criterion into actionable components for systematic evaluation.
+    - Apply reasonable judgment in assigning a score, justifying your rationale clearly.
+
+    # Output Format
+
+    - Provide a detailed review including:
+      - A final score (0-100)
+      - Specific reasoning for the assigned score, detailing all evaluated criteria
+      - Include both positive aspects and suggested improvements
+
+    # Notes
+
+    - Consider edge cases where the text may partially align with criteria.
+    - If lacking information, reasonably judge and explain your scoring approach.
     PROMPT
   end
 end

data/lib/active_genie/scoring/{recommended_reviews.rb → recommended_reviewers.rb}

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../clients/router.rb'
+require_relative '../clients/unified_client'
 
 module ActiveGenie::Scoring
-  # The RecommendedReviews class intelligently suggests appropriate reviewer roles
+  # The RecommendedReviewers class intelligently suggests appropriate reviewer roles
   # for evaluating text content based on specific criteria. It uses AI to analyze
   # the content and criteria to identify the most suitable subject matter experts.
   #
@@ -11,27 +11,25 @@ module ActiveGenie::Scoring
   # three distinct reviewer roles with complementary expertise and perspectives.
   #
   # @example Getting recommended reviewers for technical content
-  #   RecommendedReviews.call("Technical documentation about API design", 
+  #   RecommendedReviewers.call("Technical documentation about API design", 
   #                           "Evaluate technical accuracy and clarity")
   #   # => { reviewer1: "API Architect", reviewer2: "Technical Writer", 
   #   #      reviewer3: "Developer Advocate", reasoning: "..." }
   #
-  class RecommendedReviews
-    def self.call(text, criteria, options: {})
-      new(text, criteria, options:).call
+  class RecommendedReviewers
+    def self.call(...)
+      new(...).call
     end
 
     # Initializes a new reviewer recommendation instance
     #
     # @param text [String] The text content to analyze for reviewer recommendations
     # @param criteria [String] The evaluation criteria that will guide reviewer selection
-    # @param options [Hash] Additional configuration options that modify the recommendation process
-    # @option options [Boolean] :prefer_technical Whether to favor technical expertise
-    # @option options [Boolean] :prefer_domain Whether to favor domain expertise
-    def initialize(text, criteria, options: {})
+    # @param config [Hash] Additional configuration config that modify the recommendation process
+    def initialize(text, criteria, config: {})
       @text = text
       @criteria = criteria
-      @options = options
+      @config = ActiveGenie::Configuration.to_h(config)
     end
 
     def call
@@ -55,7 +53,14 @@ module ActiveGenie::Scoring
         }
       }
 
-      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+      result = client.function_calling(
+        messages,
+        function,
+        model_tier: 'lower_tier',
+        config: @config
+      )
+
+      result
     end
 
     private
@@ -74,5 +79,9 @@ module ActiveGenie::Scoring
     - Include reasoning for how each choice supports a thorough and insightful review.
     - Avoid redundant or overly similar titles/roles to maintain diversity.
     PROMPT
+
+    def client
+      ::ActiveGenie::Clients::UnifiedClient
+    end
   end
 end

data/lib/active_genie/scoring.rb

@@ -1,8 +1,8 @@
 require_relative 'scoring/basic'
-require_relative 'scoring/recommended_reviews'
+require_relative 'scoring/recommended_reviewers'
 
 module ActiveGenie
-  # Text evaluation system that provides detailed scoring and feedback using multiple expert reviewers
+  # See the [Scoring README](lib/active_genie/scoring/README.md) for more information.
   module Scoring
     module_function
 
@@ -10,8 +10,8 @@ module ActiveGenie
       Basic.call(...)
     end
 
-    def recommended_reviews(...)
-      RecommendedReviews.call(...)
+    def recommended_reviewers(...)
+      RecommendedReviewers.call(...)
     end
   end
 end

data/lib/active_genie.rb

@@ -1,15 +1,19 @@
-module ActiveGenie
-  autoload :Configuration, File.join(__dir__, 'active_genie/configuration')
+require_relative 'active_genie/logger'
+require_relative 'active_genie/configuration'
 
-  # Modules
+module ActiveGenie
   autoload :DataExtractor, File.join(__dir__, 'active_genie/data_extractor')
   autoload :Battle, File.join(__dir__, 'active_genie/battle')
   autoload :Scoring, File.join(__dir__, 'active_genie/scoring')
-  autoload :Leaderboard, File.join(__dir__, 'active_genie/leaderboard')
+  autoload :Ranking, File.join(__dir__, 'active_genie/ranking')
 
-  class << self    
+  class << self
     def configure
-      yield(config) if block_given?
+      yield(configuration) if block_given?
+    end
+
+    def configuration
+      @configuration ||= Configuration
     end
 
     def load_tasks
@@ -18,17 +22,5 @@ module ActiveGenie
       Rake::Task.define_task(:environment)
       Dir.glob(File.join(__dir__, 'tasks', '*.rake')).each { |r| load r }
     end
-
-    def config
-      @config ||= Configuration.new
-    end
-
-    def [](key)
-      config.values[key.to_s]
-    end
-
-    def config_by_model(model = nil)
-      config.values[model&.to_s&.downcase&.strip] || config.values.values.first || {}
-    end
   end
 end

data/lib/tasks/install.rake

@@ -3,10 +3,10 @@ require 'fileutils'
 namespace :active_genie do
   desc 'Install active_genie configuration file'
   task :install do
-    source = File.join(__dir__, 'templates', 'active_genie.yml')
-    target = File.join('config', 'active_genie.yml')
+    source = File.join(__dir__, 'templates', 'active_genie.rb')
+    target = File.join('config', 'initializers', 'active_genie.rb')
 
     FileUtils.cp(source, target)
-    puts "Successfully installed config/active_genie.yml to #{target}"
+    puts "Successfully installed active_genie!"
   end
 end

data/lib/tasks/templates/active_genie.rb

@@ -0,0 +1,17 @@
+
+ActiveGenie.configure do |config|
+  # example with openai and the current default for each config
+  # config.providers.openai.api_key = ENV['OPENAI_API_KEY']
+  # config.providers.openai.organization = ENV['OPENAI_ORGANIZATION']
+  # config.providers.openai.api_url = 'https://api.openai.com/v1'
+  # config.providers.openai.lower_tier_model = 'gpt-4o-mini'
+  # config.providers.openai.middle_tier_model = 'gpt-4o'
+  # config.providers.openai.upper_tier_model = 'o1-preview'
+  # config.providers.openai.client = ActiveGenie::Providers::Openai::Client.new(config)
+
+  # example how add a new provider
+  # config.providers.register(:internal_company_api, InternalCompanyApi::Configuration)
+
+  # Logs configuration
+  # config.log_level = :debug # default is :info
+end