active_genie 0.0.24 → 0.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88cd13341dc47e212e3d0cafdf530dfe6f6360c482d15e999330e231a03fbdd3
-  data.tar.gz: 44465211f6da6fa22815073156a4c2f99630272db3976a893c4b3653b83bd4fd
+  metadata.gz: 5197c43d9b0e25e91a23602e0c694eda8fe9cac3db52c5f98a93410925eb581f
+  data.tar.gz: d44255a41f1de2c2738486e0852cc3c5d3362d979bc6ac00e40e0bd08caed786
 SHA512:
-  metadata.gz: 3a8f1bee6522930c86120feb3ec5a4fd0c1e38f0afae287bb5892bfbf949449500b4e186f74048d2a06563f61254f66cf1eee26bd73d0b7876d02be923dc7213
-  data.tar.gz: f2eafe8adb628044264436b197c54e09c4a8b27af210bf62c57071bfb83e5631144bc32ac61647f1e714249b181ede1acfa4f9315516da44099ce964ace1e72a
+  metadata.gz: 26011be1a41293f5fb5968f345f19a32bf52f74fdb27080f07d30fac3036a6b5ac31b6bc4933f5f9a780bd6f16c529157fc4ea210cdf884f7b093e99851c641b
+  data.tar.gz: 1a31a9253ba70e727d54af1f1d531f90ecaee21405ed7b2989aafb972c52329e71293fc1af9b975627767da25a40a06fc928af598d8603a84c6ce1828a225e5d

data/README.md

@@ -28,7 +28,7 @@ rails g active_genie:install
 4. Configure your credentials in `config/initializers/active_genie.rb`:
 ```ruby
 ActiveGenie.configure do |config|
-  config.openai.api_key = ENV['OPENAI_API_KEY']
+  config.providers.openai.api_key = ENV['OPENAI_API_KEY']
 end
 ```
 
@@ -41,11 +41,11 @@ Extract structured data from text using AI-powered analysis, handling informal l
 ```ruby
 text = "Nike Air Max 90 - Size 42 - $199.99"
 schema = {
-  brand: { 
+  brand: {
     type: 'string',
     enum: ["Nike", "Adidas", "Puma"]
   },
-  price: { 
+  price: {
     type: 'number',
     minimum: 0
   },
@@ -59,10 +59,10 @@ schema = {
 result = ActiveGenie::DataExtractor.call(
   text,
   schema,
-  config: { provider: :openai, model: 'gpt-4o-mini' } # optional
+  config: { provider: :openai, model: 'gpt-4.1-mini' } # optional
 )
-# => { 
-#      brand: "Nike", 
+# => {
+#      brand: "Nike",
 #      brand_explanation: "Brand name found at start of text",
 #      price: 199.99,
 #      price_explanation: "Price found in USD format at end",
@@ -88,7 +88,7 @@ Text evaluation system that provides detailed scoring and feedback using multipl
 text = "The code implements a binary search algorithm with O(log n) complexity"
 criteria = "Evaluate technical accuracy and clarity"
 
-result = ActiveGenie::Scoring.basic(
+result = ActiveGenie::Scoring.call(
   text,
   criteria,
   config: { provider: :anthropic, model: 'claude-3-5-haiku-20241022' } # optional
@@ -130,8 +130,8 @@ result = ActiveGenie::Battle.call(
 )
 # => {
 #      winner_player: "Implementation uses dependency injection for better testability",
-#      reasoning: "Player 1 implementation demonstrates better maintainability through dependency injection, 
-#                 which allows for easier testing and component replacement. While Player 2 has good test coverage, 
+#      reasoning: "Player 1 implementation demonstrates better maintainability through dependency injection,
+#                 which allows for easier testing and component replacement. While Player 2 has good test coverage,
 #                 the tight coupling makes the code harder to maintain and modify.",
 #      what_could_be_changed_to_avoid_draw: "Focus on specific architectural patterns and design principles"
 #    }
@@ -221,56 +221,41 @@ See the [Benchmark README](benchmark/README.md) for detailed results, methodolog
 ActiveGenie supports adding custom providers to integrate with different LLM services. To create a new provider:
 
 1. Create a configuration class for your provider in `lib/active_genie/configuration/providers/`:
+2. Register your client
 
 ```ruby
-# Example: lib/active_genie/configuration/providers/internal_company_api_config.rb
-module ActiveGenie
-  module Configuration::Providers
-    class InternalCompanyApiConfig < BaseConfig
-      NAME = :internal_company_api
-      
-      # API key accessor with environment variable fallback
-      def api_key
-        @api_key || ENV['INTERNAL_COMPANY_API_KEY']
-      end
-      
-      # Base API URL
-      def api_url
-        @api_url || 'https://api.internal-company.com/v1'
-      end
-      
-      # Client instantiation
-      def client
-        @client ||= ::ActiveGenie::Clients::InternalCompanyApiClient.new(self)
-      end
-      
-      # Model tier definitions
-      def lower_tier_model
-        @lower_tier_model || 'internal-basic'
-      end
-      
-      def middle_tier_model
-        @middle_tier_model || 'internal-standard'
-      end
-      
-      def upper_tier_model
-        @upper_tier_model || 'internal-premium'
-      end
-    end
+class InternalCompanyApi
+  # @param messages [Array<Hash>] A list of messages representing the conversation history.
+  #   Each hash should have :role ('user', 'assistant', or 'system') and :content (String).
+  # @param function [Hash] A JSON schema definition describing the desired output format.
+  # @return [Hash, nil] The parsed JSON object matching the schema, or nil if parsing fails or content is empty.
+  def function_calling(messages, function)
+    # ...
   end
 end
+
+ActiveGenie.configure do |config|
+  config.llm.client = InternalCompanyApi
+end
+# or
+ActiveGenie::Battle.call('player_1', 'player_2', 'criteria', { client: InternalCompanyApi })
 ```
 
-2. Register your provider in your configuration:
+## Observability
+Fundamental to managing any production system, observability is crucial for GenAI features. At a minimum, track these key metrics:
+
+- Usage Rate (e.g., uses_per_minute): Detect anomalies like sudden traffic spikes (potential DDoS) or drops (feature outage or declining usage).
+- Failure/Retry Rate (e.g., retry_count, fail_count): Monitor the frequency of errors. Exceeding a defined threshold should trigger downtime or degradation alerts.
+- Token Consumption (e.g., tokens_used): Track usage to monitor costs. Set alerts if tokens_used * price_per_token exceeds budget thresholds.
 
 ```ruby
-# In config/initializers/active_genie.rb
 ActiveGenie.configure do |config|
-  # Register your custom provider
-  config.providers.register(InternalCompanyApi::Configuration)
-  
-  # Configure your provider
-  config.internal_company_api.api_key = ENV['INTERNAL_COMPANY_API_KEY']
+  config.log.add_observer(scope: { code: :llm_usage }) do |log|
+    puts "LLM Usage: #{log[:model]} - #{log[:total_tokens]} tokens"
+  end
+  config.log.add_observer(scope: { code: :retry_attempt }) do |log|
+    puts "Retry Attempt: #{log[:attempt]} of #{log[:max_retries]}"
+  end
 end
 ```
 

data/VERSION

@@ -1 +1 @@
-0.0.24
+0.0.25

data/lib/active_genie/battle/README.md

@@ -16,18 +16,18 @@ player_1 = "Implementation uses dependency injection for better testability"
 player_2 = "Code has high test coverage but tightly coupled components"
 criteria = "Evaluate code quality and maintainability"
 
-result = ActiveGenie::Battle::Basic.call(player_1, player_2, criteria)
+result = ActiveGenie::Battle.call(player_1, player_2, criteria)
 # => {
 #      winner_player: "Implementation uses dependency injection for better testability",
-#      reasoning: "Player A's implementation demonstrates better maintainability through dependency injection, 
-#                 which allows for easier testing and component replacement. While Player B has good test coverage, 
+#      reasoning: "Player A's implementation demonstrates better maintainability through dependency injection,
+#                 which allows for easier testing and component replacement. While Player B has good test coverage,
 #                 the tight coupling makes the code harder to maintain and modify.",
 #      what_could_be_changed_to_avoid_draw: "Focus on specific architectural patterns and design principles"
 #    }
 ```
 
 ## Interface
-### Basic.call(player_1, player_2, criteria, config: {})
+### .call(player_1, player_2, criteria, config: {})
 - `player_1` [String, Hash] - The content or submission from the first player
 - `player_2` [String, Hash] - The content or submission from the second player
 - `criteria` [String] - The evaluation criteria or rules to assess against
@@ -36,4 +36,4 @@ result = ActiveGenie::Battle::Basic.call(player_1, player_2, criteria)
 Returns a Hash containing:
 - `winner_player` [String, Hash] - The winning player's content (either player_1 or player_2)
 - `reasoning` [String] - Detailed explanation of why the winner was chosen
-- `what_could_be_changed_to_avoid_draw` [String] - A suggestion on how to avoid a draw
+- `what_could_be_changed_to_avoid_draw` [String] - A suggestion on how to avoid a draw

data/lib/active_genie/battle/generalist.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative '../clients/unified_client'
+
+module ActiveGenie
+  module Battle
+    # The Generalist class provides a foundation for evaluating battles between two players
+    # using AI-powered evaluation. It determines a winner based on specified criteria,
+    # analyzing how well each player meets the requirements.
+    #
+    # The battle evaluation process compares two players' content against given criteria
+    # and returns detailed feedback including the winner and reasoning for the decision.
+    #
+    # @example Generalist usage with two players and criteria
+    #   Generalist.call("Player A content", "Player B content", "Evaluate keyword usage and pattern matching")
+    #
+    class Generalist
+      def self.call(...)
+        new(...).call
+      end
+
+      # @param player_1 [String] The content or submission from the first player
+      # @param player_2 [String] The content or submission from the second player
+      # @param criteria [String] The evaluation criteria or rules to assess against
+      # @param config [Hash] Additional configuration options that modify the battle evaluation behavior
+      # @return [Hash] The evaluation result containing the winner and reasoning
+      #   @return [String] :winner The winner, either player_1 or player_2
+      #   @return [String] :reasoning Detailed explanation of why the winner was chosen
+      #   @return [String] :what_could_be_changed_to_avoid_draw A suggestion on how to avoid a draw
+      def initialize(player_1, player_2, criteria, config: {})
+        @player_1 = player_1
+        @player_2 = player_2
+        @criteria = criteria
+        @config = ActiveGenie.configuration.merge(config)
+      end
+
+      def call
+        messages = [
+          {  role: 'system', content: PROMPT },
+          {  role: 'user', content: "criteria: #{@criteria}" },
+          {  role: 'user', content: "player_1: #{@player_1}" },
+          {  role: 'user', content: "player_2: #{@player_2}" }
+        ]
+
+        response = ::ActiveGenie::Clients::UnifiedClient.function_calling(
+          messages,
+          FUNCTION,
+          config: @config
+        )
+
+        ActiveGenie::Logger.call({
+                                   code: :battle,
+                                   player_1: @player_1[0..30],
+                                   player_2: @player_2[0..30],
+                                   criteria: @criteria[0..30],
+                                   winner: response['impartial_judge_winner'],
+                                   reasoning: response['impartial_judge_winner_reasoning']
+                                 })
+
+        response_formatted(response)
+      end
+
+      private
+
+      def response_formatted(response)
+        winner = response['impartial_judge_winner']
+        loser = case response['impartial_judge_winner']
+                when 'player_1' then 'player_2'
+                when 'player_2' then 'player_1'
+                end
+
+        { 'winner' => winner, 'loser' => loser, 'reasoning' => response['impartial_judge_winner_reasoning'] }
+      end
+
+      PROMPT = <<~PROMPT
+        Based on two players, player_1 and player_2, they will battle against each other based on criteria. Criteria are vital as they provide a clear metric to compare the players. Follow these criteria strictly.
+
+        # Steps
+        1. player_1 presents their strengths and how they meet the criteria. Max of 100 words.
+        2. player_2 presents their strengths and how they meet the criteria. Max of 100 words.
+        3. player_1 argues why they should be the winner compared to player_2. Max of 100 words.
+        4. player_2 counter-argues why they should be the winner compared to player_1. Max of 100 words.
+        5. The impartial judge chooses the winner.
+
+        # Output Format
+        - The impartial judge chooses this player as the winner.
+
+        # Notes
+        - Avoid resulting in a draw. Use reasoning or make fair assumptions if needed.
+        - Critically assess each player's adherence to the criteria.
+        - Clearly communicate the reasoning behind your decision.
+      PROMPT
+
+      FUNCTION = {
+        name: 'battle_evaluation',
+        description: 'Evaluate a battle between player_1 and player_2 using predefined criteria and identify the winner.',
+        parameters: {
+          type: 'object',
+          properties: {
+            player_1_sell_himself: {
+              type: 'string',
+              description: 'player_1 presents their strengths and how they meet the criteria. Max of 100 words.'
+            },
+            player_2_sell_himself: {
+              type: 'string',
+              description: 'player_2 presents their strengths and how they meet the criteria. Max of 100 words.'
+            },
+            player_1_arguments: {
+              type: 'string',
+              description: 'player_1 arguments for why they should be the winner compared to player_2. Max of 100 words.'
+            },
+            player_2_counter: {
+              type: 'string',
+              description: 'player_2 counter arguments for why they should be the winner compared to player_1. Max of 100 words.'
+            },
+            impartial_judge_winner_reasoning: {
+              type: 'string',
+              description: 'The detailed reasoning about why the impartial judge chose the winner. Max of 100 words.'
+            },
+            impartial_judge_winner: {
+              type: 'string',
+              description: 'Who is the winner based on the impartial judge reasoning?',
+              enum: %w[player_1 player_2]
+            }
+          },
+          required: %w[player_1_sell_himself player_2_sell_himself player_1_arguments player_2_counter
+                       impartial_judge_winner_reasoning impartial_judge_winner]
+        }
+      }.freeze
+    end
+  end
+end

data/lib/active_genie/battle.rb

@@ -1,17 +1,18 @@
+# frozen_string_literal: true
 
-require_relative 'battle/basic'
+require_relative 'battle/generalist'
 
 module ActiveGenie
   # See the [Battle README](lib/active_genie/battle/README.md) for more information.
   module Battle
     module_function
 
-    def basic(...)
-      Basic.call(...)
+    def call(...)
+      Generalist.call(...)
     end
 
-    def call(...)
-      Basic.call(...)
+    def generalist(...)
+      Generalist.call(...)
     end
   end
 end

data/lib/active_genie/clients/providers/anthropic_client.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+require_relative './base_client'
+
+module ActiveGenie
+  module Clients
+    # Client for interacting with the Anthropic (Claude) API with json response
+    class AnthropicClient < BaseClient
+      class AnthropicError < ClientError; end
+      class RateLimitError < AnthropicError; end
+
+      ANTHROPIC_ENDPOINT = '/v1/messages'
+
+      # Requests structured JSON output from the Anthropic Claude model based on a schema.
+      #
+      # @param messages [Array<Hash>] A list of messages representing the conversation history.
+      #   Each hash should have :role ('user', 'assistant', or 'system') and :content (String).
+      #   Claude uses 'user', 'assistant', and 'system' roles.
+      # @param function [Hash] A JSON schema definition describing the desired output format.
+      # @return [Hash, nil] The parsed JSON object matching the schema, or nil if parsing fails or content is empty.
+      def function_calling(messages, function)
+        model = @config.llm.model || @config.providers.anthropic.tier_to_model(@config.llm.model_tier)
+
+        system_message = messages.find { |m| m[:role] == 'system' }&.dig(:content) || ''
+        user_messages = messages.select { |m| %w[user assistant].include?(m[:role]) }
+                                .map { |m| { role: m[:role], content: m[:content] } }
+
+        anthropic_function = function.dup
+        anthropic_function[:input_schema] = function[:parameters]
+        anthropic_function.delete(:parameters)
+
+        payload = {
+          model:,
+          system: system_message,
+          messages: user_messages,
+          tools: [anthropic_function],
+          tool_choice: { name: anthropic_function[:name], type: 'tool' },
+          max_tokens: @config.llm.max_tokens,
+          temperature: @config.llm.temperature || 0
+        }
+
+        headers = {
+          'x-api-key': @config.providers.anthropic.api_key,
+          'anthropic-version': @config.providers.anthropic.anthropic_version
+        }.compact
+
+        retry_with_backoff do
+          start_time = Time.now
+          url = "#{@config.providers.anthropic.api_url}#{ANTHROPIC_ENDPOINT}"
+
+          response = post(url, payload, headers: headers)
+
+          content = response.dig('content', 0, 'input')
+
+          ActiveGenie::Logger.call({
+                                     code: :llm_usage,
+                                     input_tokens: response.dig('usage', 'input_tokens'),
+                                     output_tokens: response.dig('usage', 'output_tokens'),
+                                     total_tokens: response.dig('usage',
+                                                                'input_tokens') + response.dig('usage',
+                                                                                               'output_tokens'),
+                                     model: payload[:model],
+                                     duration: Time.now - start_time,
+                                     usage: response['usage']
+                                   })
+
+          ActiveGenie::Logger.call({ code: :function_calling, payload:, parsed_response: content })
+
+          content
+        end
+      end
+    end
+  end
+end