active_genie 0.0.2 → 0.0.8

data/lib/active_genie/scoring/basic.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative '../clients/router'
+
+module ActiveGenie::Scoring
+  # The Basic class provides a foundation for scoring text content against specified criteria
+  # using AI-powered evaluation. It supports both single and multiple reviewer scenarios,
+  # with the ability to automatically recommend reviewers when none are specified.
+  #
+  # The scoring process evaluates text based on given criteria and returns detailed feedback
+  # including individual reviewer scores, reasoning, and a final aggregated score.
+  #
+  # @example Basic usage with a single reviewer
+  #   Basic.call("Sample text", "Evaluate grammar and clarity", ["Grammar Expert"])
+  #
+  # @example Usage with automatic reviewer recommendation
+  #   Basic.call("Sample text", "Evaluate technical accuracy")
+  #
+  class Basic
+    # @param text [String] The text content to be evaluated
+    # @param criteria [String] The evaluation criteria or rubric to assess against
+    # @param reviewers [Array<String>] Optional list of specific reviewers. If empty,
+    #   reviewers will be automatically recommended based on the content and criteria
+    # @param options [Hash] Additional configuration options that modify the scoring behavior
+    # @option options [Boolean] :detailed_feedback Request more detailed feedback in the reasoning
+    # @option options [Hash] :reviewer_weights Custom weights for different reviewers
+    # @return [Hash] The evaluation result containing the scores and reasoning
+    #   @return [Number] :final_score The final score of the text based on the criteria and reviewers
+    #   @return [String] :final_reasoning Detailed explanation of why the final score was reached
+    def self.call(text, criteria, reviewers = [], options: {})
+      new(text, criteria, reviewers, options:).call
+    end
+
+    def initialize(text, criteria, reviewers = [], options: {})
+      @text = text
+      @criteria = criteria
+      @reviewers = Array(reviewers).compact.uniq
+      @options = options
+    end
+
+    def call
+      messages = [
+        {  role: 'system', content: PROMPT },
+        {  role: 'user', content: "Scoring criteria: #{@criteria}" },
+        {  role: 'user', content: "Text to score: #{@text}" }, 
+      ]
+
+      properties = {}
+      get_or_recommend_reviewers.each do |reviewer|
+        properties["#{reviewer}_reasoning"] = {
+          type: 'string',
+          description: "The reasoning of the scoring process by #{reviewer}.",
+        }
+        properties["#{reviewer}_score"] = {
+          type: 'number',
+          description: "The score given by #{reviewer}.",
+          min: 0,
+          max: 100
+        }
+      end
+
+      function = {
+        name: 'scoring',
+        description: 'Score the text based on the given criteria.',
+        schema: {
+          type: "object",
+          properties: {
+            **properties,
+            final_score: {
+              type: 'number',
+              description: 'The final score based on the previous reviewers',
+            },
+            final_reasoning: {
+              type: 'string',
+              description: 'The final reasoning based on the previous reviewers',
+            }
+          }
+        }
+      }
+
+      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+    end
+
+    private
+
+    def get_or_recommend_reviewers
+      @get_or_recommend_reviewers ||= if @reviewers.count > 0 
+        @reviewers
+      else
+        recommended_reviews = RecommendedReviews.call(@text, @criteria, options: @options)
+
+        [recommended_reviews[:reviewer1], recommended_reviews[:reviewer2], recommended_reviews[:reviewer3]]
+      end
+    end
+
+    PROMPT = <<~PROMPT
+    Evaluate and score the provided text based on predefined criteria, which may include rules, keywords, or patterns. Use a scoring range of 0 to 100, with 100 representing the highest possible score. Follow the instructions below to ensure an accurate and objective assessment.
+
+    # Evaluation Process
+    1. **Analysis**: Thoroughly compare the text against each criterion to ensure comprehensive evaluation.
+    2. **Document Deviations**: Clearly identify and document any areas where the content does not align with the specified criteria.
+    3. **Highlight Strengths**: Emphasize notable features or elements that enhance the overall quality or effectiveness of the content.
+    4. **Identify Weaknesses**: Specify areas where the content fails to meet the criteria or where improvements could be made.
+
+    # Output Requirements
+    Provide a detailed review, including:
+      - A final score (0-100)
+      - Specific reasoning for the assigned score, covering all evaluated criteria.
+      - Ensure the reasoning includes both positive aspects and suggested improvements.
+
+    # Guidelines
+    - Maintain objectivity, avoiding biases or preconceived notions.
+    - Deconstruct each criterion into actionable components for a systematic evaluation.
+    - If the text lacks information, apply reasonable judgment to assign a score while clearly explaining the rationale.
+    PROMPT
+  end
+end

data/lib/active_genie/scoring/recommended_reviews.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative '../clients/router.rb'
+
+module ActiveGenie::Scoring
+  # The RecommendedReviews class intelligently suggests appropriate reviewer roles
+  # for evaluating text content based on specific criteria. It uses AI to analyze
+  # the content and criteria to identify the most suitable subject matter experts.
+  #
+  # The class ensures a balanced and comprehensive review process by recommending
+  # three distinct reviewer roles with complementary expertise and perspectives.
+  #
+  # @example Getting recommended reviewers for technical content
+  #   RecommendedReviews.call("Technical documentation about API design", 
+  #                           "Evaluate technical accuracy and clarity")
+  #   # => { reviewer1: "API Architect", reviewer2: "Technical Writer", 
+  #   #      reviewer3: "Developer Advocate", reasoning: "..." }
+  #
+  class RecommendedReviews
+    def self.call(text, criteria, options: {})
+      new(text, criteria, options:).call
+    end
+
+    # Initializes a new reviewer recommendation instance
+    #
+    # @param text [String] The text content to analyze for reviewer recommendations
+    # @param criteria [String] The evaluation criteria that will guide reviewer selection
+    # @param options [Hash] Additional configuration options that modify the recommendation process
+    # @option options [Boolean] :prefer_technical Whether to favor technical expertise
+    # @option options [Boolean] :prefer_domain Whether to favor domain expertise
+    def initialize(text, criteria, options: {})
+      @text = text
+      @criteria = criteria
+      @options = options
+    end
+
+    def call
+      messages = [
+        {  role: 'system', content: PROMPT },
+        {  role: 'user', content: "Scoring criteria: #{@criteria}" },
+        {  role: 'user', content: "Text to score: #{@text}" },
+      ]
+
+      function = {
+        name: 'identify_reviewers',
+        description: 'Discover reviewers based on the text and given criteria.',
+        schema: {
+          type: "object",
+          properties: {
+            reasoning: { type: 'string' },
+            reviewer1: { type: 'string' },
+            reviewer2: { type: 'string' },
+            reviewer3: { type: 'string' },
+          }
+        }
+      }
+
+      ::ActiveGenie::Clients::Router.function_calling(messages, function, options: @options)
+    end
+
+    private
+
+    PROMPT = <<~PROMPT
+    Identify the top 3 suitable reviewer titles or roles based on the provided text and criteria. Selected reviewers must possess subject matter expertise, offer valuable insights, and ensure diverse yet aligned perspectives on the content.
+
+    # Instructions
+    1. **Analyze the Text and Criteria**: Examine the content and criteria to identify relevant reviewer titles or roles.
+    2. **Determine Subject Matter Expertise**: Select reviewers with substantial knowledge or experience in the subject area.
+    3. **Evaluate Insight Contribution**: Prioritize titles or roles capable of providing meaningful and actionable feedback on the content.
+    4. **Incorporate Perspective Diversity**: Ensure the selection includes reviewers with varied but complementary viewpoints while maintaining alignment with the criteria.
+
+    # Constraints
+    - Selected reviewers must align with the content’s subject matter and criteria.
+    - Include reasoning for how each choice supports a thorough and insightful review.
+    - Avoid redundant or overly similar titles/roles to maintain diversity.
+    PROMPT
+  end
+end

data/lib/active_genie/scoring.rb

@@ -0,0 +1,17 @@
+require_relative 'scoring/basic'
+require_relative 'scoring/recommended_reviews'
+
+module ActiveGenie
+  # Text evaluation system that provides detailed scoring and feedback using multiple expert reviewers
+  module Scoring
+    module_function
+
+    def basic(...)
+      Basic.call(...)
+    end
+
+    def recommended_reviews(...)
+      RecommendedReviews.call(...)
+    end
+  end
+end

data/lib/active_genie/utils/math.rb

@@ -0,0 +1,15 @@
+module ActiveGenie::Utils
+  module Math
+    module_function
+
+    def self.calculate_new_elo(winner, loser, k: 32)
+      expected_score_a = 1 / (1 + 10**((loser - winner) / 400))
+      expected_score_b = 1 - expected_score_a
+
+      new_elo_winner = winner + k * (1 - expected_score_a)
+      new_elo_loser = loser + k * (1 - expected_score_b)
+
+      [new_elo_winner, new_elo_loser]
+    end
+  end
+end

data/lib/active_genie.rb

@@ -1,22 +1,34 @@
 module ActiveGenie
-  autoload :DataExtractor, File.join(__dir__, 'data_extractor/data_extractor')
   autoload :Configuration, File.join(__dir__, 'active_genie/configuration')
 
-  class << self
-    def config
-      @config ||= Configuration.new
-    end
+  # Modules
+  autoload :DataExtractor, File.join(__dir__, 'active_genie/data_extractor')
+  autoload :Battle, File.join(__dir__, 'active_genie/battle')
+  autoload :Scoring, File.join(__dir__, 'active_genie/scoring')
+  autoload :Leaderboard, File.join(__dir__, 'active_genie/leaderboard')
 
+  class << self    
     def configure
       yield(config) if block_given?
     end
-    
+
+    def load_tasks
+      return unless defined?(Rake)
+
+      Rake::Task.define_task(:environment)
+      Dir.glob(File.join(__dir__, 'tasks', '*.rake')).each { |r| load r }
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
     def [](key)
       config.values[key.to_s]
     end
 
-    def config_by_model(model)
-      config.values[model&.to_s] || config.values.values.first || {}
+    def config_by_model(model = nil)
+      config.values[model&.to_s&.downcase&.strip] || config.values.values.first || {}
     end
   end
 end

data/lib/tasks/install.rake

@@ -3,7 +3,7 @@ require 'fileutils'
 namespace :active_genie do
   desc 'Install active_genie configuration file'
   task :install do
-    source = File.join('lib', 'tasks', 'templates', 'active_genie.yml')
+    source = File.join(__dir__, 'templates', 'active_genie.yml')
     target = File.join('config', 'active_genie.yml')
 
     FileUtils.cp(source, target)

data/lib/tasks/templates/{active_ai.yml → active_genie.yml}

@@ -1,4 +1,4 @@
-# GPT-4o-mini:
+# gpt-4o-mini:
 #   api_key: <%= ENV['OPENAI_API_KEY'] %>
 #   provider: 'openai'
 # 

metadata

@@ -1,22 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: active_genie
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.8
 platform: ruby
 authors:
 - Radamés Roriz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-01-23 00:00:00.000000000 Z
+date: 2025-02-06 00:00:00.000000000 Z
 dependencies: []
-description: |2
-      ActiveGenie provides a robust toolkit for integrating AI capabilities into Ruby applications.
-      Features include:
-      * Structured data extraction from text
-      * Smart text summarization
-      * Content scoring and ranking
-      * AI-powered classification
+description: "# ActiveGenie \U0001F9DE‍♂️\n> Transform your Ruby application with
+  powerful, production-ready GenAI features\n\n[![Gem Version](https://badge.fury.io/rb/active_genie.svg?icon=si%3Arubygems)](https://badge.fury.io/rb/active_genie)\n[![Ruby](https://github.com/roriz/active_genie/actions/workflows/ruby.yml/badge.svg)](https://github.com/roriz/active_genie/actions/workflows/ruby.yml)\n\nActiveGenie
+  is a Ruby gem that provides a polished, production-ready interface for working with
+  Generative AI (GenAI) models. Just like ActiveStorage simplifies file handling in
+  Rails, ActiveGenie makes it effortless to integrate GenAI capabilities into your
+  Ruby applications.\n\n## Features\n\n- \U0001F3AF **Data Extraction**: Extract structured
+  data from unstructured text with type validation\n- \U0001F4CA **Smart Scoring**:
+  Multi-reviewer evaluation system with automatic expert selection\n- \U0001F4AD **Sentiment
+  Analysis**: Advanced sentiment analysis with customizable rules\n- \U0001F512 **Safe
+  & Secure**: Built-in validation and sanitization\n- \U0001F6E0️ **Configurable**:
+  Supports multiple GenAI providers and models\n\n## Installation\n\n1. Add to your
+  Gemfile:\n```ruby\ngem 'active_genie'\n```\n\n2. Install the gem:\n```shell\nbundle
+  install\n```\n\n3. Generate the configuration:\n```shell\necho \"ActiveGenie.load_tasks\"
+  >> Rakefile\nrails g active_genie:install\n```\n\n4. [Optional] Configure your credentials
+  in `config/active_genie.yml`:\n```yaml\nGPT-4o-mini:\n  api_key: <%= ENV['OPENAI_API_KEY']
+  %>\n  provider: \"openai\"\n\nclaude-3-5-sonnet:\n  api_key: <%= ENV['ANTHROPIC_API_KEY']
+  %>\n  provider: \"anthropic\"\n```\n\n> The first key will be used as default in
+  all modules, in this example `GPT-4o-mini`\n\n## Quick Start\n\n### Data Extractor\nExtract
+  structured data from text using AI-powered analysis, handling informal language
+  and complex expressions.\n\n```ruby\ntext = \"Nike Air Max 90 - Size 42 - $199.99\"\nschema
+  = {\n  brand: { \n    type: 'string',\n    enum: [\"Nike\", \"Adidas\", \"Puma\"]\n
+  \ },\n  price: { \n    type: 'number',\n    minimum: 0\n  },\n  size: {\n    type:
+  'integer',\n    minimum: 35,\n    maximum: 46\n  }\n}\n\nresult = ActiveGenie::DataExtractor.call(text,
+  schema)\n# => { \n#      brand: \"Nike\", \n#      brand_explanation: \"Brand name
+  found at start of text\",\n#      price: 199.99,\n#      price_explanation: \"Price
+  found in USD format at end\",\n#      size: 42,\n#      size_explanation: \"Size
+  explicitly stated in the middle\"\n#    }\n```\n\nFeatures:\n- Structured data extraction
+  with type validation\n- Schema-based extraction with custom constraints\n- Informal
+  text analysis (litotes, hedging)\n- Detailed explanations for extracted values\n\nSee
+  the [Data Extractor README](lib/active_genie/data_extractor/README.md) for informal
+  text processing, advanced schemas, and detailed interface documentation.\n\n###
+  Scoring\nText evaluation system that provides detailed scoring and feedback using
+  multiple expert reviewers. Get balanced scoring through AI-powered expert reviewers
+  that automatically adapt to your content.\n\n```ruby\ntext = \"The code implements
+  a binary search algorithm with O(log n) complexity\"\ncriteria = \"Evaluate technical
+  accuracy and clarity\"\n\nresult = ActiveGenie::Scoring::Basic.call(text, criteria)\n#
+  => {\n#      algorithm_expert_score: 95,\n#      algorithm_expert_reasoning: \"Accurately
+  describes binary search and its complexity\",\n#      technical_writer_score: 90,\n#
+  \     technical_writer_reasoning: \"Clear and concise explanation of the algorithm\",\n#
+  \     final_score: 92.5\n#    }\n```\n\nFeatures:\n- Multi-reviewer evaluation with
+  automatic expert selection\n- Detailed feedback with scoring reasoning\n- Customizable
+  reviewer weights\n- Flexible evaluation criteria\n\nSee the [Scoring README](lib/active_genie/scoring/README.md)
+  for advanced usage, custom reviewers, and detailed interface documentation.\n\n###
+  Battle\nAI-powered battle evaluation system that determines winners between two
+  players based on specified criteria.\n\n```ruby\nrequire 'active_genie'\n\nplayer_a
+  = \"Implementation uses dependency injection for better testability\"\nplayer_b
+  = \"Code has high test coverage but tightly coupled components\"\ncriteria = \"Evaluate
+  code quality and maintainability\"\n\nresult = ActiveGenie::Battle::Basic.call(player_a,
+  player_b, criteria)\n# => {\n#      winner_player: \"Implementation uses dependency
+  injection for better testability\",\n#      reasoning: \"Player A's implementation
+  demonstrates better maintainability through dependency injection, \n#                 which
+  allows for easier testing and component replacement. While Player B has good test
+  coverage, \n#                 the tight coupling makes the code harder to maintain
+  and modify.\",\n#      what_could_be_changed_to_avoid_draw: \"Focus on specific
+  architectural patterns and design principles\"\n#    }\n```\n\nFeatures:\n- Multi-reviewer
+  evaluation with automatic expert selection\n- Detailed feedback with scoring reasoning\n-
+  Customizable reviewer weights\n- Flexible evaluation criteria\n\nSee the [Battle
+  README](lib/active_genie/battle/README.md) for advanced usage, custom reviewers,
+  and detailed interface documentation.\n\n### Summarizer (WIP)\nThe summarizer is
+  a tool that can be used to summarize a given text. It uses a set of rules to summarize
+  the text out of the box. Uses the best practices of prompt engineering and engineering
+  to make the summarization as accurate as possible.\n\n```ruby\nrequire 'active_genie'\n\ntext
+  = \"Example text to be summarized. The fox jumps over the dog\"\nsummarized_text
+  = ActiveGenie::Summarizer.call(text)\nputs summarized_text # => \"The fox jumps
+  over the dog\"\n```\n\n### Language detector (WIP)\nThe language detector is a tool
+  that can be used to detect the language of a given text. It uses a set of rules
+  to detect the language of the text out of the box. Uses the best practices of prompt
+  engineering and engineering to make the language detection as accurate as possible.\n\n```ruby\nrequire
+  'active_genie'\n\ntext = \"Example text to be detected\"\nlanguage = ActiveGenie::LanguageDetector.call(text)\nputs
+  language # => \"en\"\n```\n\n### Translator (WIP)\nThe translator is a tool that
+  can be used to translate a given text. It uses a set of rules to translate the text
+  out of the box. Uses the best practices of prompt engineering and engineering to
+  make the translation as accurate as possible.\n\n```ruby\nrequire 'active_genie'\n\ntext
+  = \"Example text to be translated\"\ntranslated_text = ActiveGenie::Translator.call(text,
+  from: 'en', to: 'pt')\nputs translated_text # => \"Exemplo de texto a ser traduzido\"\n```\n\n###
+  Sentiment analyzer (WIP)\nThe sentiment analyzer is a tool that can be used to analyze
+  the sentiment of a given text. It uses a set of rules to analyze the sentiment of
+  the text out of the box. Uses the best practices of prompt engineering and engineering
+  to make the sentiment analysis as accurate as possible.\n\n```ruby\nrequire 'active_genie'\n\ntext
+  = \"Example text to be analyzed\"\nsentiment = ActiveGenie::SentimentAnalyzer.call(text)\nputs
+  sentiment # => \"positive\"\n```\n\n### Elo ranking (WIP)\nThe Elo ranking is a
+  tool that can be used to rank a set of items. It uses a set of rules to rank the
+  items out of the box. Uses the best practices of prompt engineering and engineering
+  to make the ranking as accurate as possible.\n\n```ruby\nrequire 'active_genie'\n\nitems
+  = ['Square', 'Circle', 'Triangle']\ncriterias = 'items that look rounded'\nranked_items
+  = ActiveGenie::EloRanking.call(items, criterias, rounds: 10)\nputs ranked_items
+  # => [{ name: \"Circle\", score: 1500 }, { name: \"Square\", score: 800 }, { name:
+  \"Triangle\", score: 800 }]\n```\n\n\n## Configuration Options\n\n| Option | Description
+  | Default |\n|--------|-------------|---------|\n| `provider` | LLM provider (openai,
+  anthropic, etc) | `nil` |\n| `model` | Model to use | `nil` |\n| `api_key` | Provider
+  API key | `nil` |\n| `timeout` | Request timeout in seconds | `5` |\n| `max_retries`
+  | Maximum retry attempts | `3` |\n\n> **Note:** Each module can append its own set
+  of configuration options, see the individual module documentation for details.\n\n##
+  Contributing\n\n1. Fork the repository\n2. Create your feature branch (`git checkout
+  -b feature/amazing-feature`)\n3. Commit your changes (`git commit -m 'Add amazing
+  feature'`)\n4. Push to the branch (`git push origin feature/amazing-feature`)\n5.
+  Open a Pull Request\n## License\n\nThis project is licensed under the MIT License
+  - see the [LICENSE](LICENSE) file for details.\n\n"
 email:
 - radames@roriz.dev
 executables: []
@@ -27,13 +118,29 @@ files:
 - README.md
 - VERSION
 - lib/active_genie.rb
+- lib/active_genie/battle.rb
+- lib/active_genie/battle/README.md
+- lib/active_genie/battle/basic.rb
+- lib/active_genie/clients/openai.rb
+- lib/active_genie/clients/router.rb
 - lib/active_genie/configuration.rb
-- lib/data_extractor/README.md
-- lib/data_extractor/data_extractor.rb
-- lib/requester/openai.rb
-- lib/requester/requester.rb
+- lib/active_genie/data_extractor.rb
+- lib/active_genie/data_extractor/README.md
+- lib/active_genie/data_extractor/basic.rb
+- lib/active_genie/data_extractor/from_informal.rb
+- lib/active_genie/leaderboard.rb
+- lib/active_genie/leaderboard/elo_ranking.rb
+- lib/active_genie/leaderboard/leaderboard.rb
+- lib/active_genie/leaderboard/league.rb
+- lib/active_genie/leaderboard/player.rb
+- lib/active_genie/leaderboard/players_collection.rb
+- lib/active_genie/scoring.rb
+- lib/active_genie/scoring/README.md
+- lib/active_genie/scoring/basic.rb
+- lib/active_genie/scoring/recommended_reviews.rb
+- lib/active_genie/utils/math.rb
 - lib/tasks/install.rake
-- lib/tasks/templates/active_ai.yml
+- lib/tasks/templates/active_genie.yml
 homepage: https://github.com/Roriz/active_genie
 licenses:
 - Apache-2.0
@@ -41,7 +148,6 @@ metadata:
   homepage_uri: https://github.com/Roriz/active_genie
   documentation_uri: https://github.com/Roriz/active_genie/wiki
   changelog_uri: https://github.com/Roriz/active_genie/blob/master/CHANGELOG.md
-  source_code_uri: https://github.com/Roriz/active_genie
   bug_tracker_uri: https://github.com/Roriz/active_genie/issues
   rubygems_mfa_required: 'true'
 post_install_message: 
@@ -62,6 +168,5 @@ requirements: []
 rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
-summary: Modules and classes to help you build AI features, like data extraction,
-  summarization, scoring, and ranking.
+summary: Transform your Ruby application with powerful, production-ready GenAI features
 test_files: []

data/lib/data_extractor/README.md

@@ -1,103 +0,0 @@
-# Data Extractor
-Extract structured data from text using LLM-powered analysis, handling informal language and complex expressions.
-
-## ✨ Features
-- Simple extraction   - Extract structured data from unstructured text
-- Informal extraction - Identifies and handles informal language patterns
-
-## Basic Usage
-
-Extract structured data from text using predefined schemas:
-
-```ruby
-text = "iPhone 14 Pro Max"
-schema = {
-  brand: { type: 'string' },
-  model: { type: 'string' }
-}
-result = ActiveGenie::DataExtractor.call(text, schema)
-# => { brand: "iPhone", model: "14 Pro Max" }
-
-product = "Nike Air Max 90 - Size 42 - $199.99"
-schema = {
-  brand: { 
-    type: 'string',
-    enum: ["Nike", "Adidas", "Puma"]
-  },
-  price: { 
-    type: 'number',
-    minimum: 0
-  },
-  currency: { 
-    type: 'string',
-    enum: ["USD", "EUR"]
-  },
-  size: {
-    type: 'integer',
-    minimum: 35,
-    maximum: 46
-  }
-}
-
-result = ActiveGenie::DataExtractor.call(product, schema)
-# => { brand: "Nike", price: 199.99, size: 42, currency: "USD" }
-```
-
-## Informal Text Processing
-
-The `from_informal` method helps extract structured data from casual, conversational text by interpreting common informal expressions and linguistic patterns like:
-
-- Affirmative expressions ("sure", "no problem", "you bet")
-- Negative expressions ("nah", "not really", "pass")
-- Hedging ("maybe", "I guess", "probably")
-- Litotes ("not bad", "not the worst")
-
-### Example
-
-```ruby
-text = <<~TEXT
-  system: Would you like to proceed with the installation?
-  user: not bad
-TEXT
-
-data_to_extract = {
-  user_consent: { type: 'boolean' }
-}
-
-result = ActiveGenie::DataExtractor.from_informal(text, data_to_extract)
-puts result # => { user_consent: true }
-```
-
-### Usage Notes
-- Best suited for processing conversational user inputs
-- Handles ambiguous or indirect responses
-- Useful for chatbots and conversational interfaces
-- May require more processing time and tokens
-- Accuracy depends on context clarity
-
-⚠️ Performance Impact: This method uses additional language processing, which can increase token usage and processing time. 
-
-## Interface
-`.call(text, data_to_extract, options = {})`
-Extracts structured data from text based on a schema.
-
-### Parameters
-| Name | Type | Description | Required | Example | Default |
-| --- | --- | --- | --- | --- | --- |
-| `text` | `String` | The text to extract data from. Max 1000 chars. | Yes | "These Nike shoes are red" | - |
-| `data_to_extract` | `Hash` | [JSON Schema object](https://json-schema.org/understanding-json-schema/reference/object) defining data structure | Yes | `{ category: { type: 'string', enum: ["shoes"] } }` | - |
-| `options` | `Hash` | Additional provider configuration options | No | `{ model: "gpt-4" }` | `{}` |
-
-### Returns
-`Hash` - Dynamic hash based on the `data_to_extract` schema.
-
-### Options
-| Name | Type | Description | Default |
-| --- | --- | --- | --- |
-| `model` | `String` | The model name | `YAML.load_file(config_file).first.model` |
-| `api_key` | `String` | The API key to use or api_key from model on config.yml | `YAML.load_file(config_file).first.api_key` |
-
-⚠️ Performance Considerations
-- Processes may require multiple LLM calls
-- Expect ~100 tokens per request + the text from input
-- Consider background processing for production use

data/lib/data_extractor/data_extractor.rb

@@ -1,88 +0,0 @@
-require_relative '../requester/requester.rb'
-
-module ActiveGenie
-  class DataExtractor    
-    class << self
-      # Extracts data from user_texts based on the schema defined in data_to_extract.
-      # @param text [String] The text to extract data from.
-      # @param data_to_extract [Hash] The schema to extract data from the text.
-      # @param options [Hash] The options to pass to the function.
-      # @return [Hash] The extracted data.
-      def call(text, data_to_extract, options: {})
-        messages = [
-          {  role: 'system', content: PROMPT },
-          {  role: 'user', content: text[0..1000] }
-        ]
-        function = {
-          name: 'data_extractor',
-          description: 'Extract structured and typed data from user messages.',
-          schema: {
-            type: "object",
-            properties: data_to_extract_with_explaination(data_to_extract)
-          }
-        }
-
-        ::ActiveGenie::Requester.function_calling(messages, function, options)
-      end
-
-      def from_informal(text, data_to_extract, options = {})
-        messages = [
-          {  role: 'system', content: PROMPT },
-          {  role: 'user', content: text }
-        ]
-        properties = data_to_extract_with_explaination(data_to_extract)
-        properties[:message_litote] = {
-          type: 'boolean',
-          description: 'Return true if the message is a litote. A litote is a figure of speech that uses understatement to emphasize a point by stating a negative to further affirm a positive, often incorporating double negatives for effect.'
-        }
-        properties[:litote_rephrased] = {
-          type: 'string',
-          description: 'The true meaning of the litote. Rephrase the message to a positive statement.'
-        }
-
-        function = {
-          name: 'data_extractor',
-          description: 'Extract structured and typed data from user messages.',
-          schema: { type: "object", properties: }
-        }
-
-        ::ActiveGenie::Requester.function_calling(messages, function, options)
-      end
-
-      private
-
-      PROMPT = <<~PROMPT
-      Extract structured and typed data from user messages.
-      Identify relevant information within user messages and categorize it into predefined data fields with specific data types.
-
-      # Steps
-      1. **Identify Data Types**: Determine the types of data to collect, such as names, dates, email addresses, phone numbers, etc.
-      2. **Extract Information**: Use pattern recognition and language understanding to identify and extract the relevant pieces of data from the user message.
-      3. **Categorize Data**: Assign the extracted data to the appropriate predefined fields.
-      4. **Structure Data**: Format the extracted and categorized data in a structured format, such as JSON.
-
-      # Output Format
-      The output should be a JSON object containing fields with their corresponding extracted values. If a value is not found, the field should still be included with a null value.
-
-      # Notes
-      - Handle missing or partial information gracefully.
-      - Manage multiple occurrences of similar data points by prioritizing the first one unless specified otherwise.
-      - Be flexible to handle variations in data format and language clues.
-      PROMPT
-
-      def data_to_extract_with_explaination(data_to_extract)
-        with_explaination = {}
-    
-        data_to_extract.each do |key, value|
-          with_explaination[key] = value
-          with_explaination["#{key}_explanation"] = {
-            type: 'string',
-            description: "The chain of thought that led to the conclusion about: #{key}. Can be blank if the user didn't provide any context",
-          }
-        end
-        
-        with_explaination
-      end
-    end
-  end
-end