sentiment_insights 0.1.0

data/lib/sentiment_insights/clients/entities/open_ai_client.rb

@@ -0,0 +1,113 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+
+module SentimentInsights
+  module Clients
+    module Entities
+      class OpenAIClient
+        DEFAULT_MODEL   = "gpt-3.5-turbo"
+        DEFAULT_RETRIES = 3
+
+        def initialize(api_key: ENV['OPENAI_API_KEY'], model: DEFAULT_MODEL, max_retries: DEFAULT_RETRIES)
+          @api_key = api_key or raise ArgumentError, "OpenAI API key is required"
+          @model = model
+          @max_retries = max_retries
+          @logger = Logger.new($stdout)
+        end
+
+        def extract_batch(entries, question: nil)
+          responses = []
+          entity_map = Hash.new { |h, k| h[k] = [] }
+
+          entries.each_with_index do |entry, index|
+            sentence = entry[:answer].to_s.strip
+            next if sentence.empty?
+
+            response_id = "r_#{index + 1}"
+            entities = extract_entities_from_sentence(sentence)
+
+            responses << {
+              id: response_id,
+              sentence: sentence,
+              segment: entry[:segment] || {}
+            }
+
+            entities.each do |ent|
+              key = [ent[:text].downcase, ent[:type]]
+              entity_map[key] << response_id
+            end
+          end
+
+          entity_records = entity_map.map do |(text, type), ref_ids|
+            {
+              entity: text,
+              type: type,
+              mentions: ref_ids.uniq,
+              summary: nil
+            }
+          end
+
+          { entities: entity_records, responses: responses }
+        end
+
+        private
+
+        def extract_entities_from_sentence(text)
+          prompt = <<~PROMPT
+            Extract named entities from this sentence. Return them as a JSON array with each item having "text" and "type" (e.g., PERSON, ORGANIZATION, LOCATION, PRODUCT).
+            Sentence: "#{text}"
+          PROMPT
+
+          body = build_request_body(prompt)
+          response = post_openai(body)
+
+          begin
+            raw_json = response.dig("choices", 0, "message", "content").to_s.strip
+            JSON.parse(raw_json, symbolize_names: true)
+          rescue JSON::ParserError => e
+            @logger.warn "Failed to parse entity JSON: #{e.message}"
+            []
+          end
+        end
+
+        def build_request_body(prompt)
+          {
+            model: @model,
+            messages: [{ role: "user", content: prompt }],
+            temperature: 0.3
+          }
+        end
+
+        def post_openai(body)
+          uri = URI("https://api.openai.com/v1/chat/completions")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+
+          attempt = 0
+          while attempt < @max_retries
+            attempt += 1
+
+            request = Net::HTTP::Post.new(uri)
+            request["Content-Type"] = "application/json"
+            request["Authorization"] = "Bearer #{@api_key}"
+            request.body = JSON.generate(body)
+
+            begin
+              response = http.request(request)
+              return JSON.parse(response.body) if response.code.to_i == 200
+              @logger.warn "OpenAI entity extraction failed (#{response.code}): #{response.body}"
+            rescue => e
+              @logger.error "Error during entity extraction: #{e.class} - #{e.message}"
+            end
+
+            sleep(2 ** (attempt - 1)) if attempt < @max_retries
+          end
+
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/clients/key_phrases/aws_client.rb

@@ -0,0 +1,89 @@
+require 'aws-sdk-comprehend'
+require 'logger'
+
+module SentimentInsights
+  module Clients
+    module KeyPhrases
+      class AwsClient
+        MAX_BATCH_SIZE = 25
+
+        def initialize(region: 'us-east-1')
+          @comprehend = Aws::Comprehend::Client.new(region: region)
+          @logger = Logger.new($stdout)
+        end
+
+        def extract_batch(entries, question: nil)
+          responses = []
+          phrase_map = Hash.new { |h, k| h[k] = [] }
+
+          # Split into batches for AWS Comprehend
+          entries.each_slice(MAX_BATCH_SIZE).with_index do |batch, batch_idx|
+            texts = batch.map { |e| e[:answer].to_s.strip[0...5000] }
+
+            begin
+              phrase_resp = @comprehend.batch_detect_key_phrases({
+                                                                   text_list: texts,
+                                                                   language_code: 'en'
+                                                                 })
+
+              sentiment_resp = @comprehend.batch_detect_sentiment({
+                                                                    text_list: texts,
+                                                                    language_code: 'en'
+                                                                  })
+
+              phrase_resp.result_list.each_with_index do |phrase_result, idx|
+                sentiment_result = sentiment_resp.result_list.find { |s| s.index == phrase_result.index }
+                sentiment_label = sentiment_result&.sentiment&.downcase&.to_sym || :neutral
+
+                entry_index = (batch_idx * MAX_BATCH_SIZE) + idx
+                entry = entries[entry_index]
+                sentence = texts[idx]
+                response_id = "r_#{entry_index + 1}"
+
+                responses << {
+                  id: response_id,
+                  sentence: sentence,
+                  sentiment: sentiment_label,
+                  segment: entry[:segment] || {}
+                }
+
+                phrases = phrase_result.key_phrases.map { |p| p.text.downcase.strip }.uniq
+                phrases.each { |phrase| phrase_map[phrase] << response_id }
+              end
+
+              phrase_resp.error_list.each do |error|
+                @logger.warn "AWS KeyPhrase error at index #{error.index}: #{error.error_code}"
+              end
+
+              sentiment_resp.error_list.each do |error|
+                @logger.warn "AWS Sentiment error at index #{error.index}: #{error.error_code}"
+              end
+
+            rescue Aws::Comprehend::Errors::ServiceError => e
+              @logger.error "AWS Comprehend batch error: #{e.message}"
+              batch.each_with_index do |entry, i|
+                entry_index = (batch_idx * MAX_BATCH_SIZE) + i
+                responses << {
+                  id: "r_#{entry_index + 1}",
+                  sentence: entry[:answer],
+                  sentiment: :neutral,
+                  segment: entry[:segment] || {}
+                }
+              end
+            end
+          end
+
+          phrases = phrase_map.map do |phrase, ref_ids|
+            {
+              phrase: phrase,
+              mentions: ref_ids.uniq,
+              summary: nil
+            }
+          end
+
+          { phrases: phrases, responses: responses }
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/clients/key_phrases/open_ai_client.rb

@@ -0,0 +1,119 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+require_relative '../sentiment/open_ai_client'
+
+module SentimentInsights
+  module Clients
+    module KeyPhrases
+      class OpenAIClient
+        DEFAULT_MODEL   = "gpt-3.5-turbo"
+        DEFAULT_RETRIES = 3
+
+        def initialize(api_key: ENV['OPENAI_API_KEY'], model: DEFAULT_MODEL, max_retries: DEFAULT_RETRIES)
+          @api_key = api_key or raise ArgumentError, "OpenAI API key is required"
+          @model = model
+          @max_retries = max_retries
+          @logger = Logger.new($stdout)
+          @sentiment_client = SentimentInsights::Clients::Sentiment::OpenAIClient.new(api_key: @api_key, model: @model)
+        end
+
+        # Extract key phrases from entries and enrich with sentiment
+        def extract_batch(entries, question: nil)
+          responses = []
+          phrase_map = Hash.new { |h, k| h[k] = [] }
+
+          # Fetch sentiments in batch from sentiment client
+          sentiments = @sentiment_client.analyze_entries(entries, question: question)
+
+          entries.each_with_index do |entry, index|
+            sentence = entry[:answer].to_s.strip
+            next if sentence.empty?
+
+            response_id = "r_#{index + 1}"
+            phrases = extract_phrases_from_sentence(sentence)
+
+            sentiment = sentiments[index] || { label: :neutral }
+
+            responses << {
+              id: response_id,
+              sentence: sentence,
+              sentiment: sentiment[:label],
+              segment: entry[:segment] || {}
+            }
+
+            phrases.each do |phrase|
+              phrase_map[phrase.downcase] << response_id
+            end
+          end
+
+          phrases = phrase_map.map do |phrase, ref_ids|
+            {
+              phrase: phrase,
+              mentions: ref_ids.uniq,
+              summary: nil
+            }
+          end
+
+          { phrases: phrases, responses: responses }
+        end
+
+        private
+
+        def extract_phrases_from_sentence(text)
+          prompt = <<~PROMPT
+          Extract the key phrases from this sentence:
+          "#{text}"
+          Return them as a comma-separated list.
+          PROMPT
+
+          body = build_request_body(prompt)
+          response = post_openai(body)
+          parse_phrases(response)
+        end
+
+        def build_request_body(prompt)
+          {
+            model: @model,
+            messages: [{ role: "user", content: prompt }],
+            temperature: 0.3
+          }
+        end
+
+        def post_openai(body)
+          uri = URI("https://api.openai.com/v1/chat/completions")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+
+          attempt = 0
+          while attempt < @max_retries
+            attempt += 1
+
+            request = Net::HTTP::Post.new(uri)
+            request["Content-Type"] = "application/json"
+            request["Authorization"] = "Bearer #{@api_key}"
+            request.body = JSON.generate(body)
+
+            begin
+              response = http.request(request)
+              return JSON.parse(response.body) if response.code.to_i == 200
+              @logger.warn "OpenAI request failed (#{response.code}): #{response.body}"
+            rescue => e
+              @logger.error "OpenAI HTTP error: #{e.class} - #{e.message}"
+            end
+
+            sleep(2 ** (attempt - 1)) if attempt < @max_retries
+          end
+
+          {}
+        end
+
+        def parse_phrases(response)
+          text = response.dig("choices", 0, "message", "content").to_s.strip
+          text.split(/,|\n/).map(&:strip).reject(&:empty?)
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/clients/sentiment/aws_comprehend_client.rb

@@ -0,0 +1,72 @@
+require 'aws-sdk-comprehend'
+require 'logger'
+
+module SentimentInsights
+  module Clients
+    module Sentiment
+      class AwsComprehendClient
+        MAX_BATCH_SIZE = 25  # AWS limit
+
+        def initialize(region: 'us-east-1')
+          @client = Aws::Comprehend::Client.new(region: region)
+          @logger = Logger.new($stdout)
+        end
+
+        # Analyze a batch of entries using AWS Comprehend.
+        # @param entries [Array<Hash>] each with :answer key
+        # @return [Array<Hash>] each with :label (symbol) and :score (float)
+        def analyze_entries(entries, question: nil)
+          results = []
+
+          entries.each_slice(MAX_BATCH_SIZE) do |batch|
+            texts = batch.map { |entry| entry[:answer].to_s.strip[0...5000] } # max per AWS
+
+            begin
+              resp = @client.batch_detect_sentiment({
+                                                      text_list: texts,
+                                                      language_code: "en"
+                                                    })
+
+              resp.result_list.each do |r|
+                label = r.sentiment.downcase.to_sym  # :positive, :neutral, :negative, :mixed
+                score = compute_score(r.sentiment, r.sentiment_score)
+                results << { label: label, score: score }
+              end
+
+              # handle errors (will match by index)
+              resp.error_list.each do |error|
+                @logger.warn "AWS Comprehend error at index #{error.index}: #{error.error_code}"
+                results.insert(error.index, { label: :neutral, score: 0.0 })
+              end
+
+            rescue Aws::Comprehend::Errors::ServiceError => e
+              @logger.error "AWS Comprehend batch error: #{e.message}"
+              batch.size.times { results << { label: :neutral, score: 0.0 } }
+            end
+          end
+
+          results
+        end
+
+        private
+
+        # Convert AWS sentiment score hash to a single signed score.
+        def compute_score(label, scores)
+          case label.upcase
+          when "POSITIVE"
+            scores.positive.to_f
+          when "NEGATIVE"
+            -scores.negative.to_f
+          when "NEUTRAL"
+            0.0
+          when "MIXED"
+            # Optionally: net positive - negative for mixed
+            (scores.positive.to_f - scores.negative.to_f).round(2)
+          else
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/clients/sentiment/open_ai_client.rb

@@ -0,0 +1,115 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+
+module SentimentInsights
+  module Clients
+    module Sentiment
+      class OpenAIClient
+        DEFAULT_MODEL   = "gpt-3.5-turbo"
+        DEFAULT_RETRIES = 3
+
+        def initialize(api_key: ENV['OPENAI_API_KEY'], model: DEFAULT_MODEL, max_retries: DEFAULT_RETRIES, return_scores: true)
+          @api_key = api_key or raise ArgumentError, "OpenAI API key is required"
+          @model = model
+          @max_retries = max_retries
+          @return_scores = return_scores
+          @logger = Logger.new($stdout)
+        end
+
+        def analyze_entries(entries, question: nil)
+          prompt_content = build_prompt_content(entries, question)
+          request_body = {
+            model: @model,
+            messages: [
+              { role: "user", content: prompt_content }
+            ],
+            temperature: 0.0
+          }
+
+          uri = URI("https://api.openai.com/v1/chat/completions")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+
+          response_content = nil
+          attempt = 0
+
+          while attempt < @max_retries
+            attempt += 1
+            request = Net::HTTP::Post.new(uri)
+            request["Content-Type"] = "application/json"
+            request["Authorization"] = "Bearer #{@api_key}"
+            request.body = JSON.generate(request_body)
+
+            begin
+              response = http.request(request)
+            rescue StandardError => e
+              @logger.error "OpenAI API request error: #{e.class} - #{e.message}"
+              raise
+            end
+
+            status = response.code.to_i
+            if status == 429
+              @logger.warn "Rate limit (HTTP 429) on attempt #{attempt}. Retrying..."
+              sleep(2 ** (attempt - 1))
+              next
+            elsif status != 200
+              @logger.error "Request failed (#{status}): #{response.body}"
+              raise "OpenAI API Error: #{status}"
+            else
+              data = JSON.parse(response.body)
+              response_content = data.dig("choices", 0, "message", "content")
+              break
+            end
+          end
+
+          parse_sentiments(response_content, entries.size)
+        end
+
+        private
+
+        def build_prompt_content(entries, question)
+          prompt = ""
+          prompt << "Question: #{question}\n" if question
+          prompt << <<~INSTRUCTIONS
+          For each of the following customer responses, classify the sentiment as Positive, Neutral, or Negative, and assign a score between -1.0 (very negative) and 1.0 (very positive).
+
+          Reply with a numbered list like:
+          1. Positive (0.9)
+          2. Negative (-0.8)
+          3. Neutral (0.0)
+
+          INSTRUCTIONS
+
+          entries.each_with_index do |entry, index|
+            prompt << "#{index + 1}. \"#{entry[:answer]}\"\n"
+          end
+
+          prompt
+        end
+
+        def parse_sentiments(content, expected_count)
+          sentiments = []
+
+          content.to_s.strip.split(/\r?\n/).each do |line|
+            if line.strip =~ /^\d+[\.:)]?\s*(Positive|Negative|Neutral)\s*\(([-\d\.]+)\)/i
+              label = $1.downcase.to_sym
+              score = $2.to_f
+              sentiments << { label: label, score: score }
+            end
+          end
+
+          if sentiments.size != expected_count
+            @logger.warn "Expected #{expected_count} results, got #{sentiments.size}. Padding with neutral."
+            while sentiments.size < expected_count
+              sentiments << { label: :neutral, score: 0.0 }
+            end
+          end
+
+          sentiments.first(expected_count)
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/clients/sentiment/sentimental_client.rb

@@ -0,0 +1,33 @@
+require 'sentimental'
+
+module SentimentInsights
+  module Clients
+    module Sentiment
+      # Client that uses the Sentimental gem for local sentiment analysis.
+      class SentimentalClient
+        def initialize
+          @analyzer = Sentimental.new
+          @analyzer.load_defaults  # load built-in positive/negative word scores
+        end
+
+        # Analyzes each entry's answer text and returns an array of sentiment results.
+        # @param entries [Array<Hash>] An array of response hashes (each with :answer).
+        # @param question [String, nil] (unused) Global question context, not needed for local analysis.
+        # @return [Array<Hash>] An array of hashes with sentiment classification and score for each entry.
+        def analyze_entries(entries, question: nil)
+          puts "Inside sentimental"
+          entries.map do |entry|
+            text = entry[:answer].to_s.strip
+            label = @analyzer.sentiment(text)  # :positive, :neutral, or :negative
+            score = case label
+                    when :positive then 1.0
+                    when :negative then -1.0
+                    else 0.0
+                    end
+            { label: label, score: score }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sentiment_insights/configuration.rb

@@ -0,0 +1,11 @@
+module SentimentInsights
+  class Configuration
+    attr_accessor :provider, :openai_api_key, :aws_region
+
+    def initialize
+      @provider = :openai
+      @openai_api_key = ENV["OPENAI_API_KEY"]
+      @aws_region = "us-east-1"
+    end
+  end
+end

data/lib/sentiment_insights/insights/entities.rb

@@ -0,0 +1,66 @@
+module SentimentInsights
+  module Insights
+    # Extracts and summarizes named entities from survey responses
+    class Entities
+      def initialize(provider: nil, provider_client: nil)
+        effective_provider = provider || SentimentInsights.configuration&.provider || :sentimental
+
+        @provider_client = provider_client || case effective_provider
+                                              when :openai
+                                                require_relative '../clients/entities/open_ai_client'
+                                                Clients::Entities::OpenAIClient.new
+                                              when :aws
+                                                require_relative '../clients/entities/aws_client'
+                                                Clients::Entities::AwsClient.new
+                                              when :sentimental
+                                                raise NotImplementedError, "Entity recognition is not supported for the 'sentimental' provider"
+                                              else
+                                                raise ArgumentError, "Unsupported provider: #{effective_provider}"
+                                              end
+      end
+
+      # Extract named entities and build summarized output
+      # @param entries [Array<Hash>] each with :answer and optional :segment
+      # @return [Hash] { entities: [...], responses: [...] }
+      def extract(entries, question: nil)
+        entries = entries.to_a
+        raw_result = @provider_client.extract_batch(entries, question: question)
+
+        puts "raw_result = #{raw_result}"
+        responses = raw_result[:responses] || []
+        entities  = raw_result[:entities] || []
+
+        # Index responses by ID
+        response_index = responses.each_with_object({}) { |r, h| h[r[:id]] = r }
+
+        enriched_entities = entities.map do |entity_entry|
+          mentions = entity_entry[:mentions] || []
+          mention_responses = mentions.map { |id| response_index[id] }.compact
+
+          segment_dist = Hash.new { |h, k| h[k] = Hash.new(0) }
+
+          mention_responses.each do |resp|
+            (resp[:segment] || {}).each do |seg_key, seg_val|
+              segment_dist[seg_key][seg_val] += 1
+            end
+          end
+
+          {
+            entity: entity_entry[:entity],
+            type: entity_entry[:type],
+            mentions: mentions,
+            summary: {
+              total_mentions: mentions.size,
+              segment_distribution: segment_dist
+            }
+          }
+        end
+
+        {
+          entities: enriched_entities,
+          responses: responses
+        }
+      end
+    end
+  end
+end