ragdoll 0.1.1 → 0.1.8

data/app/services/ragdoll/text_chunker.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class TextChunker
+    DEFAULT_CHUNK_SIZE = 1000
+    DEFAULT_CHUNK_OVERLAP = 200
+
+    def self.chunk(text, chunk_size: DEFAULT_CHUNK_SIZE, chunk_overlap: DEFAULT_CHUNK_OVERLAP)
+      new(text, chunk_size: chunk_size, chunk_overlap: chunk_overlap).chunk
+    end
+
+    def initialize(text, chunk_size: DEFAULT_CHUNK_SIZE, chunk_overlap: DEFAULT_CHUNK_OVERLAP)
+      @text = text.to_s
+      @chunk_size = chunk_size
+      @chunk_overlap = chunk_overlap
+    end
+
+    def chunk
+      return [] if @text.empty?
+
+      # Ensure chunk_size and chunk_overlap are valid integers
+      @chunk_size = (@chunk_size || DEFAULT_CHUNK_SIZE).to_i
+      @chunk_overlap = (@chunk_overlap || DEFAULT_CHUNK_OVERLAP).to_i
+
+      # Ensure chunk_overlap is not greater than or equal to chunk_size to prevent infinite loops
+      @chunk_overlap = [@chunk_size - 1, 0].max if @chunk_overlap >= @chunk_size
+
+      return [@text] if @text.length <= @chunk_size
+
+      chunks = []
+      start_pos = 0
+
+      while start_pos < @text.length
+        end_pos = start_pos + @chunk_size
+
+        # If this is the last chunk, take everything remaining
+        if end_pos >= @text.length
+          chunks << @text[start_pos..].strip
+          break
+        end
+
+        # Try to find a good breaking point (sentence, paragraph, or word boundary)
+        chunk_text = @text[start_pos...end_pos]
+        break_pos = find_break_position(chunk_text, @text, start_pos, end_pos)
+
+        # Extract the chunk
+        actual_end_pos = start_pos + break_pos
+        chunk_content = @text[start_pos...actual_end_pos].strip
+
+        chunks << chunk_content unless chunk_content.empty?
+
+        # Move to next chunk with overlap
+        next_start_pos = actual_end_pos - @chunk_overlap
+        next_start_pos = [next_start_pos, 0].max # Ensure we don't go negative
+
+        # Ensure forward progress - if we're not advancing, force a step forward
+        next_start_pos = start_pos + 1 if next_start_pos <= start_pos
+
+        start_pos = next_start_pos
+      end
+
+      chunks.reject(&:empty?)
+    end
+
+    private
+
+    def find_break_position(chunk_text, _full_text, _start_pos, _end_pos)
+      # Priority order for breaking points:
+      # 1. Double newline (paragraph break)
+      # 2. Single newline + sentence ending
+      # 3. Sentence ending punctuation
+      # 4. Word boundary
+      # 5. Character boundary (fallback)
+
+      # Look for paragraph breaks
+      paragraph_break = chunk_text.rindex("\n\n")
+      return paragraph_break + 2 if paragraph_break && paragraph_break > @chunk_size * 0.5
+
+      # Look for sentence endings near newlines
+      sentence_patterns = [
+        /[.!?]\s*\n/,
+        /[.!?]\s+[A-Z]/,
+        /[.!?]$/
+      ]
+
+      sentence_patterns.each do |pattern|
+        matches = chunk_text.enum_for(:scan, pattern).map { Regexp.last_match.end(0) }
+        next unless matches.any?
+
+        # Find the best sentence break (closest to chunk_size but not too small)
+        best_break = matches.select { |pos| pos > @chunk_size * 0.5 }.max
+        return best_break if best_break
+      end
+
+      # Look for word boundaries
+      word_break = chunk_text.rindex(/\s/)
+      return word_break + 1 if word_break && word_break > @chunk_size * 0.3
+
+      # Fallback to character boundary
+      @chunk_size
+    end
+
+    # Alternative chunking method for structured documents
+    def self.chunk_by_structure(text, max_chunk_size: DEFAULT_CHUNK_SIZE)
+      chunks = []
+      current_chunk = ""
+
+      # Split by paragraphs first
+      paragraphs = text.split(/\n\s*\n/)
+
+      paragraphs.each do |paragraph|
+        paragraph = paragraph.strip
+        next if paragraph.empty?
+
+        # If adding this paragraph would exceed chunk size, start new chunk
+        if !current_chunk.empty? && (current_chunk.length + paragraph.length + 2) > max_chunk_size
+          chunks << current_chunk.strip
+          current_chunk = ""
+        end
+
+        # If single paragraph is too large, split it
+        if paragraph.length > max_chunk_size
+          # Split large paragraph into sentences
+          sentences = paragraph.split(/(?<=[.!?])\s+/)
+
+          sentences.each do |sentence|
+            sentence = sentence.strip
+            next if sentence.empty?
+
+            if !current_chunk.empty? && (current_chunk.length + sentence.length + 1) > max_chunk_size
+              chunks << current_chunk.strip
+              current_chunk = ""
+            end
+
+            if sentence.length > max_chunk_size
+              # Split very long sentences by words
+              words = sentence.split(/\s+/)
+              words.each do |word|
+                if !current_chunk.empty? && (current_chunk.length + word.length + 1) > max_chunk_size
+                  chunks << current_chunk.strip
+                  current_chunk = ""
+                end
+                current_chunk += (current_chunk.empty? ? "" : " ") + word
+              end
+            else
+              current_chunk += (current_chunk.empty? ? "" : " ") + sentence
+            end
+          end
+        else
+          current_chunk += (current_chunk.empty? ? "" : "\n\n") + paragraph
+        end
+      end
+
+      chunks << current_chunk.strip unless current_chunk.strip.empty?
+      chunks.reject(&:empty?)
+    end
+
+    # Specialized chunking for code documents
+    def self.chunk_code(text, max_chunk_size: DEFAULT_CHUNK_SIZE)
+      chunks = []
+      current_chunk = ""
+
+      # Split by functions, classes, or logical blocks
+      lines = text.split("\n")
+      current_block = []
+      block_indent = nil
+
+      lines.each do |line|
+        line_indent = line[/^\s*/].length
+
+        # Detect block boundaries (functions, classes, etc.)
+        if line.match?(/^\s*(def|class|function|const|let|var)\s/) ||
+           (block_indent && line_indent <= block_indent && !line.strip.empty?)
+
+          # Process current block
+          if current_block.any?
+            block_text = current_block.join("\n")
+
+            if !current_chunk.empty? && (current_chunk.length + block_text.length + 1) > max_chunk_size
+              chunks << current_chunk.strip
+              current_chunk = ""
+            end
+
+            current_chunk += (current_chunk.empty? ? "" : "\n") + block_text
+          end
+
+          current_block = [line]
+          block_indent = line_indent
+        else
+          current_block << line
+        end
+      end
+
+      # Process final block
+      if current_block.any?
+        block_text = current_block.join("\n")
+        if !current_chunk.empty? && (current_chunk.length + block_text.length + 1) > max_chunk_size
+          chunks << current_chunk.strip
+          current_chunk = ""
+        end
+        current_chunk += (current_chunk.empty? ? "" : "\n") + block_text
+      end
+
+      chunks << current_chunk.strip unless current_chunk.strip.empty?
+      chunks.reject(&:empty?)
+    end
+  end
+end

data/app/services/ragdoll/text_generation_service.rb

@@ -0,0 +1,355 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Ragdoll
+  class TextGenerationService
+    class GenerationError < StandardError; end
+
+    def initialize(client: nil, config_service: nil, model_resolver: nil)
+      @config_service = config_service || Ragdoll::ConfigurationService.new
+      @model_resolver = model_resolver || Ragdoll::ModelResolver.new(@config_service)
+      @client = client
+      configure_ruby_llm_if_possible unless @client
+    end
+
+    def generate_summary(text, max_length: nil)
+      return "" if text.nil? || text.strip.empty?
+
+      # Skip summarization if not enabled
+      unless @config_service.config.summarization[:enable]
+        puts "⚠️  LLM summarization disabled, using fallback (first 500 chars)"
+        return text[0..500]
+      end
+
+      # Skip if content is too short
+      min_length = @config_service.config.summarization[:min_content_length]
+      return text if text.length < min_length
+
+      max_length ||= @config_service.config.summarization[:max_length]
+
+      # Clean and prepare text
+      cleaned_text = clean_text(text)
+
+      # Create summarization prompt
+      prompt = build_summary_prompt(cleaned_text, max_length)
+
+      begin
+        if @client == :ruby_llm_configured
+          # Use RubyLLM for text generation
+          # Use model resolver to get summary model with inheritance
+          model_obj = @model_resolver.resolve_for_task(:summary)
+          model = model_obj.model
+
+          chat = RubyLLM.chat.with_model(model)
+                        .with_temperature(0.3)
+          chat.add_message(role: "user", content: prompt)
+          response = chat.complete
+
+          if response.respond_to?(:content)
+            response.content.strip
+          elsif response.respond_to?(:message) && response.message.respond_to?(:content)
+            response.message.content.strip
+          elsif response && response["choices"]&.first
+            response["choices"].first["message"]["content"].strip
+          elsif response && response["content"]
+            response["content"].strip
+          else
+            raise GenerationError, "Invalid response format from text generation API"
+          end
+        elsif @client
+          # Use custom client for testing
+          model_obj = @model_resolver.resolve_for_task(:summary)
+          model = model_obj.model
+
+          response = @client.chat(
+            model: model,
+            messages: [
+              { role: "user", content: prompt }
+            ],
+            max_tokens: max_length + 50,
+            temperature: 0.3
+          )
+
+          if response && response["choices"]&.first
+            response["choices"].first["message"]["content"].strip
+          elsif response && response["content"]
+            response["content"].strip
+          else
+            raise GenerationError, "Invalid response format from text generation API"
+          end
+        else
+          # Fallback to basic summarization for testing/dev environments
+          puts "⚠️  No LLM client configured, using fallback summarization"
+          generate_basic_summary(cleaned_text, max_length)
+        end
+      rescue StandardError => e
+        # Fall back to basic summarization if API fails
+        puts "❌ LLM summary generation failed, using fallback: #{e.message}"
+        generate_basic_summary(cleaned_text, max_length)
+      end
+    end
+
+    def extract_keywords(text, max_keywords: 20)
+      return [] if text.nil? || text.strip.empty?
+
+      # Clean and prepare text
+      cleaned_text = clean_text(text)
+
+      # Create keyword extraction prompt
+      prompt = build_keyword_prompt(cleaned_text, max_keywords)
+
+      begin
+        if @client == :ruby_llm_configured
+          # Use RubyLLM for keyword extraction
+          # Use keywords model from models config, fallback to default
+          model_obj = @model_resolver.resolve_for_task(:keywords)
+          model = model_obj.model
+
+          chat = RubyLLM.chat.with_model(model).with_temperature(0.1)
+          chat.add_message(role: "user", content: prompt)
+          response = chat.complete
+
+          if response.respond_to?(:content)
+            content = response.content.strip
+            parse_keywords_response(content)
+          elsif response.respond_to?(:message) && response.message.respond_to?(:content)
+            content = response.message.content.strip
+            parse_keywords_response(content)
+          elsif response && response["choices"]&.first
+            content = response["choices"].first["message"]["content"].strip
+            parse_keywords_response(content)
+          elsif response && response["content"]
+            content = response["content"].strip
+            parse_keywords_response(content)
+          else
+            raise GenerationError, "Invalid response format from text generation API"
+          end
+        elsif @client
+          # Use custom client for testing
+          model_obj = @model_resolver.resolve_for_task(:keywords)
+          model = model_obj.model
+
+          response = @client.chat(
+            model: model,
+            messages: [
+              { role: "user", content: prompt }
+            ],
+            max_tokens: 200,
+            temperature: 0.1
+          )
+
+          if response && response["choices"]&.first
+            content = response["choices"].first["message"]["content"].strip
+            parse_keywords_response(content)
+          elsif response && response["content"]
+            content = response["content"].strip
+            parse_keywords_response(content)
+          else
+            raise GenerationError, "Invalid response format from text generation API"
+          end
+        else
+          # Fallback to basic keyword extraction for testing/dev environments
+          puts "⚠️  No LLM client configured, using fallback keyword extraction"
+          extract_basic_keywords(cleaned_text, max_keywords)
+        end
+      rescue StandardError => e
+        # Fall back to basic keyword extraction if API fails
+        puts "❌ LLM keyword extraction failed, using fallback: #{e.message}"
+        puts "Error class: #{e.class}"
+        puts "Backtrace: #{e.backtrace.first(3).join(', ')}"
+        extract_basic_keywords(cleaned_text, max_keywords)
+      end
+    end
+
+    private
+
+    def configure_ruby_llm_if_possible
+      # Only configure if we have valid configuration
+      # Use embedding provider, fallback to :openai
+      provider = @config_service.config.models[:embedding][:provider] || :openai
+      config = @config_service.config.llm_providers[provider] || {}
+
+      # Check if we have the necessary API key for the provider
+      has_api_key = case provider
+                    when :openai
+                      config[:api_key] && !config[:api_key].empty?
+                    when :anthropic
+                      config[:api_key] && !config[:api_key].empty?
+                    when :google
+                      config[:api_key] && !config[:api_key].empty?
+                    when :azure
+                      config[:api_key] && !config[:api_key].empty?
+                    when :ollama
+                      true # Ollama doesn't need API key for local setup
+                    when :huggingface
+                      config[:api_key] && !config[:api_key].empty?
+                    when :openrouter
+                      config[:api_key] && !config[:api_key].empty?
+                    else
+                      false
+                    end
+
+      return unless has_api_key
+
+      begin
+        RubyLLM.configure do |ruby_llm_config|
+          case provider
+          when :openai
+            ruby_llm_config.openai_api_key = config[:api_key]
+            ruby_llm_config.openai_organization = config[:organization] if config[:organization]
+            ruby_llm_config.openai_project = config[:project] if config[:project]
+          when :anthropic
+            ruby_llm_config.anthropic_api_key = config[:api_key]
+          when :google
+            ruby_llm_config.google_api_key = config[:api_key]
+            ruby_llm_config.google_project_id = config[:project_id] if config[:project_id]
+          when :azure
+            ruby_llm_config.azure_api_key = config[:api_key]
+            ruby_llm_config.azure_endpoint = config[:endpoint] if config[:endpoint]
+            ruby_llm_config.azure_api_version = config[:api_version] if config[:api_version]
+          when :ollama
+            ruby_llm_config.ollama_endpoint = config[:endpoint] if config[:endpoint]
+          when :huggingface
+            ruby_llm_config.huggingface_api_key = config[:api_key]
+          when :openrouter
+            ruby_llm_config.openrouter_api_key = config[:api_key]
+          end
+        end
+
+        # RubyLLM uses module-level methods, not individual provider classes
+        @client = :ruby_llm_configured
+      rescue StandardError => e
+        # If configuration fails, don't set client (will use fallback)
+        puts "❌ RubyLLM configuration failed: #{e.message}"
+        puts "   Will use fallback text processing for summaries and keywords"
+      end
+    end
+
+    def clean_text(text)
+      return "" if text.nil?
+
+      # Remove excessive whitespace and normalize
+      cleaned = text.strip
+                    .gsub(/\s+/, " ")              # Multiple spaces to single space
+                    .gsub(/\n+/, "\n")             # Multiple newlines to single newline
+                    .gsub(/\t+/, " ")              # Tabs to spaces
+
+      # Truncate if too long (most models have token limits)
+      max_chars = 12_000 # Conservative limit for most language models
+      cleaned.length > max_chars ? cleaned[0, max_chars] : cleaned
+    end
+
+    def build_summary_prompt(text, max_length)
+      <<~PROMPT
+        Please provide a concise summary of the following text. The summary should:
+        - Be approximately #{max_length} characters or less
+        - Capture the main topics and key points
+        - Be written in clear, professional language
+        - Focus on the most important information
+
+        Text to summarize:
+        #{text}
+
+        Summary:
+      PROMPT
+    end
+
+    def build_keyword_prompt(text, max_keywords)
+      <<~PROMPT
+        Please extract the most important keywords and key phrases from the following text.#{' '}
+        Provide up to #{max_keywords} keywords that best represent the content.
+
+        Requirements:
+        - Focus on nouns, important concepts, and technical terms
+        - Avoid common stop words and articles
+        - Include both single words and meaningful phrases
+        - Separate keywords with commas
+        - Order by importance (most important first)
+
+        Text to analyze:
+        #{text}
+
+        Keywords (comma-separated):
+      PROMPT
+    end
+
+    def parse_keywords_response(content)
+      # Extract keywords from the response, handling various formats
+      content
+        .gsub(/^(keywords?:?\s*)/i, "") # Remove "Keywords:" prefix
+        .split(/[,\n]/)                 # Split by commas or newlines
+        .map(&:strip)                   # Remove whitespace
+        .reject(&:empty?)               # Remove empty strings
+        .reject { |k| k.match?(/^\d+\./) } # Remove numbered list items
+        .map { |k| k.gsub(/^\d+\.\s*/, "") } # Remove numbering from start
+        .reject { |k| k.length < 2 }    # Remove very short words
+        .first(20)                      # Limit to 20 keywords
+    end
+
+    def generate_basic_summary(text, max_length)
+      # Fallback summarization method (same as before)
+      clean_text = text.gsub(/\s+/, " ").strip
+
+      # Split into sentences
+      sentences = clean_text.split(/[.!?]+/).map(&:strip).reject(&:empty?)
+
+      # If content is short, use the whole thing
+      return clean_text if clean_text.length <= max_length
+
+      # Take first 2-3 sentences or up to max_length characters
+      summary_sentences = []
+      total_length = 0
+
+      sentences.each do |sentence|
+        break unless total_length + sentence.length <= max_length && summary_sentences.length < 3
+
+        summary_sentences << sentence
+        total_length += sentence.length
+      end
+
+      summary = summary_sentences.join(". ")
+      summary += "." unless summary.end_with?(".", "!", "?")
+      summary
+    end
+
+    def extract_basic_keywords(text, max_keywords)
+      # Fallback keyword extraction method (same as before)
+      stop_words = %w[
+        a an and are as at be by for from has he in is it its of on that the
+        to was will with the this these those they them their there where when
+        what who why how which would could should shall might may can must
+        do does did done doing go goes went gone going get gets got gotten
+        getting have had having has been being am was were are is was been
+        but or not no yes if then else also too very much many most some any
+        all each every each other another one two three first second third
+        last next previous before after during while until since through
+        above below under over between among within without across around
+        near far close distant here there everywhere nowhere somewhere anywhere
+      ]
+
+      # Clean and normalize text
+      cleaned_text = text.downcase
+                         .gsub(/[^\w\s]/, " ") # Remove punctuation
+                         .gsub(/\s+/, " ") # Normalize whitespace
+                         .strip
+
+      # Split into words and filter
+      words = cleaned_text.split(" ")
+                          .reject { |word| word.length < 3 } # Remove short words
+                          .reject { |word| stop_words.include?(word) } # Remove stop words
+                          .reject { |word| word.match?(/^\d+$/) } # Remove pure numbers
+
+      # Count word frequencies
+      word_counts = Hash.new(0)
+      words.each { |word| word_counts[word] += 1 }
+
+      # Get top keywords (words that appear more than once or are significant)
+      word_counts
+        .select { |word, count| count > 1 || word.length > 6 }
+        .sort_by { |word, count| [-count, word] }
+        .first(max_keywords) # Limit to max_keywords
+        .map { |word, _count| word }
+    end
+  end
+end