ragdoll 0.1.0 → 0.1.1

data/lib/ragdoll/core/text_chunker.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    class TextChunker
+      DEFAULT_CHUNK_SIZE = 1000
+      DEFAULT_CHUNK_OVERLAP = 200
+
+      def self.chunk(text, chunk_size: DEFAULT_CHUNK_SIZE, chunk_overlap: DEFAULT_CHUNK_OVERLAP)
+        new(text, chunk_size: chunk_size, chunk_overlap: chunk_overlap).chunk
+      end
+
+      def initialize(text, chunk_size: DEFAULT_CHUNK_SIZE, chunk_overlap: DEFAULT_CHUNK_OVERLAP)
+        @text = text.to_s
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+      end
+
+      def chunk
+        return [] if @text.empty?
+
+        # Ensure chunk_size and chunk_overlap are valid integers
+        @chunk_size = (@chunk_size || DEFAULT_CHUNK_SIZE).to_i
+        @chunk_overlap = (@chunk_overlap || DEFAULT_CHUNK_OVERLAP).to_i
+
+        # Ensure chunk_overlap is not greater than or equal to chunk_size to prevent infinite loops
+        @chunk_overlap = [@chunk_size - 1, 0].max if @chunk_overlap >= @chunk_size
+
+        return [@text] if @text.length <= @chunk_size
+
+        chunks = []
+        start_pos = 0
+
+        while start_pos < @text.length
+          end_pos = start_pos + @chunk_size
+
+          # If this is the last chunk, take everything remaining
+          if end_pos >= @text.length
+            chunks << @text[start_pos..].strip
+            break
+          end
+
+          # Try to find a good breaking point (sentence, paragraph, or word boundary)
+          chunk_text = @text[start_pos...end_pos]
+          break_pos = find_break_position(chunk_text, @text, start_pos, end_pos)
+
+          # Extract the chunk
+          actual_end_pos = start_pos + break_pos
+          chunk_content = @text[start_pos...actual_end_pos].strip
+
+          chunks << chunk_content unless chunk_content.empty?
+
+          # Move to next chunk with overlap
+          next_start_pos = actual_end_pos - @chunk_overlap
+          next_start_pos = [next_start_pos, 0].max # Ensure we don't go negative
+
+          # Ensure forward progress - if we're not advancing, force a step forward
+          next_start_pos = start_pos + 1 if next_start_pos <= start_pos
+
+          start_pos = next_start_pos
+        end
+
+        chunks.reject(&:empty?)
+      end
+
+      private
+
+      def find_break_position(chunk_text, _full_text, _start_pos, _end_pos)
+        # Priority order for breaking points:
+        # 1. Double newline (paragraph break)
+        # 2. Single newline + sentence ending
+        # 3. Sentence ending punctuation
+        # 4. Word boundary
+        # 5. Character boundary (fallback)
+
+        # Look for paragraph breaks
+        paragraph_break = chunk_text.rindex("\n\n")
+        return paragraph_break + 2 if paragraph_break && paragraph_break > @chunk_size * 0.5
+
+        # Look for sentence endings near newlines
+        sentence_patterns = [
+          /[.!?]\s*\n/,
+          /[.!?]\s+[A-Z]/,
+          /[.!?]$/
+        ]
+
+        sentence_patterns.each do |pattern|
+          matches = chunk_text.enum_for(:scan, pattern).map { Regexp.last_match.end(0) }
+          next unless matches.any?
+
+          # Find the best sentence break (closest to chunk_size but not too small)
+          best_break = matches.select { |pos| pos > @chunk_size * 0.5 }.max
+          return best_break if best_break
+        end
+
+        # Look for word boundaries
+        word_break = chunk_text.rindex(/\s/)
+        return word_break + 1 if word_break && word_break > @chunk_size * 0.3
+
+        # Fallback to character boundary
+        @chunk_size
+      end
+
+      # Alternative chunking method for structured documents
+      def self.chunk_by_structure(text, max_chunk_size: DEFAULT_CHUNK_SIZE)
+        chunks = []
+        current_chunk = ""
+
+        # Split by paragraphs first
+        paragraphs = text.split(/\n\s*\n/)
+
+        paragraphs.each do |paragraph|
+          paragraph = paragraph.strip
+          next if paragraph.empty?
+
+          # If adding this paragraph would exceed chunk size, start new chunk
+          if !current_chunk.empty? && (current_chunk.length + paragraph.length + 2) > max_chunk_size
+            chunks << current_chunk.strip
+            current_chunk = ""
+          end
+
+          # If single paragraph is too large, split it
+          if paragraph.length > max_chunk_size
+            # Split large paragraph into sentences
+            sentences = paragraph.split(/(?<=[.!?])\s+/)
+
+            sentences.each do |sentence|
+              sentence = sentence.strip
+              next if sentence.empty?
+
+              if !current_chunk.empty? && (current_chunk.length + sentence.length + 1) > max_chunk_size
+                chunks << current_chunk.strip
+                current_chunk = ""
+              end
+
+              if sentence.length > max_chunk_size
+                # Split very long sentences by words
+                words = sentence.split(/\s+/)
+                words.each do |word|
+                  if !current_chunk.empty? && (current_chunk.length + word.length + 1) > max_chunk_size
+                    chunks << current_chunk.strip
+                    current_chunk = ""
+                  end
+                  current_chunk += (current_chunk.empty? ? "" : " ") + word
+                end
+              else
+                current_chunk += (current_chunk.empty? ? "" : " ") + sentence
+              end
+            end
+          else
+            current_chunk += (current_chunk.empty? ? "" : "\n\n") + paragraph
+          end
+        end
+
+        chunks << current_chunk.strip unless current_chunk.strip.empty?
+        chunks.reject(&:empty?)
+      end
+
+      # Specialized chunking for code documents
+      def self.chunk_code(text, max_chunk_size: DEFAULT_CHUNK_SIZE)
+        chunks = []
+        current_chunk = ""
+
+        # Split by functions, classes, or logical blocks
+        lines = text.split("\n")
+        current_block = []
+        block_indent = nil
+
+        lines.each do |line|
+          line_indent = line[/^\s*/].length
+
+          # Detect block boundaries (functions, classes, etc.)
+          if line.match?(/^\s*(def|class|function|const|let|var)\s/) ||
+             (block_indent && line_indent <= block_indent && !line.strip.empty?)
+
+            # Process current block
+            if current_block.any?
+              block_text = current_block.join("\n")
+
+              if !current_chunk.empty? && (current_chunk.length + block_text.length + 1) > max_chunk_size
+                chunks << current_chunk.strip
+                current_chunk = ""
+              end
+
+              current_chunk += (current_chunk.empty? ? "" : "\n") + block_text
+            end
+
+            current_block = [line]
+            block_indent = line_indent
+          else
+            current_block << line
+          end
+        end
+
+        # Process final block
+        if current_block.any?
+          block_text = current_block.join("\n")
+          if !current_chunk.empty? && (current_chunk.length + block_text.length + 1) > max_chunk_size
+            chunks << current_chunk.strip
+            current_chunk = ""
+          end
+          current_chunk += (current_chunk.empty? ? "" : "\n") + block_text
+        end
+
+        chunks << current_chunk.strip unless current_chunk.strip.empty?
+        chunks.reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/ragdoll/core/text_generation_service.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Ragdoll
+  module Core
+    class TextGenerationService
+      class GenerationError < StandardError; end
+
+      def initialize(client: nil)
+        @configuration = Ragdoll.config
+        @client = client
+        configure_ruby_llm_if_possible unless @client
+      end
+
+      def generate_summary(text, max_length: nil)
+        return "" if text.nil? || text.strip.empty?
+
+        # Skip summarization if not enabled
+        unless @configuration.summarization_config[:enable]
+          puts "⚠️  LLM summarization disabled, using fallback (first 500 chars)"
+          return text[0..500]
+        end
+
+        # Skip if content is too short
+        min_length = @configuration.summarization_config[:min_content_length]
+        return text if text.length < min_length
+
+        max_length ||= @configuration.summarization_config[:max_length]
+
+        # Clean and prepare text
+        cleaned_text = clean_text(text)
+
+        # Create summarization prompt
+        prompt = build_summary_prompt(cleaned_text, max_length)
+
+        begin
+          if @client == :ruby_llm_configured
+            # Use RubyLLM for text generation
+            # Use summary model from models config, fallback to default
+            model_string = @configuration.models[:summary] || @configuration.models[:default]
+            
+            # Parse provider/model - use only the model part for RubyLLM
+            parsed = @configuration.parse_provider_model(model_string)
+            model = parsed[:model] || model_string
+            
+            chat = RubyLLM.chat.with_model(model)
+                          .with_temperature(0.3)
+            chat.add_message(role: "user", content: prompt)
+            response = chat.complete
+
+            if response.respond_to?(:content)
+              response.content.strip
+            elsif response.respond_to?(:message) && response.message.respond_to?(:content)
+              response.message.content.strip
+            elsif response && response["choices"]&.first
+              response["choices"].first["message"]["content"].strip
+            elsif response && response["content"]
+              response["content"].strip
+            else
+              raise GenerationError, "Invalid response format from text generation API"
+            end
+          elsif @client
+            # Use custom client for testing
+            model = @configuration.models[:summary] || @configuration.models[:default]
+            
+            response = @client.chat(
+              model: model,
+              messages: [
+                { role: "user", content: prompt }
+              ],
+              max_tokens: max_length + 50,
+              temperature: 0.3
+            )
+
+            if response && response["choices"]&.first
+              response["choices"].first["message"]["content"].strip
+            elsif response && response["content"]
+              response["content"].strip
+            else
+              raise GenerationError, "Invalid response format from text generation API"
+            end
+          else
+            # Fallback to basic summarization for testing/dev environments
+            puts "⚠️  No LLM client configured, using fallback summarization"
+            generate_basic_summary(cleaned_text, max_length)
+          end
+        rescue StandardError => e
+          # Fall back to basic summarization if API fails
+          puts "❌ LLM summary generation failed, using fallback: #{e.message}"
+          generate_basic_summary(cleaned_text, max_length)
+        end
+      end
+
+      def extract_keywords(text, max_keywords: 20)
+        return [] if text.nil? || text.strip.empty?
+
+        # Clean and prepare text
+        cleaned_text = clean_text(text)
+
+        # Create keyword extraction prompt
+        prompt = build_keyword_prompt(cleaned_text, max_keywords)
+
+        begin
+          if @client == :ruby_llm_configured
+            # Use RubyLLM for keyword extraction
+            # Use keywords model from models config, fallback to default
+            model_string = @configuration.models[:keywords] || @configuration.models[:default]
+            
+            # Parse provider/model - use only the model part for RubyLLM
+            parsed = @configuration.parse_provider_model(model_string)
+            model = parsed[:model] || model_string
+            
+            chat = RubyLLM.chat.with_model(model).with_temperature(0.1)
+            chat.add_message(role: "user", content: prompt)
+            response = chat.complete
+
+            if response.respond_to?(:content)
+              content = response.content.strip
+              parse_keywords_response(content)
+            elsif response.respond_to?(:message) && response.message.respond_to?(:content)
+              content = response.message.content.strip
+              parse_keywords_response(content)
+            elsif response && response["choices"]&.first
+              content = response["choices"].first["message"]["content"].strip
+              parse_keywords_response(content)
+            elsif response && response["content"]
+              content = response["content"].strip
+              parse_keywords_response(content)
+            else
+              raise GenerationError, "Invalid response format from text generation API"
+            end
+          elsif @client
+            # Use custom client for testing  
+            model = @configuration.models[:keywords] || @configuration.models[:default]
+            
+            response = @client.chat(
+              model: model,
+              messages: [
+                { role: "user", content: prompt }
+              ],
+              max_tokens: 200,
+              temperature: 0.1
+            )
+
+            if response && response["choices"]&.first
+              content = response["choices"].first["message"]["content"].strip
+              parse_keywords_response(content)
+            elsif response && response["content"]
+              content = response["content"].strip
+              parse_keywords_response(content)
+            else
+              raise GenerationError, "Invalid response format from text generation API"
+            end
+          else
+            # Fallback to basic keyword extraction for testing/dev environments
+            puts "⚠️  No LLM client configured, using fallback keyword extraction"
+            extract_basic_keywords(cleaned_text, max_keywords)
+          end
+        rescue StandardError => e
+          # Fall back to basic keyword extraction if API fails
+          puts "❌ LLM keyword extraction failed, using fallback: #{e.message}"
+          puts "Error class: #{e.class}"
+          puts "Backtrace: #{e.backtrace.first(3).join(', ')}"
+          extract_basic_keywords(cleaned_text, max_keywords)
+        end
+      end
+
+      private
+
+      def configure_ruby_llm_if_possible
+        # Only configure if we have valid configuration
+        # Use embedding_config provider, fallback to :openai
+        provider = @configuration.embedding_config[:provider] || :openai
+        config = @configuration.ruby_llm_config[provider] || {}
+
+        # Check if we have the necessary API key for the provider
+        has_api_key = case provider
+                      when :openai
+                        config[:api_key] && !config[:api_key].empty?
+                      when :anthropic
+                        config[:api_key] && !config[:api_key].empty?
+                      when :google
+                        config[:api_key] && !config[:api_key].empty?
+                      when :azure
+                        config[:api_key] && !config[:api_key].empty?
+                      when :ollama
+                        true # Ollama doesn't need API key for local setup
+                      when :huggingface
+                        config[:api_key] && !config[:api_key].empty?
+                      when :openrouter
+                        config[:api_key] && !config[:api_key].empty?
+                      else
+                        false
+                      end
+
+        return unless has_api_key
+
+        begin
+          RubyLLM.configure do |ruby_llm_config|
+            case provider
+            when :openai
+              ruby_llm_config.openai_api_key = config[:api_key]
+              ruby_llm_config.openai_organization = config[:organization] if config[:organization]
+              ruby_llm_config.openai_project = config[:project] if config[:project]
+            when :anthropic
+              ruby_llm_config.anthropic_api_key = config[:api_key]
+            when :google
+              ruby_llm_config.google_api_key = config[:api_key]
+              ruby_llm_config.google_project_id = config[:project_id] if config[:project_id]
+            when :azure
+              ruby_llm_config.azure_api_key = config[:api_key]
+              ruby_llm_config.azure_endpoint = config[:endpoint] if config[:endpoint]
+              ruby_llm_config.azure_api_version = config[:api_version] if config[:api_version]
+            when :ollama
+              ruby_llm_config.ollama_endpoint = config[:endpoint] if config[:endpoint]
+            when :huggingface
+              ruby_llm_config.huggingface_api_key = config[:api_key]
+            when :openrouter
+              ruby_llm_config.openrouter_api_key = config[:api_key]
+            end
+          end
+
+          # RubyLLM uses module-level methods, not individual provider classes
+          @client = :ruby_llm_configured
+        rescue StandardError => e
+          # If configuration fails, don't set client (will use fallback)
+          puts "❌ RubyLLM configuration failed: #{e.message}"
+          puts "   Will use fallback text processing for summaries and keywords"
+        end
+      end
+
+      def clean_text(text)
+        return "" if text.nil?
+
+        # Remove excessive whitespace and normalize
+        cleaned = text.strip
+                      .gsub(/\s+/, " ")              # Multiple spaces to single space
+                      .gsub(/\n+/, "\n")             # Multiple newlines to single newline
+                      .gsub(/\t+/, " ")              # Tabs to spaces
+
+        # Truncate if too long (most models have token limits)
+        max_chars = 12_000 # Conservative limit for most language models
+        cleaned.length > max_chars ? cleaned[0, max_chars] : cleaned
+      end
+
+      def build_summary_prompt(text, max_length)
+        <<~PROMPT
+          Please provide a concise summary of the following text. The summary should:
+          - Be approximately #{max_length} characters or less
+          - Capture the main topics and key points
+          - Be written in clear, professional language
+          - Focus on the most important information
+
+          Text to summarize:
+          #{text}
+
+          Summary:
+        PROMPT
+      end
+
+      def build_keyword_prompt(text, max_keywords)
+        <<~PROMPT
+          Please extract the most important keywords and key phrases from the following text.#{' '}
+          Provide up to #{max_keywords} keywords that best represent the content.
+
+          Requirements:
+          - Focus on nouns, important concepts, and technical terms
+          - Avoid common stop words and articles
+          - Include both single words and meaningful phrases
+          - Separate keywords with commas
+          - Order by importance (most important first)
+
+          Text to analyze:
+          #{text}
+
+          Keywords (comma-separated):
+        PROMPT
+      end
+
+      def parse_keywords_response(content)
+        # Extract keywords from the response, handling various formats
+        content
+          .gsub(/^(keywords?:?\s*)/i, "") # Remove "Keywords:" prefix
+          .split(/[,\n]/)                 # Split by commas or newlines
+          .map(&:strip)                   # Remove whitespace
+          .reject(&:empty?)               # Remove empty strings
+          .reject { |k| k.match?(/^\d+\./) } # Remove numbered list items
+          .map { |k| k.gsub(/^\d+\.\s*/, "") } # Remove numbering from start
+          .reject { |k| k.length < 2 }    # Remove very short words
+          .first(20)                      # Limit to 20 keywords
+      end
+
+      def generate_basic_summary(text, max_length)
+        # Fallback summarization method (same as before)
+        clean_text = text.gsub(/\s+/, " ").strip
+
+        # Split into sentences
+        sentences = clean_text.split(/[.!?]+/).map(&:strip).reject(&:empty?)
+
+        # If content is short, use the whole thing
+        return clean_text if clean_text.length <= max_length
+
+        # Take first 2-3 sentences or up to max_length characters
+        summary_sentences = []
+        total_length = 0
+
+        sentences.each do |sentence|
+          break unless total_length + sentence.length <= max_length && summary_sentences.length < 3
+
+          summary_sentences << sentence
+          total_length += sentence.length
+        end
+
+        summary = summary_sentences.join(". ")
+        summary += "." unless summary.end_with?(".", "!", "?")
+        summary
+      end
+
+      def extract_basic_keywords(text, max_keywords)
+        # Fallback keyword extraction method (same as before)
+        stop_words = %w[
+          a an and are as at be by for from has he in is it its of on that the
+          to was will with the this these those they them their there where when
+          what who why how which would could should shall might may can must
+          do does did done doing go goes went gone going get gets got gotten
+          getting have had having has been being am was were are is was been
+          but or not no yes if then else also too very much many most some any
+          all each every each other another one two three first second third
+          last next previous before after during while until since through
+          above below under over between among within without across around
+          near far close distant here there everywhere nowhere somewhere anywhere
+        ]
+
+        # Clean and normalize text
+        cleaned_text = text.downcase
+                           .gsub(/[^\w\s]/, " ") # Remove punctuation
+                           .gsub(/\s+/, " ") # Normalize whitespace
+                           .strip
+
+        # Split into words and filter
+        words = cleaned_text.split(" ")
+                            .reject { |word| word.length < 3 } # Remove short words
+                            .reject { |word| stop_words.include?(word) } # Remove stop words
+                            .reject { |word| word.match?(/^\d+$/) } # Remove pure numbers
+
+        # Count word frequencies
+        word_counts = Hash.new(0)
+        words.each { |word| word_counts[word] += 1 }
+
+        # Get top keywords (words that appear more than once or are significant)
+        word_counts
+          .select { |word, count| count > 1 || word.length > 6 }
+          .sort_by { |word, count| [-count, word] }
+          .first(max_keywords) # Limit to max_keywords
+          .map { |word, _count| word }
+      end
+    end
+  end
+end

data/lib/ragdoll/core/version.rb

@@ -0,0 +1,8 @@
+# ragdoll/core/version.rb
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    VERSION = "0.1.1"
+  end
+end

data/lib/ragdoll/core.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "debug_me"
+include DebugMe
+$DEBUG_ME = true
+
+# require_relative "../extensions/openstruct_merge"  # File doesn't exist
+
+require_relative "core/version"
+require_relative "core/errors"
+require_relative "core/configuration"
+require_relative "core/database"
+require_relative "core/shrine_config"
+require_relative "core/models/document"
+require_relative "core/models/embedding"
+require_relative "core/models/text_content"
+require_relative "core/models/audio_content"
+require_relative "core/models/image_content"
+require_relative "core/document_processor"
+require_relative "core/document_management"
+require_relative "core/text_chunker"
+require_relative "core/embedding_service"
+require_relative "core/text_generation_service"
+require_relative "core/search_engine"
+require_relative "core/services/image_description_service"
+require_relative "core/jobs/generate_embeddings"
+require_relative "core/jobs/generate_summary"
+require_relative "core/jobs/extract_keywords"
+require_relative "core/client"
+
+module Ragdoll
+  def self.config
+    @config ||= Core::Configuration.new
+  end
+
+  module Core
+    extend SingleForwardable
+
+    def self.config
+      @config ||= Configuration.new
+    end
+
+    def self.configuration
+      config
+    end
+
+    def self.configure
+      yield(config)
+    end
+
+    # Reset configuration (useful for testing)
+    def self.reset_configuration!
+      @config = Configuration.new
+      @default_client = nil
+    end
+
+    # Factory method for creating clients
+    def self.client(config = nil)
+      Client.new(config)
+    end
+
+    # Delegate high-level API methods to default client
+    def_delegators :default_client, :add_document, :search, :enhance_prompt,
+                   :get_document, :document_status, :list_documents, :delete_document,
+                   :update_document, :get_context, :search_similar_content,
+                   :add_directory, :stats, :healthy?, :hybrid_search
+
+    def self.default_client
+      @default_client ||= Client.new
+    end
+  end
+end

data/lib/ragdoll-core.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "ragdoll/core"