ragdoll 0.1.8 → 0.1.10

data/app/services/ragdoll/document_processor.rb

@@ -3,6 +3,8 @@
 require "pdf-reader"
 require "docx"
 require "rmagick"
+require "yaml"
+require "date"
 # Image description service is auto-loaded from app/services
 
 module Ragdoll
@@ -137,6 +139,11 @@ module Ragdoll
         raise ParseError, "Unsupported PDF feature: #{e.message}"
       end
 
+      # Add filepath-based title as fallback if no title was found
+      if metadata[:title].nil? || (metadata[:title].is_a?(String) && metadata[:title].strip.empty?)
+        metadata[:title] = extract_title_from_filepath
+      end
+
       {
         content: content.strip,
         metadata: metadata,
@@ -192,6 +199,11 @@ module Ragdoll
         raise ParseError, "#{__LINE__} Failed to parse DOCX: #{e.message}"
       end
 
+      # Add filepath-based title as fallback if no title was found
+      if metadata[:title].nil? || (metadata[:title].is_a?(String) && metadata[:title].strip.empty?)
+        metadata[:title] = extract_title_from_filepath
+      end
+
       {
         content: content.strip,
         metadata: metadata,
@@ -212,6 +224,20 @@ module Ragdoll
                       else "text"
                       end
 
+      # Parse YAML front matter for markdown files
+      if document_type == "markdown" && content.start_with?("---\n")
+        front_matter, body_content = parse_yaml_front_matter(content)
+        if front_matter
+          metadata.merge!(front_matter)
+          content = body_content
+        end
+      end
+
+      # Add filepath-based title as fallback if no title was found
+      if metadata[:title].nil? || (metadata[:title].is_a?(String) && metadata[:title].strip.empty?)
+        metadata[:title] = extract_title_from_filepath
+      end
+
       {
         content: content,
         metadata: metadata,
@@ -225,16 +251,41 @@ module Ragdoll
         encoding: "ISO-8859-1"
       }
 
+      # Try to parse front matter with different encoding too
+      if document_type == "markdown" && content.start_with?("---\n")
+        front_matter, body_content = parse_yaml_front_matter(content)
+        if front_matter
+          metadata.merge!(front_matter)
+          content = body_content
+        end
+      end
+
+      # Add filepath-based title as fallback if no title was found
+      if metadata[:title].nil? || (metadata[:title].is_a?(String) && metadata[:title].strip.empty?)
+        metadata[:title] = extract_title_from_filepath
+      end
+
       {
         content: content,
         metadata: metadata,
-        document_type: "text"
+        document_type: document_type.nil? ? "text" : document_type
       }
     end
 
     def parse_html
       content = File.read(@file_path, encoding: "UTF-8")
 
+      # Extract title from H1 tag if present
+      h1_match = content.match(%r{<h1[^>]*>(.*?)</h1>}mi)
+      title = nil
+      if h1_match
+        # Clean up the H1 content by removing any HTML tags and normalizing whitespace
+        title = h1_match[1]
+                  .gsub(/<[^>]+>/, " ")  # Remove any nested HTML tags
+                  .gsub(/\s+/, " ")      # Normalize whitespace
+                  .strip
+      end
+
       # Basic HTML tag stripping (for more advanced parsing, consider using Nokogiri)
       clean_content = content
                       .gsub(%r{<script[^>]*>.*?</script>}mi, "") # Remove script tags
@@ -248,6 +299,13 @@ module Ragdoll
         original_format: "html"
       }
 
+      # Add title to metadata if found, otherwise use filepath fallback
+      if title && !title.empty?
+        metadata[:title] = title
+      else
+        metadata[:title] = extract_title_from_filepath
+      end
+
       {
         content: clean_content,
         metadata: metadata,
@@ -286,6 +344,9 @@ module Ragdoll
       # Use AI-generated description or fallback placeholder
       content = desc && !desc.empty? ? desc : "Image file: #{File.basename(@file_path)}"
 
+      # Add filepath-based title as fallback
+      metadata[:title] = extract_title_from_filepath
+
       puts "✅ DocumentProcessor: Image parsing complete. Content: '#{content[0..100]}...'"
 
       {
@@ -338,5 +399,67 @@ module Ragdoll
       else "application/octet-stream"
       end
     end
+
+    private
+
+    # Extract a meaningful title from the file path as a fallback
+    # @param file_path [String] the full file path
+    # @return [String] a cleaned title derived from the filename
+    def extract_title_from_filepath(file_path = @file_path)
+      filename = File.basename(file_path, File.extname(file_path))
+      
+      # Clean up common patterns in filenames to make them more readable
+      title = filename
+               .gsub(/[-_]+/, ' ')           # Replace hyphens and underscores with spaces
+               .gsub(/([a-z])([A-Z])/, '\1 \2') # Add space before capital letters (camelCase)
+               .gsub(/\s+/, ' ')             # Normalize multiple spaces
+               .strip
+      
+      # Capitalize words for better readability
+      title.split(' ').map(&:capitalize).join(' ')
+    end
+
+    # Parse YAML front matter from markdown content
+    # @param content [String] the full content of the markdown file
+    # @return [Array] returns [front_matter_hash, body_content] or [nil, original_content]
+    def parse_yaml_front_matter(content)
+      # Check if content starts with YAML front matter delimiter
+      return [nil, content] unless content.start_with?("---\n")
+
+      # Find the closing delimiter
+      lines = content.lines
+      closing_index = nil
+      
+      lines.each_with_index do |line, index|
+        next if index == 0 # Skip the opening ---
+        if line.strip == "---"
+          closing_index = index
+          break
+        end
+      end
+
+      # No closing delimiter found
+      return [nil, content] unless closing_index
+
+      # Extract YAML content and body
+      yaml_lines = lines[1...closing_index]
+      body_lines = lines[(closing_index + 1)..-1]
+
+      yaml_content = yaml_lines.join
+      body_content = body_lines&.join || ""
+
+      # Parse YAML
+      begin
+        # Allow Time objects for date fields in YAML front matter
+        front_matter = YAML.safe_load(yaml_content, permitted_classes: [Time, Date])
+        # Convert string keys to symbols for consistency
+        front_matter = front_matter.transform_keys(&:to_sym) if front_matter.is_a?(Hash)
+        [front_matter, body_content.strip]
+      rescue YAML::SyntaxError, Psych::DisallowedClass => e
+        # If YAML parsing fails, return original content
+        Rails.logger.warn "Warning: Failed to parse YAML front matter: #{e.message}" if defined?(Rails)
+        [nil, content]
+      end
+    end
   end
 end

data/app/services/ragdoll/embedding_service.rb

@@ -38,6 +38,11 @@ module Ragdoll
           embedding_config = @model_resolver.resolve_embedding(:text)
           # Use just the model name for RubyLLM
           model = embedding_config.model.model
+          
+          # If model is nil or empty, use fallback
+          if model.nil? || model.empty?
+            return generate_fallback_embedding
+          end
 
           begin
             response = RubyLLM.embed(cleaned_text, model: model)
@@ -93,6 +98,11 @@ module Ragdoll
           embedding_config = @model_resolver.resolve_embedding(:text)
           # Use just the model name for RubyLLM
           model = embedding_config.model.model
+          
+          # If model is nil or empty, use fallback
+          if model.nil? || model.empty?
+            return cleaned_texts.map { generate_fallback_embedding }
+          end
 
           cleaned_texts.map do |text|
             response = RubyLLM.embed(text, model: model)

data/app/services/ragdoll/search_engine.rb

@@ -27,25 +27,94 @@ module Ragdoll
     end
 
     def search_similar_content(query_or_embedding, options = {})
+      start_time = Time.current
       search_config = @config_service.search_config
       limit = options[:limit] || search_config[:max_results]
       threshold = options[:threshold] || search_config[:similarity_threshold]
       filters = options[:filters] || {}
+      
+      # Extract keywords option and normalize
+      keywords = options[:keywords] || []
+      keywords = Array(keywords).map(&:to_s).reject(&:empty?)
+      
+      # Extract tracking options
+      session_id = options[:session_id]
+      user_id = options[:user_id]
+      track_search = options.fetch(:track_search, true)
 
       if query_or_embedding.is_a?(Array)
         # It's already an embedding
         query_embedding = query_or_embedding
+        query_string = options[:query] # Should be provided when passing embedding directly
       else
         # It's a query string, generate embedding
-        query_embedding = @embedding_service.generate_embedding(query_or_embedding)
+        query_string = query_or_embedding
+        query_embedding = @embedding_service.generate_embedding(query_string)
         return [] if query_embedding.nil?
       end
 
-      # Search using ActiveRecord models
-      Ragdoll::Embedding.search_similar(query_embedding,
-                                       limit: limit,
-                                       threshold: threshold,
-                                       filters: filters)
+      # Add keywords to filters if provided
+      if keywords.any?
+        filters[:keywords] = keywords
+      end
+
+      # Search using ActiveRecord models with statistics
+      # Try enhanced search first, fall back to original if it fails
+      begin
+        search_response = Ragdoll::Embedding.search_similar_with_stats(query_embedding,
+                                                                      limit: limit,
+                                                                      threshold: threshold,
+                                                                      filters: filters)
+        results = search_response[:results]
+        statistics = search_response[:statistics]
+      rescue NoMethodError, PG::SyntaxError => e
+        # Fall back to original search method if enhanced version fails
+        puts "Warning: Enhanced search failed (#{e.message}), using fallback" if ENV["RAGDOLL_DEBUG"]
+        results = Ragdoll::Embedding.search_similar(query_embedding,
+                                                   limit: limit,
+                                                   threshold: threshold,
+                                                   filters: filters)
+        statistics = nil
+      end
+      
+      execution_time = ((Time.current - start_time) * 1000).round
+      
+      # Record search if tracking enabled and we have a query string
+      if track_search && query_string && !query_string.empty?
+        begin
+          # Format results for search recording
+          search_results = results.map do |result|
+            {
+              embedding_id: result[:embedding_id] || result[:id],
+              similarity: result[:similarity] || result[:similarity_score] || 0.0
+            }
+          end
+          
+          search_type = keywords.any? ? "semantic_with_keywords" : "semantic"
+          
+          Ragdoll::Search.record_search(
+            query: query_string,
+            query_embedding: query_embedding,
+            results: search_results,
+            search_type: search_type,
+            filters: filters,
+            options: { limit: limit, threshold: threshold, keywords: keywords },
+            execution_time_ms: execution_time,
+            session_id: session_id,
+            user_id: user_id
+          )
+        rescue => e
+          # Log error but don't fail the search
+          puts "Warning: Search tracking failed: #{e.message}" if ENV["RAGDOLL_DEBUG"]
+        end
+      end
+
+      # Return results with statistics for better user feedback
+      {
+        results: results,
+        statistics: statistics,
+        execution_time_ms: execution_time
+      }
     end
   end
 end

data/db/migrate/{001_enable_postgresql_extensions.rb → 20250815234901_enable_postgresql_extensions.rb}

@@ -1,8 +1,5 @@
 class EnablePostgresqlExtensions < ActiveRecord::Migration[7.0]
   def up
-    # This migration is now handled by the db:create rake task
-    # Just ensure required extensions are available
-    
     # Vector similarity search (required for embeddings)
     execute "CREATE EXTENSION IF NOT EXISTS vector"
     
@@ -15,9 +12,11 @@ class EnablePostgresqlExtensions < ActiveRecord::Migration[7.0]
   end
 
   def down
-    execute <<-SQL
-      DROP DATABASE IF EXISTS ragdoll_development;
-      DROP ROLE IF EXISTS ragdoll;
-    SQL
+    # Extensions are typically not dropped as they might be used by other databases
+    # If you really need to drop them, uncomment the following:
+    # execute "DROP EXTENSION IF EXISTS vector"
+    # execute "DROP EXTENSION IF EXISTS unaccent"
+    # execute "DROP EXTENSION IF EXISTS pg_trgm"
+    # execute "DROP EXTENSION IF EXISTS \"uuid-ossp\""
   end
-end
+end

data/db/migrate/20250815234902_create_ragdoll_documents.rb

@@ -0,0 +1,117 @@
+class CreateRagdollDocuments < ActiveRecord::Migration[7.0]
+  # For concurrent index creation (PostgreSQL)
+  disable_ddl_transaction!
+
+  def up
+    create_table :ragdoll_documents,
+      comment: "Core documents table with LLM-generated structured metadata" do |t|
+
+      t.string :location, null: false,
+        comment: "Source location of document (file path, URL, or identifier)"
+
+      t.string :title, null: false,
+        comment: "Human-readable document title for display and search"
+
+      t.text :summary, null: false, default: "",
+        comment: "LLM-generated summary of document content"
+
+      t.string :document_type, null: false, default: "text",
+        comment: "Document format type"
+
+      t.string :status, null: false, default: "pending",
+        comment: "Document processing status"
+
+      t.json :metadata, default: {},
+        comment: "LLM-generated structured metadata about the file"
+
+      t.timestamp :file_modified_at, null: false, default: -> { "CURRENT_TIMESTAMP" },
+        comment: "Timestamp when the source file was last modified"
+
+      t.timestamps null: false,
+        comment: "Standard creation and update timestamps"
+
+      # Add tsvector column for full-text search
+      t.tsvector :search_vector
+
+      # Add keywords as array column
+      t.text :keywords, array: true, default: []
+    end
+
+    ###########
+    # Indexes #
+    ###########
+
+    add_index :ragdoll_documents, :location, unique: true,
+      comment: "Unique index for document source lookup"
+
+    add_index :ragdoll_documents, :title,
+      comment: "Index for title-based search"
+
+    add_index :ragdoll_documents, :document_type,
+      comment: "Index for filtering by document type"
+
+    add_index :ragdoll_documents, :status,
+      comment: "Index for filtering by processing status"
+
+    add_index :ragdoll_documents, :created_at,
+      comment: "Index for chronological sorting"
+
+    add_index :ragdoll_documents, [:document_type, :status],
+      comment: "Composite index for type+status filtering"
+
+    # Full-text search index
+    execute <<-SQL
+      CREATE INDEX CONCURRENTLY index_ragdoll_documents_on_fulltext_search
+      ON ragdoll_documents
+      USING gin(to_tsvector('english', 
+        COALESCE(title, '') || ' ' ||
+        COALESCE(metadata->>'summary', '') || ' ' ||
+        COALESCE(metadata->>'keywords', '') || ' ' ||
+        COALESCE(metadata->>'description', '')
+      ))
+    SQL
+
+    add_index :ragdoll_documents, "(metadata->>'document_type')", 
+      name: "index_ragdoll_documents_on_metadata_type",
+      comment: "Index for filtering by document type"
+
+    add_index :ragdoll_documents, "(metadata->>'classification')", 
+      name: "index_ragdoll_documents_on_metadata_classification",
+      comment: "Index for filtering by document classification"
+
+    # GIN index on search_vector
+    add_index :ragdoll_documents, :search_vector, using: :gin, algorithm: :concurrently
+
+    # GIN index on keywords array
+    add_index :ragdoll_documents, :keywords, using: :gin, 
+      name: 'index_ragdoll_documents_on_keywords_gin'
+
+    # Trigger to keep search_vector up to date on INSERT/UPDATE
+    execute <<-SQL
+      CREATE FUNCTION ragdoll_documents_vector_update() RETURNS trigger AS $$
+      BEGIN
+        NEW.search_vector := to_tsvector('english',
+          COALESCE(NEW.title, '') || ' ' ||
+          COALESCE(NEW.metadata->>'summary', '') || ' ' ||
+          COALESCE(NEW.metadata->>'keywords', '') || ' ' ||
+          COALESCE(NEW.metadata->>'description', '')
+        );
+        RETURN NEW;
+      END
+      $$ LANGUAGE plpgsql;
+
+      CREATE TRIGGER ragdoll_search_vector_update
+      BEFORE INSERT OR UPDATE ON ragdoll_documents
+      FOR EACH ROW EXECUTE FUNCTION ragdoll_documents_vector_update();
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      DROP TRIGGER IF EXISTS ragdoll_search_vector_update ON ragdoll_documents;
+      DROP FUNCTION IF EXISTS ragdoll_documents_vector_update();
+    SQL
+    
+    drop_table :ragdoll_documents
+  end
+end

data/db/migrate/{005_create_ragdoll_embeddings.rb → 20250815234903_create_ragdoll_embeddings.rb}

@@ -3,7 +3,7 @@ class CreateRagdollEmbeddings < ActiveRecord::Migration[7.0]
     create_table :ragdoll_embeddings,
       comment: "Polymorphic vector embeddings storage for semantic similarity search" do |t|
 
-        t.references :embeddable, polymorphic: true, null: false,
+      t.references :embeddable, polymorphic: true, null: false,
         comment: "Polymorphic reference to embeddable content"
 
       t.text :content, null: false, default: "",
@@ -26,16 +26,19 @@ class CreateRagdollEmbeddings < ActiveRecord::Migration[7.0]
 
       t.timestamps null: false,
         comment: "Standard creation and update timestamps"
+    end
 
-      ###########
-      # Indexes #
-      ###########
+    ###########
+    # Indexes #
+    ###########
 
-      t.index %i[embeddable_type embeddable_id],
-        comment: "Index for finding embeddings by embeddable content"
+    add_index :ragdoll_embeddings, [:embeddable_type, :embeddable_id],
+      comment: "Index for finding embeddings by embeddable content"
 
-      t.index :embedding_vector, using: :ivfflat, opclass: :vector_cosine_ops, name: "index_ragdoll_embeddings_on_embedding_vector_cosine",
-        comment: "IVFFlat index for fast cosine similarity search"
-    end
+    add_index :ragdoll_embeddings, :embedding_vector, 
+      using: :ivfflat, 
+      opclass: :vector_cosine_ops, 
+      name: "index_ragdoll_embeddings_on_embedding_vector_cosine",
+      comment: "IVFFlat index for fast cosine similarity search"
   end
-end
+end

data/db/migrate/{006_create_ragdoll_contents.rb → 20250815234904_create_ragdoll_contents.rb}

@@ -29,19 +29,22 @@ class CreateRagdollContents < ActiveRecord::Migration[7.0]
 
       t.timestamps null: false,
         comment: "Standard creation and update timestamps"
+    end
 
-      ###########
-      # Indexes #
-      ###########
+    ###########
+    # Indexes #
+    ###########
 
-      t.index :embedding_model,
-        comment: "Index for filtering by embedding model"
+    add_index :ragdoll_contents, :embedding_model,
+      comment: "Index for filtering by embedding model"
 
-      t.index :type,
-        comment: "Index for filtering by content type"
+    add_index :ragdoll_contents, :type,
+      comment: "Index for filtering by content type"
 
-      t.index "to_tsvector('english', COALESCE(content, ''))", using: :gin, name: "index_ragdoll_contents_on_fulltext_search",
-        comment: "Full-text search index for text content"
-    end
+    execute <<-SQL
+      CREATE INDEX index_ragdoll_contents_on_fulltext_search
+      ON ragdoll_contents
+      USING gin(to_tsvector('english', COALESCE(content, '')))
+    SQL
   end
-end
+end

data/db/migrate/20250815234905_create_ragdoll_searches.rb

@@ -0,0 +1,77 @@
+class CreateRagdollSearches < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ragdoll_searches,
+      comment: "Search queries and results tracking with vector similarity support" do |t|
+
+      t.text :query, null: false,
+        comment: "Original search query text"
+
+      t.vector :query_embedding, limit: 1536, null: false,
+        comment: "Vector embedding of the search query for similarity matching"
+
+      t.string :search_type, null: false, default: "semantic",
+        comment: "Type of search performed (semantic, hybrid, fulltext)"
+
+      t.integer :results_count, null: false, default: 0,
+        comment: "Number of results returned for this search"
+
+      t.float :max_similarity_score, 
+        comment: "Highest similarity score from results"
+
+      t.float :min_similarity_score,
+        comment: "Lowest similarity score from results"
+
+      t.float :avg_similarity_score,
+        comment: "Average similarity score of results"
+
+      t.json :search_filters, default: {},
+        comment: "Filters applied during search (document_type, date_range, etc.)"
+
+      t.json :search_options, default: {},
+        comment: "Search configuration options (threshold, limit, etc.)"
+
+      t.integer :execution_time_ms,
+        comment: "Search execution time in milliseconds"
+
+      t.string :session_id,
+        comment: "User session identifier for grouping related searches"
+
+      t.string :user_id,
+        comment: "User identifier if authentication is available"
+
+      t.timestamps null: false,
+        comment: "Standard creation and update timestamps"
+    end
+
+    ###########
+    # Indexes #
+    ###########
+
+    add_index :ragdoll_searches, :query_embedding, 
+      using: :ivfflat, 
+      opclass: :vector_cosine_ops, 
+      name: "index_ragdoll_searches_on_query_embedding_cosine",
+      comment: "IVFFlat index for finding similar search queries"
+
+    add_index :ragdoll_searches, :search_type,
+      comment: "Index for filtering by search type"
+
+    add_index :ragdoll_searches, :session_id,
+      comment: "Index for grouping searches by session"
+
+    add_index :ragdoll_searches, :user_id,
+      comment: "Index for filtering searches by user"
+
+    add_index :ragdoll_searches, :created_at,
+      comment: "Index for chronological search history"
+
+    add_index :ragdoll_searches, :results_count,
+      comment: "Index for analyzing search effectiveness"
+
+    execute <<-SQL
+      CREATE INDEX index_ragdoll_searches_on_fulltext_query
+      ON ragdoll_searches
+      USING gin(to_tsvector('english', query))
+    SQL
+  end
+end

data/db/migrate/20250815234906_create_ragdoll_search_results.rb

@@ -0,0 +1,49 @@
+class CreateRagdollSearchResults < ActiveRecord::Migration[7.0]
+  def change
+    # Junction table for tracking which embeddings were returned for each search
+    create_table :ragdoll_search_results,
+      comment: "Junction table linking searches to their returned embeddings" do |t|
+
+      t.references :search, null: false, foreign_key: { to_table: :ragdoll_searches },
+        comment: "Reference to the search query"
+
+      t.references :embedding, null: false, foreign_key: { to_table: :ragdoll_embeddings },
+        comment: "Reference to the returned embedding"
+
+      t.float :similarity_score, null: false,
+        comment: "Similarity score for this result"
+
+      t.integer :result_rank, null: false,
+        comment: "Ranking position of this result (1-based)"
+
+      t.boolean :clicked, default: false,
+        comment: "Whether user interacted with this result"
+
+      t.datetime :clicked_at,
+        comment: "Timestamp when result was clicked/selected"
+
+      t.timestamps null: false,
+        comment: "Standard creation and update timestamps"
+    end
+
+    ###########
+    # Indexes #
+    ###########
+
+    add_index :ragdoll_search_results, [:search_id, :result_rank],
+      name: "idx_search_results_search_rank",
+      comment: "Index for retrieving results in ranked order"
+
+    add_index :ragdoll_search_results, [:embedding_id, :similarity_score],
+      name: "idx_search_results_embedding_score", 
+      comment: "Index for analyzing embedding performance"
+
+    add_index :ragdoll_search_results, :similarity_score,
+      name: "idx_search_results_similarity",
+      comment: "Index for similarity score analysis"
+
+    add_index :ragdoll_search_results, [:clicked, :clicked_at],
+      name: "idx_search_results_clicks",
+      comment: "Index for click-through analysis"
+  end
+end