smart_rag 0.1.0

data/lib/smart_rag/models/search_log.rb

@@ -0,0 +1,198 @@
+require_relative "model_base"
+require "sequel/plugins/validation_helpers"
+
+module SmartRAG
+  module Models
+    # SearchLog model for tracking search queries and performance
+    class SearchLog < Sequel::Model
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(:search_logs)
+      end
+      include FactoryBotHelpers
+      plugin :validation_helpers
+      plugin :timestamps, update_on_create: false
+
+      # Add bang methods for FactoryBot compatibility
+      def self.create!(attributes = {})
+        instance = new(attributes)
+        instance.save! || raise(Sequel::ValidationFailed, instance.errors.full_messages.join(", "))
+        instance
+      end
+
+      # Validation
+      def validate
+        super
+        validates_presence :query
+        validates_max_length 10000, :query  # Reasonable limit for query length
+        validates_includes %w[vector fulltext hybrid], :search_type, allow_nil: true
+        validates_integer :execution_time_ms, allow_nil: true, greater_than_or_equal_to: 0
+        validates_integer :results_count, allow_nil: true, greater_than_or_equal_to: 0
+      end
+
+      # Class methods
+      class << self
+        # Log a search query
+        def log(query:, search_type: nil, execution_time_ms: nil, results_count: nil,
+                query_vector: nil, result_ids: nil, filters: nil)
+          create(
+            query: query,
+            search_type: search_type,
+            execution_time_ms: execution_time_ms,
+            results_count: results_count,
+            query_vector: query_vector,
+            result_ids: result_ids,
+            filters: filters.is_a?(Hash) ? filters.to_json : filters
+          )
+        end
+
+        # Find logs by query type
+        def by_search_type(type)
+          where(search_type: type)
+        end
+
+        # Get slow queries
+        def slow_queries(threshold_ms: 100)
+          where(Sequel.lit('execution_time_ms IS NOT NULL AND execution_time_ms > ?', threshold_ms))
+            .order(Sequel.desc(:execution_time_ms))
+        end
+
+        # Get recent searches
+        def recent(limit: 50)
+          order(Sequel.desc(:created_at)).limit(limit)
+        end
+
+        # Get popular queries (by frequency)
+        def popular(limit: 20)
+          group_and_count(:query)
+            .order(Sequel.desc(:count))
+            .limit(limit)
+        end
+
+        # Get searches with no results
+        def with_no_results
+          where(results_count: 0).or(results_count: nil)
+        end
+
+        # Get searches with many results
+        def with_many_results(threshold: 100)
+          where(Sequel.lit('results_count > ?', threshold))
+        end
+
+        # Get average execution time by search type
+        def average_execution_time_by_type
+          select(:search_type,
+                 Sequel.function(:avg, :execution_time_ms).as(:avg_time),
+                 Sequel.function(:count, :*).as(:count))
+            .where(Sequel.lit('execution_time_ms IS NOT NULL'))
+            .group(:search_type)
+            .order(Sequel.desc(:avg_time))
+        end
+
+        # Find similar queries (based on vector similarity)
+        def find_similar_queries(query_vector, limit: 10)
+          return [] unless query_vector
+
+          # Find queries with similar vectors and non-zero results
+          where(Sequel.lit('query_vector IS NOT NULL AND results_count > 0'))
+            .where(Sequel.lit('query_vector <=> ? < ?', query_vector.to_s, 0.3))
+            .order(Sequel.lit('query_vector <=> ?', query_vector.to_s))
+            .limit(limit)
+        end
+
+        # Get search analytics (by time period)
+        def analytics_by_period(start_time:, end_time:)
+          where(created_at: start_time..end_time)
+            .select(
+              Sequel.function(:count, :*).as(:total_searches),
+              Sequel.function(:avg, :execution_time_ms).as(:avg_response_time),
+              Sequel.function(:sum, Sequel.lit("CASE WHEN results_count > 0 THEN 1 ELSE 0 END")).as(:successful_searches),
+              Sequel.function(:sum, Sequel.lit("CASE WHEN results_count = 0 OR results_count IS NULL THEN 1 ELSE 0 END")).as(:failed_searches)
+            )
+            .first
+        end
+
+        # Clean old logs (keep only last N days)
+        def cleanup(days_to_keep: 30)
+          cutoff_date = Time.now - (days_to_keep * 24 * 60 * 60)
+          where(Sequel.lit('created_at < ?', cutoff_date)).delete
+        end
+
+        # Export search logs
+        def export(start_time:, end_time:, format: :json)
+          logs = where(created_at: start_time..end_time).all
+
+          case format
+          when :json
+            logs.map(&:to_hash).to_json
+          when :csv
+            # Convert to CSV (simplified)
+            require 'csv'
+            CSV.generate do |csv|
+              csv << [:id, :query, :search_type, :execution_time_ms, :results_count, :created_at]
+              logs.each do |log|
+                csv << [log.id, log.query, log.search_type, log.execution_time_ms, log.results_count, log.created_at]
+              end
+            end
+          else
+            logs
+          end
+        end
+      end
+
+      # Instance methods
+
+      # Check if search was successful (had results)
+      def successful?
+        results_count && results_count > 0
+      end
+
+      # Check if search was slow
+      def slow?(threshold_ms: 100)
+        execution_time_ms && execution_time_ms > threshold_ms
+      end
+
+      # Get filters as hash
+      def filters_hash
+        begin
+          filters.is_a?(String) ? JSON.parse(filters) : filters
+        rescue
+          {}
+        end
+      end
+
+      # Get result IDs as array
+      def result_ids_array
+        return [] unless result_ids
+        result_ids.is_a?(String) ? JSON.parse(result_ids) : result_ids
+      end
+
+      # Query vector as array
+      def query_vector_array
+        return nil unless query_vector
+        query_vector.to_s.gsub(/[<>]/, '').split(',').map(&:to_f)
+      rescue
+        nil
+      end
+
+      # Log info hash
+      def info
+        {
+          id: id,
+          query: query,
+          search_type: search_type,
+          execution_time_ms: execution_time_ms,
+          results_count: results_count,
+          successful: successful?,
+          slow: slow?,
+          created_at: created_at
+        }
+      end
+
+      # String representation
+      def to_s
+        "<SearchLog: #{id} - #{query[0..50]}#{query.length > 50 ? '...' : ''} (#{search_type})>"
+      end
+    end
+  end
+end

data/lib/smart_rag/models/section_fts.rb

@@ -0,0 +1,170 @@
+require_relative "model_base"
+require "sequel/plugins/validation_helpers"
+
+module SmartRAG
+  module Models
+    # SectionFts model for full-text search optimization
+    class SectionFts < Sequel::Model(:section_fts)
+      include FactoryBotHelpers
+      plugin :validation_helpers
+      plugin :timestamps, update_on_create: true
+
+      # Add bang methods for FactoryBot compatibility
+      def self.create!(attributes = {})
+        instance = new(attributes)
+        instance.save! || raise(Sequel::ValidationFailed, instance.errors.full_messages.join(", "))
+        instance
+      end
+
+      # Relationships
+      many_to_one :section, class: '::SmartRAG::Models::SourceSection', key: :section_id
+      many_to_one :document, class: '::SmartRAG::Models::SourceDocument', key: :document_id
+
+      # Validation
+      def validate
+        super
+        validates_presence :section_id
+        validates_presence :language, allow_nil: true
+        validates_format /\A[a-z]{2}\z/, :language, allow_nil: true, message: 'must be ISO 639-1 code'
+      end
+
+      # Class methods
+      class << self
+        # Full-text search using tsvector
+        def search(query, language: nil, fields: [:fts_combined], limit: 20)
+          # Ensure we have valid fields
+          valid_fields = [:fts_title, :fts_content, :fts_combined]
+          fields = Array(fields) & valid_fields
+
+          # Build the tsquery
+          tsquery = build_tsquery(query, language || 'simple')
+
+          # Build the search query
+          search_conditions = fields.map do |field|
+            Sequel.lit("#{field} @@ #{tsquery}")
+          end
+
+          # Union of all field searches (OR condition)
+          where(Sequel.|(*search_conditions))
+            .order(Sequel.desc(Sequel.lit("ts_rank(#{fields.join(' || ')}, #{tsquery})")))
+            .limit(limit)
+        end
+
+        # Find by document
+        def by_document(document_id)
+          where(document_id: document_id)
+        end
+
+        # Find by section
+        def by_section(section_id)
+          where(section_id: section_id)
+        end
+
+        # Find by language
+        def by_language(lang)
+          where(language: lang)
+        end
+
+        # Custom ranking search
+        def search_with_ranking(query, language: nil, weights: '1.0, 0.5, 0.2', limit: 20)
+          tsquery = build_tsquery(query, language || 'simple')
+
+          select(:*,
+                 Sequel.lit("ts_rank('{#{weights}}', fts_combined, #{tsquery})").as(:rank))
+            .where(Sequel.lit("fts_combined @@ #{tsquery}"))
+            .order(Sequel.desc(:rank))
+            .limit(limit)
+        end
+
+        # Build tsquery from text
+        def build_tsquery(query, language = 'simple')
+          # Convert query to tsquery format
+          # Replace spaces with & for AND search
+          # Add * for prefix matching
+          terms = query.to_s.split.map { |term| "#{term}:*" }
+          terms.join(' & ')
+        end
+
+        # Find sections with fresh FTS data
+        def fresh(max_age: 3600)
+          # Sections updated within the last hour
+          where(Sequel.lit('updated_at > ?', Time.now - max_age))
+        end
+
+        # Find stale FTS entries
+        def stale
+          # Find sections without FTS or with old FTS
+          subquery = db[:source_sections]
+                     .left_join(:section_fts, section_id: :id)
+                     .where(Sequel.|(
+                       { Sequel[:section_fts][:section_id] => nil },
+                       Sequel.lit('source_sections.updated_at > section_fts.updated_at')
+                     ))
+                     .select(:source_sections__id)
+
+          where(section_id: subquery)
+        end
+
+        # Create or update FTS entry
+        def create_or_update(section_id, attrs)
+          existing = find(section_id: section_id)
+          if existing
+            existing.update(attrs)
+            existing
+          else
+            create(attrs.merge(section_id: section_id))
+          end
+        end
+      end
+
+      # Instance methods
+
+      # Check if FTS data is up to date with section
+      def up_to_date?(section_updated_at)
+        return false unless updated_at
+        updated_at >= section_updated_at
+      end
+
+      # Get search vectors as hash
+      def vectors
+        {
+          title: fts_title,
+          content: fts_content,
+          combined: fts_combined
+        }
+      end
+
+      # Update search vectors
+      def update_vectors(title_vector, content_vector, combined_vector)
+        update(
+          fts_title: title_vector,
+          fts_content: content_vector,
+          fts_combined: combined_vector,
+          updated_at: Time.now
+        )
+      end
+
+      # Get rank for a query
+      def rank_for(query, language: nil, weights: '1.0, 0.5, 0.2')
+        tsquery = self.class.build_tsquery(query, language || 'simple')
+        db[Sequel.lit("SELECT ts_rank('{#{weights}}', ?, ?) as rank", fts_combined, tsquery)].first[:rank]
+      end
+
+      # Section FTS info
+      def info
+        {
+          section_id: section_id,
+          document_id: document_id,
+          language: language,
+          has_vectors: fts_combined.present?,
+          updated_at: updated_at
+        }
+      end
+
+      # String representation
+      def to_s
+        "<SectionFts: section_id=#{section_id}>"
+      end
+    end
+  end
+end

data/lib/smart_rag/models/section_tag.rb

@@ -0,0 +1,81 @@
+require_relative "model_base"
+require "sequel/plugins/validation_helpers"
+
+module SmartRAG
+  module Models
+    # SectionTag model for many-to-many relationship between sections and tags
+    class SectionTag < Sequel::Model(:section_tags)
+      include FactoryBotHelpers
+      plugin :validation_helpers
+      plugin :timestamps, update_on_create: false
+
+      # Allow mass assignment of primary keys
+      unrestrict_primary_key
+
+      # Add bang methods for FactoryBot compatibility
+      def self.create!(attributes = {})
+        instance = new(attributes)
+        instance.save! || raise(Sequel::ValidationFailed, instance.errors.full_messages.join(", "))
+        instance
+      end
+
+      # Relationships (explicit, though many_to_many is used in main models)
+      many_to_one :section, class: '::SmartRAG::Models::SourceSection', key: :section_id
+      many_to_one :tag, class: '::SmartRAG::Models::Tag', key: :tag_id
+
+      # Validation
+      def validate
+        super
+        validates_presence [:section_id, :tag_id]
+        validates_unique [:section_id, :tag_id]
+      end
+
+      # Class methods
+      class << self
+        # Find by section and tag
+        def find_by_section_and_tag(section_id, tag_id)
+          where(section_id: section_id, tag_id: tag_id).first
+        end
+
+        # Get all tags for a section
+        def tags_for_section(section_id)
+          where(section_id: section_id).all
+        end
+
+        # Get all sections for a tag
+        def sections_for_tag(tag_id)
+          where(tag_id: tag_id).all
+        end
+
+        # Delete all tags for a section
+        def delete_all_for_section(section_id)
+          where(section_id: section_id).delete
+        end
+
+        # Delete all sections for a tag
+        def delete_all_for_tag(tag_id)
+          where(tag_id: tag_id).delete
+        end
+
+        # Bulk create associations
+        def bulk_create(associations)
+          db.transaction do
+            dataset.multi_insert(associations)
+          end
+        end
+
+        # Check if section has a specific tag
+        def has_tag?(section_id, tag_id)
+          where(section_id: section_id, tag_id: tag_id).count > 0
+        end
+      end
+
+      # Instance methods
+
+      # String representation
+      def to_s
+        "<SectionTag: section:#{section_id} => tag:#{tag_id}>"
+      end
+    end
+  end
+end

data/lib/smart_rag/models/source_document.rb

@@ -0,0 +1,204 @@
+require_relative "model_base"
+require "sequel/plugins/validation_helpers"
+
+module SmartRAG
+  module Models
+    # Source document model representing original documents
+    class SourceDocument < Sequel::Model
+      # Set dataset after database is connected
+      def self.set_dataset_from_db
+        set_dataset(:source_documents)
+      end
+      plugin :validation_helpers
+      plugin :timestamps, update_on_create: true
+      include FactoryBotHelpers
+
+      # Constants
+      DOWNLOAD_STATES = {
+        pending: 0,
+        completed: 1,
+        failed: 2
+      }.freeze
+      SOURCE_TYPES = %w[url file manual memory_snapshot other].freeze
+
+      # Relationships
+      one_to_many :sections, class: '::SmartRAG::Models::SourceSection', key: :document_id
+      one_to_many :section_fts, class: '::SmartRAG::Models::SectionFts', key: :document_id
+      one_to_many :section_tags, class: '::SmartRAG::Models::SectionTag', key: :section_id
+
+      # Validation
+      def validate
+        super
+        validates_presence :title
+        validates_integer :download_state, allow_nil: true
+        validates_includes DOWNLOAD_STATES.values, :download_state, allow_nil: true
+        validates_format /\A[a-z]{2}\z/, :language, allow_nil: true, message: 'must be ISO 639-1 code'
+        validates_includes SOURCE_TYPES, :source_type, allow_nil: true
+      end
+
+      # Class methods
+      class << self
+        # Find documents by download state
+        def by_download_state(state)
+          where(download_state: state)
+        end
+
+        # Find completed documents
+        def completed
+          by_download_state(DOWNLOAD_STATES[:completed])
+        end
+
+        # Find pending documents
+        def pending
+          by_download_state(DOWNLOAD_STATES[:pending])
+        end
+
+        # Find failed documents
+        def failed
+          by_download_state(DOWNLOAD_STATES[:failed])
+        end
+
+        # Find documents by language
+        def by_language(lang)
+          where(language: lang)
+        end
+
+        # Search documents by title or description
+        def search(query)
+          where(Sequel.like(:title, "%#{query}%"))
+            .or(Sequel.like(:description, "%#{query}%"))
+        end
+
+        # Order by publication date
+        def order_by_publication_date(direction = :desc)
+          order(Sequel.send(direction, :publication_date))
+        end
+
+        # Recent documents
+        def recent(days: 30)
+          where(Sequel.lit('publication_date >= ?', Date.today - days))
+        end
+
+        # Delete old documents and their sections
+        def delete_old_documents(days: 90)
+          cutoff_date = Time.now - (days * 24 * 60 * 60)
+
+          db.transaction do
+            # Delete related embeddings
+            db[:embeddings].where(source_id: db[:source_sections].select(:id).where(document_id: db.from(:source_documents).where(Sequel.lit('created_at < ?', cutoff_date)).select(:id))).delete
+
+            # Delete section FTS
+            db[:section_fts].where(document_id: db.from(:source_documents).where(Sequel.lit('created_at < ?', cutoff_date)).select(:id)).delete
+
+            # Delete sections
+            db[:source_sections].where(document_id: db.from(:source_documents).where(Sequel.lit('created_at < ?', cutoff_date)).select(:id)).delete
+
+            # Delete documents
+            where(Sequel.lit('created_at < ?', cutoff_date)).delete
+          end
+        end
+
+        # Create or update document
+        def create_or_update(attributes)
+          if attributes[:source_uri] && attributes[:content_hash]
+            existing = where(source_uri: attributes[:source_uri], content_hash: attributes[:content_hash]).first
+            if existing
+              existing.update(attributes)
+              return existing
+            end
+          end
+
+          if existing = find_by_url(attributes[:url])
+            existing.update(attributes)
+            existing
+          else
+            create(attributes)
+          end
+        end
+
+        # Find by URL
+        def find_by_url(url)
+          where(url: url).first
+        end
+
+        # Find by multiple fields
+        def find_by_criteria(criteria)
+          query = self
+          criteria.each do |field, value|
+            query = query.where(field => value)
+          end
+          query.all
+        end
+
+        # Batch insert documents
+        def batch_insert(documents)
+          db.transaction do
+            dataset.multi_insert(documents)
+          end
+        end
+
+        # Update download state
+        def update_download_state(id, state)
+          where(id: id).update(download_state: state, updated_at: Time.now)
+        end
+      end
+
+      # Instance methods
+
+      # Check if document is completed
+      def completed?
+        download_state == DOWNLOAD_STATES[:completed]
+      end
+
+      # Check if document is pending
+      def pending?
+        download_state == DOWNLOAD_STATES[:pending]
+      end
+
+      # Check if document is failed
+      def failed?
+        download_state == DOWNLOAD_STATES[:failed]
+      end
+
+      # Set download state
+      def set_download_state(state)
+        update(download_state: DOWNLOAD_STATES[state])
+      end
+
+      # Get all sections with their embeddings
+      def sections_with_embeddings
+        sections.eager(:embedding).all
+      end
+
+      # Count sections
+      def section_count
+        sections.count
+      end
+
+      # Get document info hash
+      def info
+        {
+          id: id,
+          title: title,
+          url: url,
+          author: author,
+          publication_date: publication_date,
+          language: language,
+          description: description,
+          source_type: source_type,
+          source_uri: source_uri,
+          content_hash: content_hash,
+          download_state: download_state,
+          section_count: section_count,
+          created_at: created_at,
+          updated_at: updated_at
+        }
+      end
+
+      # String representation
+      def to_s
+        "<SourceDocument: #{id} - #{title}>"
+      end
+    end
+  end
+end