smart_rag 0.1.0

data/examples/05_advanced_patterns.rb

@@ -0,0 +1,154 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require_relative "common"
+
+include Examples::Common
+
+class ContextualSearch
+  def initialize(smart_rag)
+    @smart_rag = smart_rag
+  end
+
+  def search_with_context(query, user_context = {})
+    enhanced_query = enhance_query(query, user_context)
+    filters = build_filters(user_context)
+
+    @smart_rag.search(
+      enhanced_query,
+      search_type: "hybrid",
+      limit: 10,
+      filters: filters,
+    )
+  end
+
+  private
+
+  def enhance_query(query, context)
+    case context[:domain]
+    when "healthcare"
+      "#{query} medical clinical"
+    when "finance"
+      "#{query} finance banking"
+    else
+      query
+    end
+  end
+
+  def build_filters(context)
+    filters = {}
+    filters[:document_ids] = context[:document_ids] if context[:document_ids]
+    filters[:tag_ids] = context[:preferred_tags] if context[:preferred_tags]
+    filters
+  end
+end
+
+class SearchPipeline
+  def initialize(smart_rag)
+    @smart_rag = smart_rag
+    @processors = []
+  end
+
+  def add_processor(&block)
+    @processors << block
+    self
+  end
+
+  def search(query, options = {})
+    results = @smart_rag.search(query, options)
+    @processors.each { |processor| results = processor.call(results, query, options) }
+    results
+  end
+end
+
+class MemorySearchCache
+  def initialize(ttl_seconds: 300)
+    @ttl_seconds = ttl_seconds
+    @store = {}
+  end
+
+  def fetch(query, options)
+    key = cache_key(query, options)
+    cached = @store[key]
+    if cached && cached[:expires_at] > Time.now
+      return cached[:payload]
+    end
+
+    payload = yield
+    @store[key] = { payload: payload, expires_at: Time.now + @ttl_seconds }
+    payload
+  end
+
+  private
+
+  def cache_key(query, options)
+    Digest::MD5.hexdigest("#{query}:#{options.sort_by { |k, _| k.to_s }.to_h.to_json}")
+  end
+end
+
+class QASystem
+  def initialize(smart_rag)
+    @smart_rag = smart_rag
+  end
+
+  def answer(question, context_limit: 5)
+    search_results = @smart_rag.search(
+      question,
+      search_type: "hybrid",
+      limit: context_limit,
+      include_content: true,
+    )
+    results = search_results.fetch(:results, [])
+
+    {
+      question: question,
+      answer: generate_answer(results),
+      sources: results.map { |r| { section_id: r[:section_id], title: r[:section_title] } },
+      confidence: results.empty? ? 0.0 : [results.first[:combined_score].to_f, 1.0].min,
+    }
+  end
+
+  private
+
+  def generate_answer(results)
+    return "I do not have enough information in the current knowledge base." if results.empty?
+
+    context = results.map { |r| r[:content].to_s }.join("\n---\n")
+    "Draft answer from retrieved context: #{context[0, 400]}..."
+  end
+end
+
+print_header("Advanced Patterns")
+smart_rag = build_client
+
+contextual = ContextualSearch.new(smart_rag)
+contextual_results = contextual.search_with_context(
+  "risk assessment",
+  domain: "finance",
+  document_ids: [],
+  preferred_tags: [],
+)
+puts "Contextual search results: #{contextual_results.fetch(:results, []).length}"
+
+pipeline = SearchPipeline.new(smart_rag)
+pipeline.add_processor do |results, _query, options|
+  min_score = options[:min_score] || 0.5
+  results[:results] = results.fetch(:results, []).select do |r|
+    (r[:combined_score] || r[:similarity] || 0.0) >= min_score
+  end
+  results
+end
+pipeline_results = pipeline.search("neural networks", search_type: "hybrid", limit: 20, min_score: 0.7)
+puts "Pipeline filtered results: #{pipeline_results.fetch(:results, []).length}"
+
+cache = MemorySearchCache.new(ttl_seconds: 600)
+cached_1 = cache.fetch("deep learning", { limit: 5 }) { smart_rag.search("deep learning", limit: 5) }
+cached_2 = cache.fetch("deep learning", { limit: 5 }) { smart_rag.search("deep learning", limit: 5) }
+puts "Cache demo result sizes: #{cached_1.fetch(:results, []).length}, #{cached_2.fetch(:results, []).length}"
+
+qa = QASystem.new(smart_rag)
+qa_response = qa.answer("What are common applications of transformers in NLP?", context_limit: 3)
+print_json("Q&A Response", qa_response)
+

data/examples/06_error_handling_and_retry.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "common"
+
+include Examples::Common
+
+class RetryableSmartRAG
+  def initialize(smart_rag, retries: 3, base_interval: 0.5)
+    @smart_rag = smart_rag
+    @retries = retries
+    @base_interval = base_interval
+  end
+
+  def add_document(path, **options)
+    with_retry(on: [SmartRAG::Errors::EmbeddingGenerationError]) do
+      @smart_rag.add_document(path, options)
+    end
+  end
+
+  def search(query, **options)
+    with_retry(on: [SmartRAG::Errors::DatabaseError]) do
+      @smart_rag.search(query, options)
+    end
+  end
+
+  private
+
+  def with_retry(on:)
+    attempt = 0
+    begin
+      attempt += 1
+      yield
+    rescue *on => e
+      raise e if attempt >= @retries
+
+      sleep @base_interval * (2**(attempt - 1))
+      retry
+    end
+  end
+end
+
+print_header("Error Handling And Retry")
+smart_rag = build_client
+retryable = RetryableSmartRAG.new(smart_rag)
+
+begin
+  # Intentional invalid call to show argument error handling.
+  smart_rag.search("", search_type: "hybrid")
+rescue SmartRAG::Errors::InvalidParameterError, SmartRAG::Errors::InvalidQueryError, ArgumentError => e
+  puts "Argument error: #{e.message}"
+rescue SmartRAG::Errors::DatabaseError => e
+  puts "Database error: #{e.message}"
+rescue SmartRAG::Errors::EmbeddingGenerationError => e
+  puts "Embedding generation error: #{e.message}"
+rescue SmartRAG::Errors::DocumentProcessingError => e
+  puts "Document processing error: #{e.message}"
+rescue StandardError => e
+  puts "Unexpected error: #{e.class} - #{e.message}"
+end
+
+safe_query = ARGV[0] || "machine learning"
+safe_result = retryable.search(safe_query, search_type: "hybrid", limit: 5)
+puts "Retry search result count: #{safe_result.fetch(:results, []).length}"

data/examples/README.md

@@ -0,0 +1,42 @@
+# SmartRAG Examples
+
+These examples are extracted and organized from `docs/USAGE_EXAMPLES.md`.
+
+## Prerequisites
+
+1. Install dependencies:
+   - `bundle install`
+2. Configure environment variables (or rely on defaults in `examples/common.rb`):
+   - `SMARTRAG_DB_HOST`
+   - `SMARTRAG_DB_NAME`
+   - `SMARTRAG_DB_USER`
+   - `SMARTRAG_DB_PASSWORD`
+   - `OPENAI_API_KEY`
+3. Ensure your database is migrated and has test data if needed.
+
+## Run Examples
+
+- Quick start:
+  - `ruby examples/01_quick_start.rb`
+- Document management:
+  - `ruby examples/02_document_management.rb test/python_basics.md`
+  - Optional delete demo:
+  - `DELETE=1 ruby examples/02_document_management.rb test/python_basics.md`
+- Search operations:
+  - `ruby examples/03_search_operations.rb`
+- Topics and tags:
+  - `ruby examples/04_topics_and_tags.rb`
+- Advanced patterns:
+  - `ruby examples/05_advanced_patterns.rb`
+- Error handling and retry:
+  - `ruby examples/06_error_handling_and_retry.rb`
+
+## Mapping To `docs/USAGE_EXAMPLES.md`
+
+- `01_quick_start.rb`: Quick Start
+- `02_document_management.rb`: Document Management
+- `03_search_operations.rb`: Search Operations
+- `04_topics_and_tags.rb`: Research Topic Management + Tag Management
+- `05_advanced_patterns.rb`: Advanced Usage Patterns + Q&A + Caching
+- `06_error_handling_and_retry.rb`: Error Handling + Retry Logic
+- `common.rb`: shared setup/config/logging utilities

data/examples/common.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "logger"
+require "dotenv"
+require_relative "../lib/smart_rag"
+
+module Examples
+  module Common
+    module_function
+
+    def load_env!
+      Dotenv.load(".env.local", ".env")
+    rescue StandardError
+      # Keep examples usable even if dotenv loading fails unexpectedly.
+    end
+
+    def default_config
+      load_env!
+      {
+        database: {
+          adapter: "postgresql",
+          host: ENV["SMARTRAG_DB_HOST"] || "localhost",
+          port: (ENV["SMARTRAG_DB_PORT"] || "5432").to_i,
+          database: ENV["SMARTRAG_DB_NAME"] || "smart_rag_development",
+          user: ENV["SMARTRAG_DB_USER"] || "smart_rag_user",
+          password: ENV["SMARTRAG_DB_PASSWORD"],
+        },
+        llm: {
+          provider: ENV["SMARTRAG_LLM_PROVIDER"] || ENV["LLM_PROVIDER"] || "openai",
+          api_key: ENV["OPENAI_API_KEY"] || ENV["LLM_API_KEY"] || "ollama-local",
+          endpoint: ENV["LLM_ENDPOINT"] || "http://localhost:11434/v1/chat/completions",
+          model: ENV["LLM_MODEL"] || "qwen3",
+        },
+      }
+    end
+
+    def build_client(log_level: Logger::INFO)
+      client = SmartRAG::SmartRAG.new(default_config)
+      client.logger = Logger.new($stdout)
+      client.logger.level = log_level
+      client
+    end
+
+    def print_header(title)
+      puts "\n#{'=' * 72}"
+      puts title
+      puts "=" * 72
+    end
+
+    def print_json(name, data)
+      require "json"
+      puts "#{name}:"
+      puts JSON.pretty_generate(data)
+    end
+  end
+end

data/lib/smart_rag/chunker/markdown_chunker.rb

@@ -0,0 +1,315 @@
+module SmartRAG
+  module Chunker
+    # MarkdownChunker splits markdown content into chunks based on headings
+    class MarkdownChunker
+      attr_reader :chunk_size, :overlap, :heading_levels
+
+      # @param [Integer] chunk_size Target size for each chunk (in characters)
+      # @param [Integer] overlap Overlap between chunks (in characters)
+      # @param [Array<Integer>] heading_levels Which heading levels to split on (default: [1, 2, 3])
+      def initialize(chunk_size: 2000, overlap: 200, heading_levels: [1, 2, 3])
+        @chunk_size = chunk_size
+        @overlap = overlap
+        @heading_levels = heading_levels
+      end
+
+      # Split markdown content into chunks
+      # @param [String] markdown_content Content to chunk
+      # @return [Array<Hash>] Array of chunk hashes with :title and :content
+      def chunk(markdown_content)
+        return [] if markdown_content.nil? || markdown_content.empty?
+
+        # First, try to split by headings
+        heading_chunks = split_by_headings(markdown_content)
+
+        # If we have heading-based chunks with actual headings, use them
+        # Otherwise, fall back to size-based splitting
+        if heading_chunks.length > 0 && heading_chunks.first[:title]
+          # Further split large heading chunks by size
+          result = []
+          heading_chunks.each do |chunk|
+            if chunk[:content].length > chunk_size * 1.5
+              # Split large chunk further
+              sub_chunks = split_large_chunk(chunk)
+              result.concat(sub_chunks)
+            else
+              result << chunk
+            end
+          end
+          result
+        else
+          # No headings found, use size-based splitting
+          split_by_size(markdown_content)
+        end
+      end
+
+      private
+
+      # Split content by markdown headings - Handles nested headings properly
+      def split_by_headings(content)
+        chunks = []
+        content = content.sub(/\A\uFEFF/, '')
+        lines = content.lines
+        return chunks if lines.empty?
+
+        # Check if we have any headings
+        heading_pattern = /^(#{@heading_levels.map { |l| '#' * l }.join('|')})\s+(.+)$/
+
+        # Find all heading positions first
+        heading_positions = []
+        lines.each_with_index do |line, idx|
+          if line.match?(heading_pattern)
+            heading_positions << idx
+          end
+        end
+
+        # If no headings, use fallback to size-based splitting
+        if heading_positions.empty?
+          cleaned = clean_chunk_content(content)
+          return cleaned.length > 10 ? [{ title: nil, content: cleaned }] : []
+        end
+
+        # Build chunks based on heading hierarchy
+        # Split at any heading that matches our configured heading levels
+        current_chunk_start = nil
+        current_chunk_title = nil
+        first_h1_processed = false
+        first_h1_title = nil
+        first_h1_skipped = false
+        intro_lines = []
+        intro_text = nil
+        intro_pending = false
+
+        lines.each_with_index do |line, idx|
+          # Check if this line is a heading
+          if (match = line.match(heading_pattern))
+            heading_level = match[1].length
+            heading_title = match[2].strip
+
+            # Handle the first H1 - treat as document title (skip for chunking)
+            if !first_h1_processed && heading_level == 1
+              first_h1_title = heading_title
+              first_h1_processed = true
+              first_h1_skipped = true
+              next
+            end
+
+            # Check if this heading level should create a chunk
+            if @heading_levels.include?(heading_level) && !(idx == 0 && heading_level == 1)
+              # Save previous chunk if it exists
+              if current_chunk_start && current_chunk_title
+                # Extract content from start of this section to current line
+                content_lines = lines[current_chunk_start...idx]
+                if content_lines && content_lines.length > 1
+                  # Remove the heading line
+                  section_content = clean_chunk_content(content_lines[1..-1].join)
+                  if intro_pending && intro_text && section_content.length > 0
+                    section_content = [intro_text, section_content].join("\n\n")
+                    intro_pending = false
+                  elsif intro_pending && intro_text && section_content.empty?
+                    section_content = intro_text
+                    intro_pending = false
+                  end
+                  if section_content.length > 0
+                    chunks << { title: current_chunk_title, content: section_content }
+                  end
+                end
+              end
+
+              if first_h1_skipped && current_chunk_start.nil? && intro_text.nil?
+                cleaned_intro = clean_chunk_content(intro_lines.join)
+                if cleaned_intro.length > 0
+                  intro_text = cleaned_intro
+                  intro_pending = true
+                end
+              end
+
+              # Start new chunk
+              current_chunk_start = idx
+              current_chunk_title = heading_title
+            end
+          elsif first_h1_skipped && current_chunk_start.nil?
+            intro_lines << line
+          end
+        end
+
+        # Save the last chunk
+        if current_chunk_start && current_chunk_title
+          end_idx = heading_positions[heading_positions.index(current_chunk_start) + 1] || lines.length
+          content_lines = lines[current_chunk_start...end_idx]
+          if content_lines && content_lines.length > 1
+            section_content = clean_chunk_content(content_lines[1..-1].join)
+            if intro_pending && intro_text && section_content.length > 0
+              section_content = [intro_text, section_content].join("\n\n")
+              intro_pending = false
+            elsif intro_pending && intro_text && section_content.empty?
+              section_content = intro_text
+              intro_pending = false
+            end
+            if section_content.length > 0
+              chunks << { title: current_chunk_title, content: section_content }
+            end
+          end
+        end
+
+        # If we skipped the first h1 as a document title but no chunks were created,
+        # go back and create a chunk for it
+        if first_h1_skipped && chunks.empty? && first_h1_title
+          # Get content after the first h1
+          if lines.length > 1
+            content_after_h1 = lines[1..-1].join
+            cleaned_content = clean_chunk_content(content_after_h1)
+            if cleaned_content.length > 0
+              chunks << { title: first_h1_title, content: cleaned_content }
+            end
+          end
+        end
+
+        chunks
+      end
+
+      # Clean chunk content by removing heading lines
+      def clean_chunk_content(content)
+        return "" if content.nil? || content.empty?
+
+        content = content.strip
+        return "" if content.empty?
+
+        lines = content.lines
+
+        # Remove leading heading lines
+        while lines.first && lines.first =~ /^#+\s+/
+          lines.shift
+        end
+
+        result = lines.join.strip
+        result
+      end
+
+      # SPLIT_CONTENT
+
+      # Split content by size without considering headings
+      def split_by_size(content)
+        chunks = []
+        return chunks if content.nil? || content.empty?
+
+        position = 0
+        chunk_index = 0
+        content_length = content.length
+
+        while position < content_length
+          # Calculate the end position for this chunk
+          end_pos = [position + chunk_size, content_length].min
+
+          # Look backward for a sentence boundary if we're not at the end
+          if end_pos < content_length
+            # Search backward for a sentence boundary (., !, ?)
+            search_start = end_pos
+            search_end = [position + chunk_size * 0.8, content_length].min # Don't look too far back
+
+            (search_start - 1).downto(search_end.to_i) do |i|
+              if content[i] =~ /[.!?]/
+                # Found a sentence boundary
+                # Make sure it's followed by whitespace or end of content
+                if i + 1 >= content_length || content[i + 1] =~ /\s/
+                  end_pos = i + 1
+                  break
+                end
+              end
+            end
+          end
+
+          # Extract the chunk
+          chunk_content = content[position...end_pos].strip
+
+          if chunk_content.length > 50
+            chunks << {
+              title: generate_chunk_title(chunk_content, chunk_index),
+              content: chunk_content
+            }
+            chunk_index += 1
+          end
+
+          # Move position forward, keeping overlap
+          next_position = end_pos
+          if next_position < content_length
+            # Add overlap
+            overlap_chars = [overlap, chunk_content.length].min
+            position = [next_position - overlap_chars, position + 50].max # Don't overlap too much
+          else
+            position = content_length
+          end
+        end
+
+        chunks
+      end
+
+      # Split a large chunk that exceeded size limits
+      def split_large_chunk(chunk)
+        # Use the heading as title and split the content
+        sub_chunks = split_by_size(chunk[:content])
+
+        # Prepend the original heading to each sub-chunk
+        sub_chunks.each_with_index do |sub_chunk, i|
+          sub_chunk[:title] = if i == 0
+                                chunk[:title]
+                              else
+                                "#{chunk[:title]} (Part #{i + 1})"
+                              end
+        end
+
+        sub_chunks
+      end
+
+      # Generate a title for a chunk based on its content
+      def generate_chunk_title(content, index)
+        return nil if content.nil? || content.empty?
+
+        # Try to extract first heading (if present)
+        lines = content.strip.lines
+        lines.each do |line|
+          if line =~ /^#+\s+(.+)$/
+            return $1.strip
+          end
+        end
+
+        # Use first sentence
+        first_line = lines.first&.strip
+        if first_line && first_line =~ /^(.*?[.!?])/
+          sentence = $1.strip
+          return sentence.length > 100 ? sentence[0...100] + '...' : sentence
+        elsif first_line && !first_line.empty?
+          # Just use first line (without period)
+          return first_line.length > 100 ? first_line[0...100] + '...' : first_line
+        end
+
+        # Fallback to chunk number
+        "Section #{index + 1}"
+      end
+    end
+  end
+
+  # Monkey-patch String to add reverse match method
+  class ::String
+    # Find last match of pattern in string
+    # @param [Regexp] pattern Pattern to match
+    # @param [Integer] start_pos Position to start searching from (default: end of string)
+    # @return [MatchData] Last match or nil
+    def rmatch(pattern, start_pos = nil)
+      start_pos ||= length
+      start_pos = length + start_pos if start_pos < 0
+      start_pos = length if start_pos > length
+
+      # Search backwards from start_pos
+      (0..start_pos).reverse_each do |i|
+        # Try to match at position i
+        substring = self[i...length]
+        match = pattern.match(substring)
+        return match if match
+      end
+      nil
+    rescue
+      nil
+    end
+  end
+end