deepsearch-rb 0.1.0

data/lib/deepsearch/engine/steps/parallel_search/search.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'async/semaphore'
+
+module Deepsearch
+  class Engine
+    module Steps
+      module ParallelSearch
+        # Performs concurrent web searches for a list of queries using a given search adapter.
+        # It manages concurrency, retries with exponential backoff for failed searches,
+        # and aggregates the unique results.
+        class Search
+          MAX_CONCURRENCY = 2
+          MAX_RETRIES = 1
+          INITIAL_BACKOFF = 1
+
+          def initialize(initial_query, sub_queries, search_adapter, **options)
+            @all_queries = [initial_query] + sub_queries
+            @search_adapter = search_adapter
+            @max_total_search_results = options[:max_total_search_results]
+            @search_options = build_search_options
+            @logger = Deepsearch.configuration.logger
+          end
+
+          def output
+            return [] if @all_queries.empty?
+
+            results = perform_all_searches
+            results.flatten.uniq { |result| result['url'] }
+          end
+
+          private
+
+          def build_search_options
+            return {} unless @max_total_search_results
+
+            max_results_per_search = (@max_total_search_results.to_f / @all_queries.size).ceil
+            { max_results: max_results_per_search }
+          end
+
+          def perform_all_searches
+            @logger.debug("Starting parallel search for #{@all_queries.size} queries with max concurrency of #{MAX_CONCURRENCY}")
+
+            Sync do |task|
+              semaphore = Async::Semaphore.new(MAX_CONCURRENCY, parent: task)
+              
+              tasks = @all_queries.each_with_index.map do |query, index|
+                # Add a small delay for subsequent tasks to avoid overwhelming the search api
+                sleep(1) if index > 0
+                
+                semaphore.async do |sub_task|
+                  sub_task.annotate("query ##{index + 1}: #{query}")
+                  perform_search_with_retries(query, index + 1)
+                end
+              end
+
+              tasks.map(&:wait)
+            end
+          end
+
+          def perform_search_with_retries(query, query_number)
+            (MAX_RETRIES + 1).times do |attempt|
+              @logger.debug("Task #{query_number}: Searching '#{query}' (Attempt #{attempt + 1})")
+              
+              results = @search_adapter.search(query, @search_options)
+              extracted = extract_results(results)
+              @logger.debug("✓ Task #{query_number} completed with #{extracted.size} results for '#{query}'")
+              return extracted
+
+            rescue StandardError => e
+              @logger.debug("✗ Task #{query_number} error for '#{query}': #{e.message}")
+              
+              break if attempt >= MAX_RETRIES
+
+              sleep_duration = (INITIAL_BACKOFF * (2**attempt)) + rand(0.1..0.5)
+              @logger.debug("   Retrying Task #{query_number} in #{sleep_duration.round(2)}s...")
+              sleep(sleep_duration)
+            end
+
+            @logger.debug("✗ Task #{query_number} failed permanently for '#{query}' after #{MAX_RETRIES} retries.")
+            []
+          end
+
+          def extract_results(results)
+            return [] if results.nil?
+            return results unless results.is_a?(Hash)
+
+            results['results'] || results[:results] || []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/prepare_subqueries/process.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'result'
+
+module Deepsearch
+  class Engine
+    module Steps
+      module PrepareSubqueries        
+        class Process
+          def initialize(original_query)
+            @original_query = original_query
+          end
+
+          def execute
+            validate_input
+            process_query
+          rescue StandardError => e
+            PrepareSubqueries::Result.new(
+              cleaned_query: "",
+              sub_queries: [],
+              original_query: @original_query.to_s,
+              error: e.message
+            )
+          end
+
+          private
+
+          def validate_input
+            unless @original_query && !@original_query.strip.empty?
+              raise StandardError, "Original query is required for preprocessing"
+            end
+          end
+
+          def process_query
+            cleaned_query = clean_query(@original_query)
+            subqueries = generate_subqueries(cleaned_query)
+            
+            PrepareSubqueries::Result.new(
+              cleaned_query: cleaned_query,
+              original_query: @original_query,
+              sub_queries: subqueries
+            )
+          end
+
+          def clean_query(query)
+            query.strip.gsub(/\s+/, ' ')
+          end
+
+          def generate_subqueries(query)
+            begin
+              Deepsearch.configuration.logger.debug("Attempting to generate subqueries using LLM...")
+              chat = RubyLLM.chat
+              
+              prompt = Deepsearch.configuration.prompts.subquery_prompt(query: query)
+              Deepsearch.configuration.logger.debug("Sending prompt to LLM...")
+              response = chat.ask(prompt)
+              
+              Deepsearch.configuration.logger.debug("Received response from LLM")
+              subqueries = parse_subqueries(response.content)
+              Deepsearch.configuration.logger.debug("Generated #{subqueries.size} subqueries")
+              subqueries
+            rescue StandardError => e
+              Deepsearch.configuration.logger.debug("Error generating subqueries: #{e.message}")
+              Deepsearch.configuration.logger.debug("Error class: #{e.class}")
+              Deepsearch.configuration.logger.debug("Backtrace: #{e.backtrace.first(3).join('\n')}")
+              []
+            end
+          end
+
+          def parse_subqueries(response_content)
+            return [] unless response_content
+
+            subqueries = response_content.split("\n")
+                                      .map(&:strip)
+                                      .reject(&:empty?)
+                                      .map { |line| line.gsub(/^\d+\.\s*|^[-*]\s*/, '') }
+                                      .map { |query| query.gsub(/^["']|["']$/, '') }
+                                      .reject(&:empty?)
+                                      .first(5)
+
+            subqueries
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/prepare_subqueries/result.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module PrepareSubqueries
+        # Represents the result of the sub-query preparation step.
+        # It holds the cleaned original query, the generated sub-queries, and any potential error.
+        class Result
+          attr_reader :cleaned_query, :sub_queries, :original_query, :error
+
+          def initialize(cleaned_query:, sub_queries:, original_query:, error: nil)
+            @cleaned_query = cleaned_query
+            @sub_queries = sub_queries
+            @original_query = original_query
+            @error = error
+          end
+
+          def success?
+            error.nil?
+          end
+
+          def failure?
+            !success?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/chunker.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        # Splits a large piece of text content into smaller, overlapping chunks.
+        # This is a prerequisite for generating embeddings and performing similarity searches in a RAG pipeline.
+        class Chunker
+          MAX_CHUNK_SIZE = 7500
+          OVERLAP_SIZE = 300
+
+          def chunk(content)
+            return [Values::Chunk.new(text: content)] if content.length <= MAX_CHUNK_SIZE
+
+            chunks = []
+            step = MAX_CHUNK_SIZE - OVERLAP_SIZE
+
+            i = 0
+            while i < content.length
+              chunk_text = content.slice(i, MAX_CHUNK_SIZE)
+              chunks << Values::Chunk.new(text: chunk_text)
+              i += step
+            end
+            chunks
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/process.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative 'values/chunk'
+require_relative 'values/query'
+require_relative 'values/result'
+require_relative 'chunker'
+require_relative 'similarity'
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        # Implements the core Retrieval-Augmented Generation (RAG) logic.
+        # It takes a query and a set of parsed websites, then:
+        # 1. Chunks the website content into smaller pieces.
+        # 2. Generates embeddings for all text chunks in batches.
+        # 3. Uses a similarity search to find the chunks most relevant to the query.
+        # 4. Returns a result containing the relevant chunks.
+        class Process
+          CHUNK_BATCH_SIZE = 100
+          MAX_TOTAL_CHUNKS = 500
+          MAX_CHUNKS_PER_WEBSITE = 15
+
+          def initialize(query:, parsed_websites:)
+            @query = Values::Query.new(text: query)
+            @documents = parsed_websites.map do |website|
+              { url: website.url, content: website.content }
+            end
+          end
+
+          def execute
+            begin
+              chunker = Chunker.new
+              all_chunks = @documents.each_with_object([]) do |doc, chunks|
+                next if doc[:content].to_s.strip.empty?
+
+                doc_chunks = chunker.chunk(doc[:content])
+                if doc_chunks.count > MAX_CHUNKS_PER_WEBSITE
+                  Deepsearch.configuration.logger.debug("Truncating chunks for #{doc[:url]} from #{doc_chunks.count} to #{MAX_CHUNKS_PER_WEBSITE}")
+                  doc_chunks = doc_chunks.first(MAX_CHUNKS_PER_WEBSITE)
+                end
+                doc_chunks.each { |chunk| chunk.document_url = doc[:url] }
+                chunks.concat(doc_chunks)
+              end
+
+              Deepsearch.configuration.logger.debug("Chunked #{@documents.count} documents into #{all_chunks.count} chunks")
+
+              if all_chunks.count > MAX_TOTAL_CHUNKS
+                Deepsearch.configuration.logger.debug("Chunk count (#{all_chunks.count}) exceeds limit of #{MAX_TOTAL_CHUNKS}. Truncating.")
+                all_chunks = all_chunks.first(MAX_TOTAL_CHUNKS)
+              end
+
+              all_chunks.each_slice(CHUNK_BATCH_SIZE) do |batch|
+                texts = batch.map(&:text)
+                embeddings = RubyLLM.embed(texts).vectors
+                batch.each_with_index { |chunk, i| chunk.embedding = embeddings[i] }
+              end
+
+              Deepsearch.configuration.logger.debug("Generated embeddings for #{all_chunks.count} chunks, initiating similarity match..")
+              relevant_chunks = Similarity.new.find_relevant(@query, all_chunks)
+              Deepsearch.configuration.logger.debug("Found #{relevant_chunks.count} relevant chunks for query: '#{@query.text}'")
+
+              Values::Result.new(
+                query: @query,
+                relevant_chunks: relevant_chunks
+              )
+            rescue StandardError => e
+              Values::Result.new(
+                query: @query,
+                relevant_chunks: [],
+                error: e.message
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/similarity.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        # Calculates and filters text chunks based on their semantic similarity to a query.
+        # It uses cosine similarity to score chunks against a query embedding and employs a two-step
+        # filtering process: first, it retrieves a fixed number of top candidates (top-k), and
+        # second, it filters these candidates based on a score relative to the best-scoring chunk.
+        class Similarity
+          TOP_K_CANDIDATES = 75
+          RELATIVE_SCORE_THRESHOLD = 0.85
+
+          def find_relevant(query, chunks, threshold: RELATIVE_SCORE_THRESHOLD)
+            return [] if chunks.empty?
+
+            similarities = calculate(chunks.map(&:embedding), query.embedding)
+
+            top_candidates = top_k_with_scores(similarities, TOP_K_CANDIDATES)
+
+            return [] if top_candidates.empty?
+
+            best_score = top_candidates.first.first
+            cutoff_score = best_score * threshold
+
+            relevant_chunks = top_candidates.select { |score, _| score >= cutoff_score }
+                                            .map { |_, index| chunks[index] }
+
+            relevant_chunks
+          end
+
+          private
+
+          def calculate(embeddings, query_embedding)
+            embeddings.map { |embedding| cosine_similarity(embedding, query_embedding) }
+          end
+
+          def top_k_with_scores(similarities, k)
+            similarities.each_with_index
+                        .sort_by { |score, _| -score }
+                        .first(k)
+          end
+
+          def cosine_similarity(vec_a, vec_b)
+            return 0.0 unless vec_a.is_a?(Array) && vec_b.is_a?(Array)
+            return 0.0 if vec_a.empty? || vec_b.empty?
+
+            dot_product = vec_a.zip(vec_b).sum { |a, b| a * b }
+            magnitude_a = Math.sqrt(vec_a.sum { |v| v**2 })
+            magnitude_b = Math.sqrt(vec_b.sum { |v| v**2 })
+
+            return 0.0 if magnitude_a.zero? || magnitude_b.zero?
+            dot_product / (magnitude_a * magnitude_b)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/values/chunk.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        module Values
+          # Represents a chunk of text from a document, along with its embedding and source URL.
+          # This is the fundamental unit of data used in the RAG process.
+          class Chunk
+            attr_accessor :text, :embedding, :document_url
+      
+            def initialize(text:, embedding: nil, document_url: nil)
+              @text = text
+              @embedding = embedding
+              @document_url = document_url
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/values/query.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        module Values
+          # Represents a user query that has been prepared for the RAG process.
+          # It enriches the original query text with LLM-generated tags to improve
+          # embedding quality and then computes the embedding vector.
+          class Query
+            attr_reader :text, :embedding
+
+            def initialize(text:)
+              raise ArgumentError, "Query text cannot be blank" if text.to_s.strip.empty?
+
+              @text = text
+              enriched_text = enrich_query_with_tags(text)
+              @embedding = RubyLLM.embed(enriched_text).vectors
+            end
+
+            private
+
+            def enrich_query_with_tags(original_text)
+              prompt = Deepsearch.configuration.prompts.enrich_query_prompt(query: original_text)
+
+              Deepsearch.configuration.logger.debug("Enriching query with LLM-generated tags...")
+              response = RubyLLM.chat.ask(prompt)
+              tags_list = response.content.strip
+              Deepsearch.configuration.logger.debug("Generated tags for query enrichment: #{tags_list}")
+
+              enriched_text = "#{original_text} - related concepts: #{tags_list}"
+              Deepsearch.configuration.logger.debug("Enriched query for embedding: \"#{enriched_text}\"")
+              enriched_text
+            rescue StandardError => e
+              Deepsearch.configuration.logger.debug("Failed to enrich query due to '#{e.message}'. Using original query for embedding.")
+              original_text
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/rag/values/result.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Rag
+        module Values
+          # Represents the result of the RAG processing step.
+          # It contains the original query object and a list of text chunks
+          # deemed most relevant to the query.
+          class Result
+            attr_reader :query, :relevant_chunks, :error, :success
+
+            def initialize(query: nil, relevant_chunks: [], error: nil)
+              @query = query
+              @relevant_chunks = relevant_chunks
+              @success = error.nil?
+              @error = error
+            end
+
+            def success?
+              @success
+            end
+
+            def failure?
+              !success?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine/steps/summarization/process.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'values/result'
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Summarization
+        # Generates a final, synthesized answer to the user's query based on relevant text chunks.
+        # It constructs a detailed prompt for an LLM, including the query, context from chunks,
+        # and instructions for citing sources, then returns the LLM's response.
+        class Process
+          attr_reader :query, :relevant_chunks
+
+          def initialize(query:, relevant_chunks:)
+            @query = query
+            @relevant_chunks = relevant_chunks
+          end
+
+          def execute
+            return Values::Result.new(summary: "No relevant content found to summarize.") if relevant_chunks.empty?
+
+            prompt = build_summary_prompt
+            Deepsearch.configuration.logger.debug("Summarizing content with LLM...")
+            response = RubyLLM.chat.ask(prompt)
+            Deepsearch.configuration.logger.debug("Summarization complete.")
+
+            Values::Result.new(summary: response.content)
+          rescue StandardError => e
+            Deepsearch.configuration.logger.debug("Error during summarization: #{e.message}")
+            Values::Result.new(summary: nil, error: e.message)
+          end
+
+          private
+
+          def build_summary_prompt
+            chunks_by_url = relevant_chunks.group_by(&:document_url)
+            citation_map = chunks_by_url.keys.each_with_index.to_h { |url, i| [url, i + 1] }
+ 
+            context_text = chunks_by_url.map do |url, chunks|
+              citation_number = citation_map[url]
+              chunk_contents = chunks.map(&:text).join("\n\n")
+              "Source [#{citation_number}]:\n#{chunk_contents}"
+            end.join("\n\n---\n\n")
+ 
+            sources_list = citation_map.map { |url, number| "[#{number}]: #{url}" }.join("\n")
+            Deepsearch.configuration.prompts.summarization_prompt(query: @query.text, context_text: context_text, sources_list: sources_list)
+          end
+        end
+      end
+    end
+  end
+end          

data/lib/deepsearch/engine/steps/summarization/values/result.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  class Engine
+    module Steps
+      module Summarization
+        module Values
+          # Represents the result of the summarization step.
+          # It holds the final, synthesized summary and any potential error message.
+          class Result
+            attr_reader :summary, :error, :success
+
+            def initialize(summary: nil, error: nil)
+              @summary = summary
+              @success = error.nil?
+              @error = error
+            end
+
+            def success?
+              @success
+            end
+
+            def failure?
+              !success?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deepsearch/engine.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "engine/pipeline"
+
+module Deepsearch
+  # The main entry point for performing a deep search.
+  # This class initializes the search pipeline with the configured or specified
+  # search adapter and provides a `search` method to execute the query.
+  class Engine
+    attr_reader :pipeline
+
+    def initialize(adapter_type: nil)
+      adapter_source = adapter_type ||
+                       Deepsearch.configuration.custom_search_adapter_class ||
+                       Deepsearch.configuration.search_adapter
+
+      search_adapter = Deepsearch::SearchAdapters.create(adapter_source)
+      @pipeline = Engine::Pipeline.new(search_adapter)
+    end
+
+    def search(query, **options)
+      @pipeline.execute(query, **options)
+    end
+  end
+end

data/lib/deepsearch/logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'forwardable'
+
+module Deepsearch
+  # A custom logger that wraps Ruby's standard `::Logger` to provide a default format.
+  class Logger
+    extend Forwardable
+
+    def_delegators :@logger, :debug, :level=, :level, :progname=, :progname, :formatter=, :formatter
+
+    # Re-exporting constants from ::Logger for compatibility.
+    DEBUG = ::Logger::DEBUG
+
+    def initialize(logdev, level: DEBUG, progname: 'DeepSearch', formatter: nil)
+      @logger = ::Logger.new(logdev)
+      @logger.level = level
+      @logger.progname = progname
+      @logger.formatter = formatter || default_formatter
+    end
+
+    private
+
+    def default_formatter
+      proc do |severity, datetime, progname, msg|
+        formatted_time = datetime.strftime('%Y-%m-%d %H:%M:%S.%L')
+        "[#{formatted_time}] #{severity.ljust(5)} -- #{progname}: #{msg}\n"
+      end
+    end
+  end
+end