deepsearch-rb 0.1.0

data/lib/deepsearch/prompts_config.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  # Defines the default prompts used in various steps of the Deepsearch pipeline.
+  # Users can provide their own prompt configuration object to customize these prompts.
+  # The custom object should respond to the same methods as this class.
+  class PromptsConfig
+    # @param query [String] The original user query.
+    # @return [String] The prompt for generating subqueries.
+    def subquery_prompt(query:)
+      <<~PROMPT
+        You are a search query expansion expert. Given a search query, generate 3-5 related subqueries that would help find more comprehensive information about the topic.
+
+        Original query: "#{query}"
+
+        Please generate subqueries that:
+        1. Cover different aspects of the topic
+        2. Use varied terminology and synonyms
+        3. Are specific enough to be useful but broad enough to capture relevant information
+        4. Focus on different angles (technical, practical, conceptual, etc.)
+
+        Format your response as a simple list, one subquery per line, without numbering or bullet points.
+      PROMPT
+    end
+
+    # @param query [String] The user query to be enriched.
+    # @return [String] The prompt for enriching the query with tags for embedding.
+    def enrich_query_prompt(query:)
+      <<~PROMPT
+        You are a search query expansion assistant. Based on the user's query, generate 5 to 7 highly relevant tags or keywords that expand its core concepts. These tags are used to improve vector search results.
+
+        INSTRUCTIONS:
+        - Do NOT add any explanation, numbering, or introductory text.
+        - Provide ONLY a single line of comma-separated tags.
+
+        USER QUERY: "#{query}"
+
+        TAGS:
+      PROMPT
+    end
+
+    # @param query [String] The original user query.
+    # @param context_text [String] The aggregated text from relevant sources.
+    # @param sources_list [String] The formatted list of sources for citation.
+    # @return [String] The prompt for summarizing the context and answering the query.
+    def summarization_prompt(query:, context_text:, sources_list:)
+      <<~PROMPT
+        You are a research assistant. Your task is to synthesize a comprehensive answer to the user's query based *only* on the provided text snippets.
+
+        User Query: "#{query}"
+
+        Provided Context:
+        ---
+        #{context_text}
+        ---
+
+        Sources:
+        #{sources_list}
+
+        Instructions:
+        0.  Format: Markdown
+        1.  Carefully read the user's query and all the provided context from the sources.
+        2.  When you use information from a source, you MUST cite it using the following markdown link format: `[<source_number>](<source_link>)`.
+            *   **To find the `<source_link>`:** The 'Sources' section provides a numbered list where each source is in the format `[<number>]: <url>`. Extract this URL for the specific source number you are citing.
+            *   **Example Citation:** If a fact comes from source number 1, and the 'Sources' section lists `[1]: https://example.com/source1`, your citation should be `[1](https://example.com/source1)`.
+            *   **Placement:** Place citations immediately after the sentence or clause where the information is used.
+            *   **Multiple Sources:** If a single statement draws information from multiple sources, cite each source individually, for example: `[1](https://link1.com) [3](https://link3.com)`.
+            *   **Crucial:** Always use the exact format `[<source_number>](<source_link>)` with square brackets around the number.
+        3.  Structure your response with clear headings and bullet points where appropriate.
+        4.  If the provided context does not contain enough information to answer the query, state that clearly: "I cannot answer this query based on the provided sources."
+        5.  At the end of your response, include a "## References" section that lists all used sources in the following format:
+            ```
+            ## References
+            1. [Source Title](<source_link>)
+            2. [Another Source](<another_link>)
+            ```
+            Where possible, extract meaningful titles from the context or use the domain name if no title is available.
+        6.  Do not include sources in the References section that you didn't actually cite in your response.
+      PROMPT
+    end
+  end
+end

data/lib/deepsearch/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  VERSION = "0.1.0"
+end

data/lib/deepsearch.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "deepsearch/version"
+require_relative "deepsearch/logger"
+require_relative "deepsearch/configuration"
+require_relative "deepsearch/engine"
+require_relative "search_adapters"
+
+# The main module for the Deepsearch gem.
+# It provides the primary interface for configuration and for initiating searches.
+module Deepsearch
+  # A generic error class for exceptions raised by the Deepsearch gem,
+  # from which more specific errors can inherit.
+  class Error < StandardError; end
+  
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+      configuration.configure_llm!
+    end
+
+    def reset_configuration!
+      @configuration = Configuration.new
+    end
+
+    def search(query, adapter_type: nil, **options)
+      engine = Engine.new(adapter_type: adapter_type)
+      engine.search(query, **options)
+    end
+
+    def engine(adapter_type: nil)
+      Engine.new(adapter_type: adapter_type)
+    end
+  end
+end

data/lib/search_adapters/mock_adapter.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Deepsearch
+  module SearchAdapters
+    # A mock search adapter for testing purposes.
+    # It returns a static, predefined set of search results without making any external API calls.
+    # This is useful for testing the pipeline's behavior in isolation from live search services.
+    class MockAdapter
+      def initialize(api_key = nil); end
+
+      def search(query, options = {})
+        return mock_results(query, options)
+      end
+
+      private
+
+      def mock_results(query, options = {})
+        max_results = options[:max_results] || 10
+        include_answer = options[:include_answer] != false
+
+        results = [
+          {
+            "title" => "Ruby (programming language) - Wikipedia",
+            "url" => "https://en.wikipedia.org/wiki/Ruby_(programming_language)",
+            "content" => "Ruby is a general-purpose programming language. It was designed with an emphasis on programming productivity and simplicity. In Ruby, everything is an object, including primitive data types. It was developed in the mid-1990s by Yukihiro Matsumoto in Japan.",
+            "published_date" => "2024-01-15"
+          },
+          {
+            "title" => "Ruby Programming Language - GeeksforGeeks",
+            "url" => "https://www.geeksforgeeks.org/ruby/ruby-programming-language/",
+            "content" => "Ruby is a dynamic, reflective, object-oriented, general-purpose programming language. Ruby is a pure object-oriented language developed by Yukihiro Matsumoto. Everything in Ruby is an object except the blocks but there are replacements too.",
+            "published_date" => "2024-02-10"
+          },
+          {
+            "title" => "Ruby Programming Language",
+            "url" => "https://www.ruby-lang.org/en/",
+            "content" => "# Ruby # Ruby is... A dynamic, open source programming language with a focus on natural to read and easy to write code. It has an elegant syntax that is natural to read and easy to write.",
+            "published_date" => "2024-03-01"
+          },
+          {
+            "title" => "About Ruby",
+            "url" => "https://www.ruby-lang.org/en/about/",
+            "content" => "About Ruby About Ruby About Ruby's Growth In Ruby, everything is an object. ruby Ruby follows the information-hiding principle where the implementation details are hidden and only the necessary information is exposed.",
+            "published_date" => "2024-02-20"
+          },
+          {
+            "title" => "What's Ruby used for most nowadays? - Reddit",
+            "url" => "https://www.reddit.com/r/ruby/comments/yhe3t4/whats_ruby_used_for_most_nowadays/",
+            "content" => "Ruby is mainly used in web app development because that's what makes money. However, Ruby is also used for automation, data processing, DevOps tools, and many other applications where its expressive syntax shines.",
+            "published_date" => "2024-01-30"
+          }
+        ]
+
+        limited_results = results.take(max_results)
+
+        response = {
+          "query" => query,
+          "follow_up_questions" => [],
+          "images" => [],
+          "results" => limited_results,
+          "search_depth" => options[:search_depth] || "basic",
+          "search_time" => 0.1
+        }
+
+        if include_answer
+          response["answer"] = "Ruby is a dynamic, open-source programming language with a focus on simplicity and productivity. It was created by Yukihiro Matsumoto in the mid-1990s and follows the principle that everything is an object. Ruby is particularly popular for web development, especially with the Ruby on Rails framework, but it's also used for automation, data processing, and various other applications."
+        end
+
+        response
+      end
+    end
+  end
+end

data/lib/search_adapters/serper_adapter.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+require 'uri'
+
+module Deepsearch
+  module SearchAdapters
+    # An adapter for the Serper Search API (google.serper.dev).
+    class SerperAdapter
+      BASE_URL = 'https://google.serper.dev/search'
+
+      def initialize(api_key = nil)
+        @api_key = api_key || Deepsearch.configuration.serper_api_key
+        validate_api_key!
+      end
+
+      # @param query [String] The search query
+      # @param options [Hash] Additional search options
+      # @option options [Integer] :max_results Maximum number of results to return. Serper calls this `num`.
+      # @return [Hash] Parsed and transformed response from Serper API
+      # @raise [SerperError] If the API request fails
+      def search(query, options = {})
+        raise ArgumentError, "Query cannot be empty" if query.nil? || query.strip.empty?
+
+        payload = build_payload(query, options)
+        response = make_request(payload)
+        parsed_response = parse_response(response)
+        transform_response(parsed_response)
+      rescue Net::HTTPError, Net::ReadTimeout, Net::OpenTimeout => e
+        raise SerperError, "Network error: #{e.message}"
+      rescue JSON::ParserError => e
+        raise SerperError, "Invalid JSON response: #{e.message}"
+      end
+
+      private
+
+      def validate_api_key!
+        raise SerperError, "API key is required" if @api_key.nil? || @api_key.strip.empty?
+      end
+
+      def build_payload(query, options)
+        payload = {
+          q: query
+        }
+        # Tavily uses `max_results`, Serper uses `num`. Let's support `max_results` from options.
+        payload[:num] = options[:max_results] if options[:max_results]
+        payload
+      end
+
+      def make_request(payload)
+        uri = URI(BASE_URL)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.read_timeout = 15
+        http.open_timeout = 5
+
+        request = Net::HTTP::Post.new(uri)
+        request['Content-Type'] = 'application/json'
+        request['X-API-KEY'] = @api_key
+        request.body = payload.to_json
+
+        response = http.request(request)
+
+        handle_error_response(response) unless response.is_a?(Net::HTTPSuccess)
+
+        response
+      end
+
+      def parse_response(response)
+        JSON.parse(response.body)
+      rescue JSON::ParserError => e
+        raise SerperError, "Failed to parse response: #{e.message}"
+      end
+
+      def transform_response(response_body)
+        results = response_body['organic']&.map do |item|
+          {
+            'url' => item['link'],
+            'title' => item['title'],
+            'content' => item['snippet']
+          }
+        end || []
+        { 'results' => results }
+      end
+
+      def handle_error_response(response)
+        case response.code.to_i
+        when 400
+          raise SerperError, "Bad request: #{response.body}"
+        when 401, 402
+          raise SerperError, "Unauthorized or payment required: #{response.body}"
+        when 429
+          raise SerperError, "Rate limit exceeded"
+        when 500..599
+          raise SerperError, "Server error: #{response.code}"
+        else
+          raise SerperError, "HTTP error: #{response.code} - #{response.body}"
+        end
+      end
+    end
+
+    # Custom error class for exceptions raised by the SerperAdapter.
+    class SerperError < StandardError; end
+  end
+end

data/lib/search_adapters/tavily_adapter.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+require 'uri'
+
+module Deepsearch
+  module SearchAdapters
+    # An adapter for the Tavily Search API.
+    class TavilyAdapter
+      BASE_URL = 'https://api.tavily.com/search'
+
+      def initialize(api_key = nil)
+        @api_key = api_key || Deepsearch.configuration.tavily_api_key
+        validate_api_key!
+      end
+
+      # @param query [String] The search query
+      # @param options [Hash] Additional search options
+      # @option options [Integer] :max_results Maximum number of results (default: 10)
+      # @option options [Array<String>] :include_domains Domains to include in search
+      # @option options [Array<String>] :exclude_domains Domains to exclude from search
+      # @option options [Boolean] :include_answer Whether to include AI-generated answer (default: true)
+      # @option options [Boolean] :include_raw_content Whether to include raw content (default: false)
+      # @option options [String] :search_depth Search depth: 'basic' or 'advanced' (default: 'basic')
+      # @return [Hash] Parsed response from Tavily API
+      # @raise [TavilyError] If the API request fails
+      def search(query, options = {})
+        raise ArgumentError, "Query cannot be empty" if query.nil? || query.strip.empty?
+
+        payload = build_payload(query, options)
+        response = make_request(payload)
+        parse_response(response)
+      rescue Net::HTTPError, Net::ReadTimeout, Net::OpenTimeout => e
+        raise TavilyError, "Network error: #{e.message}"
+      rescue JSON::ParserError => e
+        raise TavilyError, "Invalid JSON response: #{e.message}"
+      end
+
+      private
+
+      def validate_api_key!
+        if @api_key.nil? || @api_key.strip.empty?
+          raise TavilyError, "API key is required"
+        end
+
+        unless @api_key.start_with?('tvly-')
+          raise TavilyError, "Invalid API key format. Expected format: tvly-YOUR_API_KEY"
+        end
+      end
+
+      def build_payload(query, options)
+        payload = {
+          query: query,
+          max_results: options[:max_results] || 10,
+          include_answer: options.fetch(:include_answer, true),
+          include_raw_content: options.fetch(:include_raw_content, false),
+          search_depth: options[:search_depth] || 'basic'
+        }
+
+        payload[:include_domains] = options[:include_domains] if options[:include_domains]
+        payload[:exclude_domains] = options[:exclude_domains] if options[:exclude_domains]
+
+        payload
+      end
+
+      def make_request(payload)
+        uri = URI(BASE_URL)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.read_timeout = 15
+        http.open_timeout = 5
+
+        request = Net::HTTP::Post.new(uri)
+        request['Content-Type'] = 'application/json'
+        request['Authorization'] = "Bearer #{@api_key}"
+        request.body = payload.to_json
+
+        response = http.request(request)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          handle_error_response(response)
+        end
+
+        response
+      end
+
+      def parse_response(response)
+        JSON.parse(response.body)
+      rescue JSON::ParserError => e
+        raise TavilyError, "Failed to parse response: #{e.message}"
+      end
+
+      def handle_error_response(response)
+        case response.code.to_i
+        when 400
+          raise TavilyError, "Bad request: #{response.body}"
+        when 401
+          raise TavilyError, "Unauthorized: Invalid API key"
+        when 429
+          raise TavilyError, "Rate limit exceeded"
+        when 500..599
+          raise TavilyError, "Server error: #{response.code}"
+        else
+          raise TavilyError, "HTTP error: #{response.code} - #{response.body}"
+        end
+      end
+    end
+
+    # Custom error class for exceptions raised by the TavilyAdapter.
+    class TavilyError < StandardError; end
+  end
+end

data/lib/search_adapters.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'search_adapters/tavily_adapter'
+require_relative 'search_adapters/mock_adapter'
+require_relative 'search_adapters/serper_adapter'
+
+module Deepsearch
+  module SearchAdapters
+    def self.create(type, *args)
+      return type.new(*args) if type.is_a?(Class)
+
+      case type.to_sym
+      when :tavily
+        TavilyAdapter.new(*args)
+      when :serper
+        SerperAdapter.new(*args)
+      when :mock
+        MockAdapter.new(*args)
+      else
+        raise ArgumentError, "Unknown or invalid adapter type: #{type}"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: deepsearch-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alexander Shagov
+bindir: bin
+cert_chain: []
+date: 1980-01-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: ruby_llm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: A ruby gem for performing LLM-powered automated web search.
+email:
+- shagov.dev@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/deepsearch.rb
+- lib/deepsearch/configuration.rb
+- lib/deepsearch/engine.rb
+- lib/deepsearch/engine/pipeline.rb
+- lib/deepsearch/engine/steps/data_aggregation/parsed_website.rb
+- lib/deepsearch/engine/steps/data_aggregation/process.rb
+- lib/deepsearch/engine/steps/data_aggregation/result.rb
+- lib/deepsearch/engine/steps/parallel_search/process.rb
+- lib/deepsearch/engine/steps/parallel_search/result.rb
+- lib/deepsearch/engine/steps/parallel_search/search.rb
+- lib/deepsearch/engine/steps/prepare_subqueries/process.rb
+- lib/deepsearch/engine/steps/prepare_subqueries/result.rb
+- lib/deepsearch/engine/steps/rag/chunker.rb
+- lib/deepsearch/engine/steps/rag/process.rb
+- lib/deepsearch/engine/steps/rag/similarity.rb
+- lib/deepsearch/engine/steps/rag/values/chunk.rb
+- lib/deepsearch/engine/steps/rag/values/query.rb
+- lib/deepsearch/engine/steps/rag/values/result.rb
+- lib/deepsearch/engine/steps/summarization/process.rb
+- lib/deepsearch/engine/steps/summarization/values/result.rb
+- lib/deepsearch/logger.rb
+- lib/deepsearch/prompts_config.rb
+- lib/deepsearch/version.rb
+- lib/search_adapters.rb
+- lib/search_adapters/mock_adapter.rb
+- lib/search_adapters/serper_adapter.rb
+- lib/search_adapters/tavily_adapter.rb
+homepage: https://github.com/alexshagov/deepsearch-rb
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/alexshagov/deepsearch-rb
+  source_code_uri: https://github.com/alexshagov/deepsearch-rb
+  changelog_uri: https://github.com/alexshagov/deepsearch-rb/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: A ruby gem for performing LLM-powered automated web search.
+test_files: []