prescient 0.5.0 → 0.7.0

data/lib/prescient/cli.rb

@@ -3,6 +3,7 @@
 require 'json'
 require 'optparse'
 require 'yaml'
+require_relative '../prescient'
 
 # Command-line interface for common Prescient operations.
 class Prescient::CLI
@@ -46,8 +47,8 @@ class Prescient::CLI
         chat_model: llama3.2:3b
         # prompt_templates:
         #   system_prompt: You are a concise assistant.
-        #   no_context_template: "%<system_prompt>s\\n\\nUser: %<query>s"
-        #   with_context_template: "%<system_prompt>s\\n\\nContext:\\n%<context>s\\n\\nUser: %<query>s"
+        #   no_context_template: "%<system_prompt>s\x5Cn\x5CnUser: %<query>s"
+        #   with_context_template: "%<system_prompt>s\x5Cn\x5CnContext:\x5Cn%<context>s\x5Cn\x5CnUser: %<query>s"
 
       # Uncomment a cloud provider and set its credential in the environment.
       # openai:
@@ -57,7 +58,7 @@ class Prescient::CLI
       #   chat_model: gpt-4.1-mini
       #   prompt_templates:
       #     system_prompt: You are a concise assistant.
-      #     no_context_template: "%<system_prompt>s\n\nUser: %<query>s"
+      #     no_context_template: "%<system_prompt>s\x5Cn\x5CnUser: %<query>s"
 
       # anthropic:
       #   type: anthropic
@@ -92,6 +93,62 @@ class Prescient::CLI
       #   api_key_env: HUGGINGFACE_API_KEY
       #   embedding_model: sentence-transformers/all-MiniLM-L6-v2
       #   chat_model: google/gemma-2-2b-it
+
+    # External tools are opt-in and separate from AI providers. They can be
+    # used directly with `prescient search`, or as context with
+    # `prescient search --generate`.
+    #
+    # The CLI also registers `web_search` automatically when SEARXNG_URL is
+    # set and no YAML tool configuration is provided.
+    tools:
+      # Local SearXNG example. Uncomment this block to configure a tool in YAML.
+      # web_search:
+      #   type: searxng
+      #   url: http://localhost:8080
+      #   timeout: 5
+      #   max_results: 5
+      #   language: en
+      #   categories:
+      #     - general
+      #     - science
+      #   max_response_bytes: 1048576
+
+      # SearchApi example. It uses SearchApi's Google engine by default and
+      # authenticates with a Bearer token from the environment.
+      # searchapi_web:
+      #   type: searchapi
+      #   api_key_env: SEARCHAPI_API_KEY
+      #   engine: google
+      #   location: New York
+      #   hl: en
+      #   gl: us
+      #   timeout: 10
+      #   max_results: 5
+
+      # Capability fallback. Adapters are tried in order, and fallback occurs
+      # only for transient connection or rate-limit failures.
+      # resilient_search:
+      #   adapters:
+      #     - type: searxng
+      #       url_env: SEARXNG_URL
+      #     - type: searchapi
+      #       api_key_env: SEARCHAPI_API_KEY
+      #       engine: google
+
+      # Prefer an environment reference when the URL differs by environment
+      # or should not be committed. Use `--tool research_search` to select a
+      # tool with a custom name.
+      # research_search:
+      #   type: searxng
+      #   url_env: SEARXNG_URL
+      #   timeout_env: SEARXNG_TIMEOUT
+      #   max_results_env: SEARXNG_MAX_RESULTS
+      #   language_env: SEARXNG_LANGUAGE
+      #   categories_env: SEARXNG_CATEGORIES
+
+      # Search results are returned directly by default. Add `--generate` to
+      # feed normalized results to the selected AI provider. Omit `--generate`
+      # when the caller should handle the search results itself.
   YAML
 
   # Raised when command-line arguments are invalid or incomplete.
@@ -148,6 +205,7 @@ class Prescient::CLI
     when 'health' then health
     when 'generate' then generate
     when 'embed' then embed
+    when 'search' then search
     when 'config' then config
     when 'help', '--help', '-h' then print_help(0)
     else
@@ -213,6 +271,52 @@ class Prescient::CLI
     0
   end
 
+  def search
+    options = parse_options(
+      'Search with a configured external tool',
+      fallback: true, tool: true, limit: true, generate: true,
+    )
+    return options if options.is_a?(Integer)
+
+    query = read_text(options[:arguments], 'query')
+    tool_name = (options[:tool] || 'web_search').to_sym
+    return generate_search_response(query, tool_name, options) if options[:generate]
+
+    tool = Prescient.tool(tool_name)
+    raise UsageError, "tool not configured: #{tool_name}" unless tool
+
+    result = tool.search(query, limit: options[:limit])
+    if options[:format] == 'json'
+      print_json(result)
+    else
+      print_search_results(result[:results])
+    end
+    0
+  end
+
+  def generate_search_response(query, tool_name, options)
+    response = Prescient.search_and_generate(
+      query,
+      tool:             tool_name,
+      provider:         options[:provider]&.to_sym,
+      limit:            options[:limit],
+      enable_fallback:  options[:fallback],
+      provider_options: provider_options(options),
+      **model_options(options),
+    )
+    options[:format] == 'json' ? print_json(response) : @output.puts(response[:response])
+    0
+  end
+
+  def print_search_results(results)
+    results.each do |item|
+      @output.puts item[:title]
+      @output.puts item[:url]
+      @output.puts item[:snippet] unless item[:snippet].empty?
+      @output.puts
+    end
+  end
+
   def config
     subcommand = @arguments.shift
     case subcommand
@@ -250,16 +354,24 @@ class Prescient::CLI
       raise Prescient::Error, 'default provider is not configured'
     end
 
-    configuration.providers.each_key { |name| configuration.provider(name) }
+    configuration.providers.each_key do |name|
+      configuration.provider(name)
+    end
+    configuration.tools.each_key do |name|
+      configuration.tool(name)
+    end
   end
 
-  def parse_options(description, fallback: false)
+  def parse_options(description, fallback: false, tool: false, limit: false, generate: false)
     options = { format: 'text', fallback: fallback }
     parser = OptionParser.new do |parser|
       parser.banner = "Usage: prescient #{@arguments.first || 'command'} [options]"
       parser.separator description
+      parser.separator ''
+      parser.separator 'Global options:'
       add_common_options(parser, options)
       parser.on('--no-fallback', 'Disable provider fallback') { options[:fallback] = false } if fallback
+      add_tool_options(parser, options, tool:, limit:, generate:)
       parser.on('-h', '--help', 'Show command help') do
         @output.puts parser
         throw :help_shown, 0
@@ -273,6 +385,18 @@ class Prescient::CLI
     options
   end
 
+  def add_tool_options(parser, options, tool:, limit:, generate:)
+    return unless tool || limit || generate
+
+    parser.separator ''
+    parser.separator 'Search options:'
+    parser.on('--tool NAME', 'Use a configured external tool') { |value| options[:tool] = value } if tool
+    parser.on('--generate', 'Use search results as AI provider context') { options[:generate] = true } if generate
+    return unless limit
+
+    parser.on('--limit COUNT', Integer, 'Limit the number of results') { |value| options[:limit] = value }
+  end
+
   def add_common_options(parser, options)
     parser.on('--config PATH', 'Load configuration from a YAML file') do |value|
       options[:config] = value
@@ -424,10 +548,11 @@ class Prescient::CLI
         health          Check provider health
         generate TEXT   Generate a text response
         embed TEXT      Generate an embedding
+        search TEXT     Search with a configured external tool
         config validate Validate the current configuration
         config example  Generate an annotated YAML configuration example
 
-      Options:
+      Global options:
         --config PATH            Load configuration from a YAML file
         --provider NAME          Select a provider
         --model NAME             Override the selected operation's model
@@ -443,6 +568,11 @@ class Prescient::CLI
         --api-key KEY            Use an API key for the operation
         --api-key-env NAME       Read the API key from an environment variable
         --format FORMAT          Use text or json output
+
+      Search options:
+        --tool NAME              Select an external tool for search
+        --generate               Use search results as AI provider context
+        --limit COUNT            Limit search results
     HELP
     status
   end

data/lib/prescient/client.rb

@@ -247,6 +247,32 @@ module Prescient
     client(provider, enable_fallback: enable_fallback).generate_response(prompt, context_items, **options)
   end
 
+  # Search with an explicit external tool and optionally use the normalized
+  # results as context for a configured AI provider.
+  #
+  # @param query [String] Search query and generation prompt
+  # @param tool [Symbol, String] Configured external tool name
+  # @param provider [Symbol, nil] Provider to use for generation
+  # @param limit [Integer, nil] Maximum number of search results
+  # @param enable_fallback [Boolean] Whether provider fallback is enabled
+  # @param provider_options [Hash] Temporary provider configuration overrides
+  # @return [Hash] Normalized provider response
+  def self.search_and_generate(query, tool: :web_search, provider: nil, limit: nil,
+                               enable_fallback: true, provider_options: {}, **options)
+    search_tool = self.tool(tool)
+    raise Prescient::ToolConfigurationError, "tool not configured: #{tool}" unless search_tool
+
+    search_result = search_tool.search(query, limit: limit)
+    context_items = search_result[:results]
+    raise Prescient::ToolInvalidResponseError, 'tool results must be an array' unless context_items.is_a?(Array)
+
+    client(provider, enable_fallback:, provider_options:).generate_response(
+      query,
+      context_items,
+      **options,
+    )
+  end
+
   # Return the health status of a configured provider.
   #
   # @param provider [Symbol, nil] Provider to check

data/lib/prescient/configuration_loader.rb

@@ -16,6 +16,7 @@ class Prescient::ConfigurationLoader
     'fallback_providers',
     'fallback_providers_env',
     'providers',
+    'tools',
     'retry_attempts',
     'retry_attempts_env',
     'retry_delay',
@@ -39,6 +40,12 @@ class Prescient::ConfigurationLoader
     'xai'         => Prescient::Provider::XAI,
   }.freeze
 
+  # Tool names mapped to lazily resolved adapter constants.
+  TOOL_TYPES = {
+    'searchapi' => :SearchApi,
+    'searxng'   => :SearXNG,
+  }.freeze
+
   # Provider-specific keys shared by all supported adapters.
   COMMON_PROVIDER_KEYS = [
     'api_key',
@@ -61,6 +68,32 @@ class Prescient::ConfigurationLoader
     'url_env',
   ].freeze
 
+  # Tool-specific keys accepted by all configured adapters.
+  COMMON_TOOL_KEYS = [
+    'api_key',
+    'api_key_env',
+    'categories',
+    'categories_env',
+    'engine',
+    'engine_env',
+    'gl',
+    'gl_env',
+    'hl',
+    'hl_env',
+    'language',
+    'language_env',
+    'location',
+    'location_env',
+    'max_response_bytes',
+    'max_response_bytes_env',
+    'max_results',
+    'max_results_env',
+    'timeout',
+    'timeout_env',
+    'url',
+    'url_env',
+  ].freeze
+
   # Prompt template keys supported by provider configuration.
   PROMPT_TEMPLATE_KEYS = ['system_prompt', 'no_context_template', 'with_context_template'].freeze
 
@@ -131,6 +164,7 @@ class Prescient::ConfigurationLoader
   def load_hash(data, source: nil)
     configuration = Prescient::Configuration.new
     Prescient.send(:configure_default_providers, configuration, @env)
+    Prescient.send(:configure_default_tools, configuration, @env)
     apply!(configuration, data, source:)
     configuration
   end
@@ -147,6 +181,7 @@ class Prescient::ConfigurationLoader
 
     apply_scalar_settings(configuration, normalized, source:)
     apply_provider_settings(configuration, normalized, source:)
+    apply_tool_settings(configuration, normalized, source:)
     configuration
   end
 
@@ -234,6 +269,28 @@ class Prescient::ConfigurationLoader
     end
   end
 
+  def apply_tool_settings(configuration, data, source:)
+    return unless key_present?(data, :tools)
+
+    tools = data[:tools]
+    unless tools.is_a?(Hash)
+      raise Prescient::Error, "Configuration#{" in #{source}" if source} tools must be a mapping"
+    end
+
+    tools.each do |name, tool_data|
+      tool_name = name.to_sym
+      tool_settings = normalize_keys(tool_data)
+      validate_tool!(tool_name, tool_settings, source:)
+
+      if tool_settings.key?(:adapters)
+        configuration.add_tool_group(tool_name, resolve_tool_adapters(tool_settings, source:))
+      else
+        tool_options = resolve_tool_options(tool_settings, source:)
+        configuration.add_tool(tool_name, tool_class_for(tool_settings.fetch(:type).to_s), **tool_options)
+      end
+    end
+  end
+
   def validate_provider!(name, provider_data, source:)
     validate_provider_shape!(name, provider_data, source:)
     validate_provider_type!(name, provider_data, source:)
@@ -280,6 +337,58 @@ class Prescient::ConfigurationLoader
     raise Prescient::Error, message
   end
 
+  def validate_tool!(name, tool_data, source:)
+    validate_tool_shape!(name, tool_data, source:)
+    return validate_tool_group!(name, tool_data, source:) if tool_data.key?(:adapters)
+
+    validate_tool_type!(name, tool_data, source:)
+    validate_tool_keys!(name, tool_data, source:)
+  end
+
+  def validate_tool_group!(name, tool_data, source:)
+    unknown_keys = tool_data.keys.map(&:to_s) - ['adapters']
+    unless unknown_keys.empty?
+      source_suffix = " in #{source}" if source
+      raise Prescient::Error,
+            "Unknown tool group configuration key#{'s' if unknown_keys.length > 1} for #{name}: " \
+            "#{unknown_keys.join(', ')}#{source_suffix}"
+    end
+
+    adapters = tool_data[:adapters]
+    unless adapters.is_a?(Array) && adapters.any?
+      raise Prescient::Error, "Tool #{name} adapters must be a non-empty array"
+    end
+
+    adapters.each_with_index do |adapter_data, index|
+      adapter_name = "#{name} adapter #{index + 1}"
+      validate_tool_shape!(adapter_name, adapter_data, source:)
+      validate_tool_type!(adapter_name, adapter_data, source:)
+      validate_tool_keys!(adapter_name, adapter_data, source:)
+    end
+  end
+
+  def validate_tool_shape!(name, tool_data, source:)
+    return if tool_data.is_a?(Hash)
+
+    raise Prescient::Error, "Tool #{name.inspect}#{" in #{source}" if source} must be a mapping"
+  end
+
+  def validate_tool_type!(name, tool_data, source:)
+    return if tool_data.key?(:type)
+
+    raise Prescient::Error, "Tool #{name}#{" in #{source}" if source} must define type"
+  end
+
+  def validate_tool_keys!(name, tool_data, source:)
+    unknown_keys = tool_data.keys.map(&:to_s) - (['type'] + COMMON_TOOL_KEYS)
+    return if unknown_keys.empty?
+
+    raise Prescient::Error,
+          "Unknown tool configuration key#{'s' if unknown_keys.length > 1} for #{name}: " \
+          "#{unknown_keys.join(', ')}" \
+          "#{" in #{source}" if source}"
+  end
+
   def resolve_provider_options(provider_data, source:)
     provider_type = provider_data[:type].to_s
     provider_class = PROVIDER_TYPES[provider_type]
@@ -294,6 +403,57 @@ class Prescient::ConfigurationLoader
     end
   end
 
+  def resolve_tool_options(tool_data, source:)
+    tool_type = tool_data[:type].to_s
+    tool_class = tool_class_for(tool_type)
+    unless tool_class
+      raise Prescient::Error,
+            "Unknown tool type #{tool_type.inspect}#{" in #{source}" if source}"
+    end
+
+    tool_data.each_with_object({}) do |(key, value), options|
+      option = resolve_tool_option(tool_data, key, value, source:)
+      options[option.first] = option.last if option
+    end
+  end
+
+  def resolve_tool_adapters(tool_data, source:)
+    tool_data[:adapters].map do |adapter_data|
+      tool_type = adapter_data[:type].to_s
+      {
+        class:   tool_class_for(tool_type),
+        options: resolve_tool_options(adapter_data, source:),
+      }
+    end
+  end
+
+  # Resolve a configured tool adapter only when configuration uses it.
+  # @param tool_type [String] Configured tool type
+  # @return [Class, nil] Tool adapter class
+  def tool_class_for(tool_type)
+    tool_name = TOOL_TYPES[tool_type]
+    return unless tool_name
+
+    Prescient::Tool.const_get(tool_name, false)
+  end
+
+  def resolve_tool_option(tool_data, key, value, source:)
+    return if key == :type
+    return if key.to_s.end_with?('_env') && value.nil?
+
+    if key.to_s.end_with?('_env')
+      base_key = key.to_s.delete_suffix('_env').to_sym
+      if tool_data.key?(base_key)
+        raise Prescient::Error,
+              "Tool configuration cannot combine #{base_key} and #{key}"
+      end
+
+      [base_key, resolve_env_value(value, source:)]
+    else
+      [key, coerce_tool_option(key, resolve_value(value, source:))]
+    end
+  end
+
   def resolve_provider_option(provider_data, key, value, source:)
     return if key == :type
     return if key.to_s.end_with?('_env') && value.nil?
@@ -425,6 +585,17 @@ class Prescient::ConfigurationLoader
     end
   end
 
+  def coerce_tool_option(key, value)
+    case key.to_sym
+    when :max_results, :max_response_bytes
+      coerce_integer(value, key.to_s)
+    when :timeout
+      coerce_float(value, 'timeout')
+    else
+      value
+    end
+  end
+
   def validate_version!(data, source:)
     return unless key_present?(data, :version)
 

data/lib/prescient/errors.rb

@@ -37,6 +37,18 @@ module Prescient
   # Raised when an AI provider reports a transient service-side failure
   class ProviderError < Error; end
 
+  # Base error class for external tool failures.
+  class ToolError < Error; end
+
+  # Raised when an external tool is configured incorrectly.
+  class ToolConfigurationError < ToolError; end
+
+  # Raised when an external tool cannot be reached.
+  class ToolConnectionError < ToolError; end
+
+  # Raised when an external tool returns an invalid response.
+  class ToolInvalidResponseError < ToolError; end
+
   # Container module for AI provider implementations
   #
   # All provider classes should be defined within this module and inherit

data/lib/prescient/provider/openai.rb

@@ -83,8 +83,6 @@ class Prescient::Provider::OpenAI < Prescient::Base
 
       validate_response!(response, 'text generation')
 
-      puts "response.parsed_response: #{response.parsed_response.inspect}"
-
       content = response.parsed_response.dig('choices', 0, 'message', 'content')
       raise Prescient::InvalidResponseError, 'No response generated' unless content
 

data/lib/prescient/tool/search_api.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'httparty'
+require 'net/http'
+require 'timeout'
+
+# SearchApi web-search tool adapter.
+class Prescient::Tool::SearchApi < Prescient::Tool::Base
+  # SearchApi's stable search endpoint.
+  API_URL = 'https://www.searchapi.io/api/v1/search'
+  # Network failures that should be reported as connection errors.
+  NETWORK_ERRORS = [Net::OpenTimeout, Net::ReadTimeout, Timeout::Error, SocketError,
+                    Errno::ECONNREFUSED].freeze
+
+  # @param options [Hash] SearchApi configuration options
+  # @option options [String] :api_key SearchApi API key
+  # @option options [String] :engine SearchApi engine, defaulting to Google
+  # @option options [String] :location Optional search location
+  # @option options [String] :hl Optional interface language
+  # @option options [String] :gl Optional country code
+  def initialize(**options)
+    super
+    @api_key = required_api_key(options[:api_key])
+  end
+
+  # Search SearchApi and return normalized result metadata.
+  # @param query [String] Search query
+  # @param limit [Integer, nil] Maximum number of results
+  # @param timeout [Numeric, nil] Request timeout in seconds
+  # @return [Hash] Normalized search result envelope
+  def search(query, limit: nil, timeout: nil)
+    normalized_query = validate_query(query)
+    requested_limit = result_limit(limit)
+    requested_timeout = request_timeout(timeout)
+    response = HTTParty.get(
+      API_URL,
+      headers: request_headers,
+      query:   request_parameters(normalized_query, requested_limit),
+      timeout: requested_timeout,
+    )
+    validate_response!(response)
+
+    {
+      tool:    'web_search',
+      query:   normalized_query,
+      source:  'searchapi',
+      results: normalize_results(response.parsed_response, requested_limit),
+    }
+  rescue Prescient::Error
+    raise
+  rescue *NETWORK_ERRORS => e
+    raise Prescient::ToolConnectionError, "SearchApi request failed: #{e.class}"
+  rescue StandardError => e
+    raise Prescient::ToolError, "SearchApi request failed: #{e.message}"
+  end
+
+  protected
+
+  # Validate SearchApi-specific configuration.
+  # @return [void]
+  def validate_configuration!
+    required_api_key(@options[:api_key])
+    max_response_bytes
+    result_limit(nil)
+    request_timeout(nil)
+  end
+
+  private
+
+  def required_api_key(value)
+    return value if value.is_a?(String) && !value.empty?
+
+    raise Prescient::ToolConfigurationError, 'SearchApi requires an api_key'
+  end
+
+  def request_headers
+    { 'Authorization' => "Bearer #{@api_key}" }
+  end
+
+  def request_parameters(query, limit)
+    parameters = {
+      engine: @options.fetch(:engine, 'google'),
+      q:      query,
+      num:    limit,
+    }
+    [:location, :hl, :gl].each do |key|
+      parameters[key] = @options[key] if @options[key]
+    end
+    parameters
+  end
+
+  def validate_response!(response)
+    if response.respond_to?(:body) && response.body.to_s.bytesize > max_response_bytes
+      raise Prescient::ToolInvalidResponseError, 'SearchApi response exceeds configured size limit'
+    end
+
+    return if response.success?
+
+    error_class = case response.code.to_i
+                  when 401, 403 then Prescient::AuthenticationError
+                  when 429 then Prescient::RateLimitError
+                  when 500..599 then Prescient::ToolConnectionError
+                  else Prescient::ToolError
+                  end
+    raise error_class, "SearchApi returned HTTP #{response.code}"
+  end
+
+  def normalize_results(payload, limit)
+    raw_results = payload.is_a?(Hash) ? payload['organic_results'] : nil
+    unless raw_results.is_a?(Array)
+      raise Prescient::ToolInvalidResponseError, 'SearchApi response did not contain organic_results'
+    end
+
+    raw_results.filter_map { |result|
+      next unless result.is_a?(Hash)
+      next unless result['link'].is_a?(String) && !result['link'].empty?
+
+      {
+        title:   result['title'].to_s,
+        url:     result['link'],
+        snippet: result['snippet'].to_s,
+        source:  'searchapi',
+      }
+    }.first(limit)
+  end
+end