prescient 0.4.0 → 0.5.0

data/examples/vector_search.rb

@@ -1,330 +1,94 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Example: Vector similarity search with Prescient gem and PostgreSQL pgvector
-# This example demonstrates how to store embeddings and perform similarity search
+# Example: Vector similarity search with Prescient and PostgreSQL pgvector.
+# The Store owns only its embedding table; applications own their documents.
 
 require_relative '../lib/prescient'
 require 'pg'
-require 'json'
 
-puts "=== Vector Similarity Search Example ==="
-puts "This example shows how to use Prescient with PostgreSQL pgvector for semantic search."
+puts '=== Vector Similarity Search Example ==='
 
-# Database connection configuration
 DB_CONFIG = {
-  host: ENV.fetch('DB_HOST', 'localhost'),
-  port: ENV.fetch('DB_PORT', '5432'),
-  dbname: ENV.fetch('DB_NAME', 'prescient_development'),
-  user: ENV.fetch('DB_USER', 'prescient'),
-  password: ENV.fetch('DB_PASSWORD', 'prescient_password')
+  host:     ENV.fetch('DB_HOST', 'localhost'),
+  port:     ENV.fetch('DB_PORT', '5432'),
+  dbname:   ENV.fetch('DB_NAME', 'prescient_development'),
+  user:     ENV.fetch('DB_USER', 'prescient'),
+  password: ENV.fetch('DB_PASSWORD', 'prescient_password'),
 }.freeze
 
 class VectorSearchExample
-  def initialize
-    @db = PG.connect(DB_CONFIG)
-    @client = Prescient.client(:ollama)
-  end
-
-  def run_example
-    puts "\n--- Setting up vector search example ---"
-    
-    # Check if services are available
-    unless check_services_available
-      puts "❌ Required services not available. Please start with: docker compose up -d"
-      return
-    end
-
-    # 1. Generate and store embeddings for existing documents
-    puts "\n📊 Generating embeddings for sample documents..."
-    generate_document_embeddings
-
-    # 2. Perform similarity search
-    puts "\n🔍 Performing similarity searches..."
-    search_examples
-
-    # 3. Advanced search with filtering
-    puts "\n🎯 Advanced search with metadata filtering..."
-    advanced_search_examples
-
-    # 4. Demonstrate different distance functions
-    puts "\n📏 Comparing different distance functions..."
-    compare_distance_functions
-
-    puts "\n🎉 Vector search example completed!"
-  end
-
-  private
+  EMBEDDING_DIMENSIONS = 768
+  EMBEDDING_MODEL = 'nomic-embed-text'
 
-  def check_services_available
-    # Check database connection
-    begin
-      result = @db.exec("SELECT 1")
-      puts "✅ PostgreSQL connected"
-    rescue PG::Error => e
-      puts "❌ PostgreSQL connection failed: #{e.message}"
-      return false
-    end
-
-    # Check pgvector extension
-    begin
-      result = @db.exec("SELECT * FROM pg_extension WHERE extname = 'vector'")
-      if result.ntuples > 0
-        puts "✅ pgvector extension available"
-      else
-        puts "❌ pgvector extension not found"
-        return false
-      end
-    rescue PG::Error => e
-      puts "❌ pgvector check failed: #{e.message}"
-      return false
-    end
-
-    # Check Ollama connection
-    if @client.available?
-      puts "✅ Ollama connected"
-    else
-      puts "❌ Ollama not available"
-      return false
-    end
-
-    true
+  def initialize
+    @connection = PG.connect(DB_CONFIG)
+    @client = Prescient.client(:ollama, enable_fallback: false)
+    @store = Prescient::Pgvector::Store.new(
+      connection: @connection,
+      dimensions: EMBEDDING_DIMENSIONS,
+    )
   end
 
-  def generate_document_embeddings
-    # Get documents that don't have embeddings yet
-    query = <<~SQL
-      SELECT d.id, d.title, d.content 
-      FROM documents d
-      LEFT JOIN document_embeddings de ON d.id = de.document_id 
-        AND de.embedding_provider = 'ollama' 
-        AND de.embedding_model = 'nomic-embed-text'
-      WHERE de.id IS NULL
-      LIMIT 10
-    SQL
-
-    result = @db.exec(query)
-    
-    if result.ntuples == 0
-      puts "   All documents already have embeddings"
-      return
+  def run
+    @store.install!
+    %i[cosine euclidean inner_product].each { |metric| @store.create_index!(metric:) }
+
+    documents.each do |document|
+      embedding = @client.generate_embedding(document[:content], model: EMBEDDING_MODEL)
+      @store.upsert(
+        id: document[:id],
+        embedding:,
+        provider: 'ollama',
+        model: EMBEDDING_MODEL,
+        content: document[:content],
+        metadata: { title: document[:title] },
+      )
     end
 
-    result.each do |row|
-      document_id = row['id']
-      title = row['title']
-      content = row['content']
-      
-      puts "   Generating embedding for: #{title}"
-      
-      begin
-        # Generate embedding using Prescient
-        embedding = @client.generate_embedding(content)
-        
-        # Store in database
-        insert_embedding(document_id, embedding, content, 'ollama', 'nomic-embed-text', 768)
-        
-        puts "   ✅ Stored embedding (#{embedding.length} dimensions)"
-        
-      rescue Prescient::Error => e
-        puts "   ❌ Failed to generate embedding: #{e.message}"
-      end
+    query = 'How do I improve database performance?'
+    embedding = @client.generate_embedding(query, model: EMBEDDING_MODEL)
+    results = @store.search(
+      embedding:,
+      limit: 3,
+      metric: :cosine,
+      provider: 'ollama',
+      model: EMBEDDING_MODEL,
+    )
+
+    puts "\nQuery: #{query}"
+    results.each_with_index do |result, index|
+      title = result[:metadata].fetch('title')
+      puts "#{index + 1}. #{title} (distance: #{result[:distance].round(4)})"
+      puts "   #{result[:content]}"
     end
+  rescue Prescient::Error, PG::Error => e
+    warn "Vector search failed: #{e.message}"
+  ensure
+    @connection&.close
   end
 
-  def insert_embedding(document_id, embedding, text, provider, model, dimensions)
-    # Convert Ruby array to PostgreSQL vector format
-    vector_str = "[#{embedding.join(',')}]"
-    
-    query = <<~SQL
-      INSERT INTO document_embeddings 
-      (document_id, embedding_provider, embedding_model, embedding_dimensions, embedding, embedding_text)
-      VALUES ($1, $2, $3, $4, $5, $6)
-    SQL
-    
-    @db.exec_params(query, [document_id, provider, model, dimensions, vector_str, text])
-  end
+  private
 
-  def search_examples
-    search_queries = [
-      "How to learn programming?",
-      "What is machine learning?",
-      "Database optimization techniques",
-      "API security best practices"
+  def documents
+    [
+      {
+        id:      'postgres-indexes',
+        title:   'PostgreSQL indexes',
+        content: 'Indexes can reduce query latency when their columns match common filters and ordering.',
+      },
+      {
+        id:      'ruby-performance',
+        title:   'Ruby performance',
+        content: 'Measure allocations and repeated work before optimizing a Ruby application.',
+      },
+      {
+        id:      'api-security',
+        title:   'API security',
+        content: 'Protect API credentials, validate inputs, and avoid exposing provider response bodies.',
+      },
     ]
-
-    search_queries.each do |query_text|
-      puts "\n🔍 Searching for: '#{query_text}'"
-      perform_similarity_search(query_text, limit: 3)
-    end
-  end
-
-  def perform_similarity_search(query_text, limit: 5, distance_function: 'cosine')
-    begin
-      # Generate embedding for query
-      query_embedding = @client.generate_embedding(query_text)
-      query_vector = "[#{query_embedding.join(',')}]"
-      
-      # Choose distance operator based on function
-      distance_op = case distance_function
-                   when 'cosine' then '<=>'
-                   when 'l2' then '<->'
-                   when 'inner_product' then '<#>'
-                   else '<=>'
-                   end
-
-      # Perform similarity search
-      search_query = <<~SQL
-        SELECT 
-          d.title,
-          d.content,
-          d.metadata,
-          de.embedding #{distance_op} $1::vector AS distance,
-          1 - (de.embedding <=> $1::vector) AS cosine_similarity
-        FROM documents d
-        JOIN document_embeddings de ON d.id = de.document_id
-        WHERE de.embedding_provider = 'ollama' 
-          AND de.embedding_model = 'nomic-embed-text'
-        ORDER BY de.embedding #{distance_op} $1::vector
-        LIMIT $2
-      SQL
-
-      result = @db.exec_params(search_query, [query_vector, limit])
-      
-      if result.ntuples == 0
-        puts "   No results found"
-        return
-      end
-
-      result.each_with_index do |row, index|
-        similarity = (row['cosine_similarity'].to_f * 100).round(1)
-        puts "   #{index + 1}. #{row['title']} (#{similarity}% similar)"
-        puts "      #{row['content'][0..100]}..."
-        
-        # Show metadata if available
-        if row['metadata'] && !row['metadata'].empty?
-          metadata = JSON.parse(row['metadata'])
-          tags = metadata['tags']&.join(', ')
-          puts "      Tags: #{tags}" if tags
-        end
-        puts
-      end
-      
-    rescue Prescient::Error => e
-      puts "   ❌ Search failed: #{e.message}"
-    rescue PG::Error => e
-      puts "   ❌ Database error: #{e.message}"
-    end
-  end
-
-  def advanced_search_examples
-    # Search with metadata filtering
-    puts "\n🎯 Search for programming content with beginner difficulty:"
-    advanced_search("programming basics", tags: ["programming"], difficulty: "beginner")
-    
-    puts "\n🎯 Search for AI/ML content:"
-    advanced_search("artificial intelligence", tags: ["ai", "machine-learning"])
   end
-
-  def advanced_search(query_text, filters = {})
-    begin
-      query_embedding = @client.generate_embedding(query_text)
-      query_vector = "[#{query_embedding.join(',')}]"
-      
-      # Build WHERE clause for metadata filtering
-      where_conditions = ["de.embedding_provider = 'ollama'", "de.embedding_model = 'nomic-embed-text'"]
-      params = [query_vector]
-      param_index = 2
-
-      filters.each do |key, value|
-        case key
-        when :tags
-          # Filter by tags array overlap
-          where_conditions << "d.metadata->'tags' ?| $#{param_index}::text[]"
-          params << value
-          param_index += 1
-        when :difficulty
-          # Filter by exact difficulty match
-          where_conditions << "d.metadata->>'difficulty' = $#{param_index}"
-          params << value
-          param_index += 1
-        when :source_type
-          # Filter by source type
-          where_conditions << "d.source_type = $#{param_index}"
-          params << value
-          param_index += 1
-        end
-      end
-
-      search_query = <<~SQL
-        SELECT 
-          d.title,
-          d.content,
-          d.metadata,
-          de.embedding <=> $1::vector AS cosine_distance,
-          1 - (de.embedding <=> $1::vector) AS cosine_similarity
-        FROM documents d
-        JOIN document_embeddings de ON d.id = de.document_id
-        WHERE #{where_conditions.join(' AND ')}
-        ORDER BY de.embedding <=> $1::vector
-        LIMIT 3
-      SQL
-
-      result = @db.exec_params(search_query, params)
-      
-      if result.ntuples == 0
-        puts "   No results found with the specified filters"
-        return
-      end
-
-      result.each_with_index do |row, index|
-        similarity = (row['cosine_similarity'].to_f * 100).round(1)
-        puts "   #{index + 1}. #{row['title']} (#{similarity}% similar)"
-        
-        metadata = JSON.parse(row['metadata'])
-        puts "      Difficulty: #{metadata['difficulty']}"
-        puts "      Tags: #{metadata['tags']&.join(', ')}"
-        puts "      #{row['content'][0..80]}..."
-        puts
-      end
-      
-    rescue Prescient::Error => e
-      puts "   ❌ Search failed: #{e.message}"
-    rescue PG::Error => e
-      puts "   ❌ Database error: #{e.message}"
-    end
-  end
-
-  def compare_distance_functions
-    query_text = "programming languages and development"
-    
-    puts "\n📏 Comparing distance functions for: '#{query_text}'"
-    
-    %w[cosine l2 inner_product].each do |distance_func|
-      puts "\n   #{distance_func.upcase} Distance:"
-      perform_similarity_search(query_text, limit: 2, distance_function: distance_func)
-    end
-  end
-
-  def cleanup
-    @db.close if @db
-  end
-end
-
-# Run the example
-begin
-  example = VectorSearchExample.new
-  example.run_example
-rescue StandardError => e
-  puts "❌ Example failed: #{e.message}"
-  puts e.backtrace.first(5).join("\n")
-ensure
-  example&.cleanup
 end
 
-puts "\n💡 Next steps:"
-puts "   - Try different embedding models (OpenAI, HuggingFace)"
-puts "   - Implement hybrid search (vector + keyword)"
-puts "   - Add document chunking for large texts"
-puts "   - Experiment with different similarity thresholds"
-puts "   - Add result re-ranking and filtering"
+VectorSearchExample.new.run

data/lib/prescient/cli.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'optparse'
+require 'yaml'
 
 # Command-line interface for common Prescient operations.
 class Prescient::CLI
@@ -9,6 +10,90 @@ class Prescient::CLI
   # @return [Array<String>] Output format names
   FORMATS = ['text', 'json'].freeze
 
+  # Schema URL and annotated starter configuration for `config example`.
+  CONFIGURATION_EXAMPLE = <<~YAML
+    # yaml-language-server: $schema=https://raw.githubusercontent.com/kanutocd/prescient/refs/heads/main/schema/prescient.configuration.schema.json
+    #
+    # Prescient configuration example.
+    #
+    # Precedence, from lowest to highest:
+    # 1. Built-in defaults and provider environment variables.
+    # 2. Values in this YAML file.
+    # 3. Per-operation CLI overrides such as --provider and --chat-model.
+    #
+    # Use `prescient config validate` after editing this file.
+    # Keep credentials out of source control; use *_env references instead.
+    version: 1
+
+    # Global behavior.
+    default_provider: ollama
+    timeout: 30
+    retry_attempts: 3
+    retry_delay: 1.0
+    fallback_providers: []
+    sensitive_keys:
+      - api_key
+      - password
+      - token
+      - secret
+
+    providers:
+      # Local Ollama requires no API key.
+      ollama:
+        type: ollama
+        url: http://localhost:11434
+        embedding_model: nomic-embed-text
+        chat_model: llama3.2:3b
+        # prompt_templates:
+        #   system_prompt: You are a concise assistant.
+        #   no_context_template: "%<system_prompt>s\\n\\nUser: %<query>s"
+        #   with_context_template: "%<system_prompt>s\\n\\nContext:\\n%<context>s\\n\\nUser: %<query>s"
+
+      # Uncomment a cloud provider and set its credential in the environment.
+      # openai:
+      #   type: openai
+      #   api_key_env: OPENAI_API_KEY
+      #   embedding_model: text-embedding-3-small
+      #   chat_model: gpt-4.1-mini
+      #   prompt_templates:
+      #     system_prompt: You are a concise assistant.
+      #     no_context_template: "%<system_prompt>s\n\nUser: %<query>s"
+
+      # anthropic:
+      #   type: anthropic
+      #   api_key_env: ANTHROPIC_API_KEY
+      #   model: claude-sonnet-4-20250514
+
+      # gemini:
+      #   type: gemini
+      #   api_key_env: GEMINI_API_KEY
+      #   embedding_model: gemini-embedding-001
+      #   chat_model: gemini-2.5-flash
+
+      # mistral:
+      #   type: mistral
+      #   api_key_env: MISTRAL_API_KEY
+      #   embedding_model: mistral-embed
+      #   chat_model: mistral-large-latest
+
+      # DeepSeek supports text generation, but not embeddings.
+      # deepseek:
+      #   type: deepseek
+      #   api_key_env: DEEPSEEK_API_KEY
+      #   chat_model: deepseek-v4-flash
+
+      # xai:
+      #   type: xai
+      #   api_key_env: XAI_API_KEY
+      #   chat_model: grok-4.5
+
+      # huggingface:
+      #   type: huggingface
+      #   api_key_env: HUGGINGFACE_API_KEY
+      #   embedding_model: sentence-transformers/all-MiniLM-L6-v2
+      #   chat_model: google/gemma-2-2b-it
+  YAML
+
   # Raised when command-line arguments are invalid or incomplete.
   class UsageError < StandardError; end
 
@@ -42,10 +127,22 @@ class Prescient::CLI
     @errors = errors
   end
 
+  # Execute the CLI command and return its process status.
+  # @return [Integer] Process exit status
   def run
+    config_path = extract_global_config_path
+    Prescient.load_configuration(config_path) if config_path || ENV['PRESCIENT_CONFIG']
+
     command = @arguments.shift
     return print_help(2) unless command
 
+    run_command(command)
+  end
+
+  # Dispatch a parsed command to its handler.
+  # @param command [String] Command name
+  # @return [Integer] Process exit status
+  def run_command(command)
     case command
     when 'providers' then providers
     when 'health' then health
@@ -118,8 +215,15 @@ class Prescient::CLI
 
   def config
     subcommand = @arguments.shift
-    raise UsageError, "unknown config command #{subcommand.inspect}" unless subcommand == 'validate'
+    case subcommand
+    when 'validate' then validate_config_command
+    when 'example' then configuration_example_command
+    else
+      raise UsageError, "unknown config command #{subcommand.inspect}"
+    end
+  end
 
+  def validate_config_command
     options = parse_options('Validate the current configuration')
     return options if options.is_a?(Integer)
 
@@ -132,6 +236,14 @@ class Prescient::CLI
     0
   end
 
+  def configuration_example_command
+    options = parse_options('Generate an annotated YAML configuration example')
+    return options if options.is_a?(Integer)
+
+    @output.write(CONFIGURATION_EXAMPLE)
+    0
+  end
+
   def validate_configuration
     configuration = Prescient.configuration
     unless configuration.provider(configuration.default_provider)
@@ -162,6 +274,9 @@ class Prescient::CLI
   end
 
   def add_common_options(parser, options)
+    parser.on('--config PATH', 'Load configuration from a YAML file') do |value|
+      options[:config] = value
+    end
     parser.on('--format FORMAT', FORMATS, "Output format (#{FORMATS.join(', ')})") do |value|
       options[:format] = value
     end
@@ -206,6 +321,18 @@ class Prescient::CLI
     parser.on('--chat-model NAME', 'Override the chat model') do |value|
       options[:chat_model] = value
     end
+    parser.on('--system-prompt TEXT', 'Override the system prompt') do |value|
+      options[:system_prompt] = value
+    end
+    parser.on('--no-context-template TEXT', 'Override the no-context prompt template') do |value|
+      options[:no_context_template] = value
+    end
+    parser.on('--with-context-template TEXT', 'Override the with-context prompt template') do |value|
+      options[:with_context_template] = value
+    end
+    parser.on('--prompt-templates-file PATH', 'Load prompt templates from a YAML file') do |value|
+      options[:prompt_templates_file] = value
+    end
   end
 
   def add_credential_options(parser, options)
@@ -228,12 +355,38 @@ class Prescient::CLI
 
   def provider_options(options)
     {
-      api_key:         api_key_override(options),
-      embedding_model: options[:embedding_model],
-      chat_model:      options[:chat_model],
+      api_key:          api_key_override(options),
+      embedding_model:  options[:embedding_model],
+      chat_model:       options[:chat_model],
+      prompt_templates: prompt_templates(options),
     }.compact
   end
 
+  def prompt_templates(options)
+    templates = if options[:prompt_templates_file]
+                  data = YAML.safe_load_file(
+                    options[:prompt_templates_file],
+                    permitted_classes: [],
+                    permitted_symbols: [],
+                    aliases:           true,
+                  )
+                  raise UsageError, 'prompt templates file must contain a mapping' unless data.is_a?(Hash)
+
+                  data.transform_keys(&:to_sym)
+                else
+                  {}
+                end
+
+    [:system_prompt, :no_context_template, :with_context_template].each do |key|
+      templates[key] = options[key] if options[key]
+    end
+    templates.empty? ? nil : templates
+  rescue Errno::ENOENT
+    raise UsageError, "prompt templates file not found: #{options[:prompt_templates_file]}"
+  rescue Psych::SyntaxError => e
+    raise UsageError, "invalid prompt templates YAML: #{e.message}"
+  end
+
   def api_key_override(options)
     return options[:api_key] if options[:api_key]
     return ENV.fetch(options[:api_key_env]) if options[:api_key_env]
@@ -272,16 +425,55 @@ class Prescient::CLI
         generate TEXT   Generate a text response
         embed TEXT      Generate an embedding
         config validate Validate the current configuration
+        config example  Generate an annotated YAML configuration example
 
       Options:
+        --config PATH            Load configuration from a YAML file
         --provider NAME          Select a provider
         --model NAME             Override the selected operation's model
         --chat-model NAME        Override the chat model
         --embedding-model NAME   Override the embedding model
+        --system-prompt TEXT     Override the system prompt
+        --no-context-template TEXT
+                                 Override the no-context prompt template
+        --with-context-template TEXT
+                                 Override the with-context prompt template
+        --prompt-templates-file PATH
+                                 Load prompt templates from a YAML file
         --api-key KEY            Use an API key for the operation
         --api-key-env NAME       Read the API key from an environment variable
         --format FORMAT          Use text or json output
     HELP
     status
   end
+
+  def extract_global_config_path
+    config_path = nil
+    filtered_arguments = []
+    index = 0
+
+    while index < @arguments.length
+      argument = @arguments[index]
+      if argument == '--config'
+        value = @arguments[index + 1]
+        raise UsageError, '--config requires a path' unless value
+
+        config_path = value
+        index += 2
+        next
+      end
+
+      if argument.start_with?('--config=')
+        config_path = argument.split('=', 2).last
+        index += 1
+        next
+      end
+
+      filtered_arguments << argument
+      index += 1
+    end
+
+    @arguments = filtered_arguments
+    config_path
+  end
 end