prescient 0.0.0 → 0.1.0

data/examples/vector_search.rb

@@ -0,0 +1,330 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example: Vector similarity search with Prescient gem and PostgreSQL pgvector
+# This example demonstrates how to store embeddings and perform similarity search
+
+require_relative '../lib/prescient'
+require 'pg'
+require 'json'
+
+puts "=== Vector Similarity Search Example ==="
+puts "This example shows how to use Prescient with PostgreSQL pgvector for semantic search."
+
+# Database connection configuration
+DB_CONFIG = {
+  host: ENV.fetch('DB_HOST', 'localhost'),
+  port: ENV.fetch('DB_PORT', '5432'),
+  dbname: ENV.fetch('DB_NAME', 'prescient_development'),
+  user: ENV.fetch('DB_USER', 'prescient'),
+  password: ENV.fetch('DB_PASSWORD', 'prescient_password')
+}.freeze
+
+class VectorSearchExample
+  def initialize
+    @db = PG.connect(DB_CONFIG)
+    @client = Prescient.client(:ollama)
+  end
+
+  def run_example
+    puts "\n--- Setting up vector search example ---"
+    
+    # Check if services are available
+    unless check_services_available
+      puts "❌ Required services not available. Please start with: docker-compose up -d"
+      return
+    end
+
+    # 1. Generate and store embeddings for existing documents
+    puts "\n📊 Generating embeddings for sample documents..."
+    generate_document_embeddings
+
+    # 2. Perform similarity search
+    puts "\n🔍 Performing similarity searches..."
+    search_examples
+
+    # 3. Advanced search with filtering
+    puts "\n🎯 Advanced search with metadata filtering..."
+    advanced_search_examples
+
+    # 4. Demonstrate different distance functions
+    puts "\n📏 Comparing different distance functions..."
+    compare_distance_functions
+
+    puts "\n🎉 Vector search example completed!"
+  end
+
+  private
+
+  def check_services_available
+    # Check database connection
+    begin
+      result = @db.exec("SELECT 1")
+      puts "✅ PostgreSQL connected"
+    rescue PG::Error => e
+      puts "❌ PostgreSQL connection failed: #{e.message}"
+      return false
+    end
+
+    # Check pgvector extension
+    begin
+      result = @db.exec("SELECT * FROM pg_extension WHERE extname = 'vector'")
+      if result.ntuples > 0
+        puts "✅ pgvector extension available"
+      else
+        puts "❌ pgvector extension not found"
+        return false
+      end
+    rescue PG::Error => e
+      puts "❌ pgvector check failed: #{e.message}"
+      return false
+    end
+
+    # Check Ollama connection
+    if @client.available?
+      puts "✅ Ollama connected"
+    else
+      puts "❌ Ollama not available"
+      return false
+    end
+
+    true
+  end
+
+  def generate_document_embeddings
+    # Get documents that don't have embeddings yet
+    query = <<~SQL
+      SELECT d.id, d.title, d.content 
+      FROM documents d
+      LEFT JOIN document_embeddings de ON d.id = de.document_id 
+        AND de.embedding_provider = 'ollama' 
+        AND de.embedding_model = 'nomic-embed-text'
+      WHERE de.id IS NULL
+      LIMIT 10
+    SQL
+
+    result = @db.exec(query)
+    
+    if result.ntuples == 0
+      puts "   All documents already have embeddings"
+      return
+    end
+
+    result.each do |row|
+      document_id = row['id']
+      title = row['title']
+      content = row['content']
+      
+      puts "   Generating embedding for: #{title}"
+      
+      begin
+        # Generate embedding using Prescient
+        embedding = @client.generate_embedding(content)
+        
+        # Store in database
+        insert_embedding(document_id, embedding, content, 'ollama', 'nomic-embed-text', 768)
+        
+        puts "   ✅ Stored embedding (#{embedding.length} dimensions)"
+        
+      rescue Prescient::Error => e
+        puts "   ❌ Failed to generate embedding: #{e.message}"
+      end
+    end
+  end
+
+  def insert_embedding(document_id, embedding, text, provider, model, dimensions)
+    # Convert Ruby array to PostgreSQL vector format
+    vector_str = "[#{embedding.join(',')}]"
+    
+    query = <<~SQL
+      INSERT INTO document_embeddings 
+      (document_id, embedding_provider, embedding_model, embedding_dimensions, embedding, embedding_text)
+      VALUES ($1, $2, $3, $4, $5, $6)
+    SQL
+    
+    @db.exec_params(query, [document_id, provider, model, dimensions, vector_str, text])
+  end
+
+  def search_examples
+    search_queries = [
+      "How to learn programming?",
+      "What is machine learning?",
+      "Database optimization techniques",
+      "API security best practices"
+    ]
+
+    search_queries.each do |query_text|
+      puts "\n🔍 Searching for: '#{query_text}'"
+      perform_similarity_search(query_text, limit: 3)
+    end
+  end
+
+  def perform_similarity_search(query_text, limit: 5, distance_function: 'cosine')
+    begin
+      # Generate embedding for query
+      query_embedding = @client.generate_embedding(query_text)
+      query_vector = "[#{query_embedding.join(',')}]"
+      
+      # Choose distance operator based on function
+      distance_op = case distance_function
+                   when 'cosine' then '<=>'
+                   when 'l2' then '<->'
+                   when 'inner_product' then '<#>'
+                   else '<=>'
+                   end
+
+      # Perform similarity search
+      search_query = <<~SQL
+        SELECT 
+          d.title,
+          d.content,
+          d.metadata,
+          de.embedding #{distance_op} $1::vector AS distance,
+          1 - (de.embedding <=> $1::vector) AS cosine_similarity
+        FROM documents d
+        JOIN document_embeddings de ON d.id = de.document_id
+        WHERE de.embedding_provider = 'ollama' 
+          AND de.embedding_model = 'nomic-embed-text'
+        ORDER BY de.embedding #{distance_op} $1::vector
+        LIMIT $2
+      SQL
+
+      result = @db.exec_params(search_query, [query_vector, limit])
+      
+      if result.ntuples == 0
+        puts "   No results found"
+        return
+      end
+
+      result.each_with_index do |row, index|
+        similarity = (row['cosine_similarity'].to_f * 100).round(1)
+        puts "   #{index + 1}. #{row['title']} (#{similarity}% similar)"
+        puts "      #{row['content'][0..100]}..."
+        
+        # Show metadata if available
+        if row['metadata'] && !row['metadata'].empty?
+          metadata = JSON.parse(row['metadata'])
+          tags = metadata['tags']&.join(', ')
+          puts "      Tags: #{tags}" if tags
+        end
+        puts
+      end
+      
+    rescue Prescient::Error => e
+      puts "   ❌ Search failed: #{e.message}"
+    rescue PG::Error => e
+      puts "   ❌ Database error: #{e.message}"
+    end
+  end
+
+  def advanced_search_examples
+    # Search with metadata filtering
+    puts "\n🎯 Search for programming content with beginner difficulty:"
+    advanced_search("programming basics", tags: ["programming"], difficulty: "beginner")
+    
+    puts "\n🎯 Search for AI/ML content:"
+    advanced_search("artificial intelligence", tags: ["ai", "machine-learning"])
+  end
+
+  def advanced_search(query_text, filters = {})
+    begin
+      query_embedding = @client.generate_embedding(query_text)
+      query_vector = "[#{query_embedding.join(',')}]"
+      
+      # Build WHERE clause for metadata filtering
+      where_conditions = ["de.embedding_provider = 'ollama'", "de.embedding_model = 'nomic-embed-text'"]
+      params = [query_vector]
+      param_index = 2
+
+      filters.each do |key, value|
+        case key
+        when :tags
+          # Filter by tags array overlap
+          where_conditions << "d.metadata->'tags' ?| $#{param_index}::text[]"
+          params << value
+          param_index += 1
+        when :difficulty
+          # Filter by exact difficulty match
+          where_conditions << "d.metadata->>'difficulty' = $#{param_index}"
+          params << value
+          param_index += 1
+        when :source_type
+          # Filter by source type
+          where_conditions << "d.source_type = $#{param_index}"
+          params << value
+          param_index += 1
+        end
+      end
+
+      search_query = <<~SQL
+        SELECT 
+          d.title,
+          d.content,
+          d.metadata,
+          de.embedding <=> $1::vector AS cosine_distance,
+          1 - (de.embedding <=> $1::vector) AS cosine_similarity
+        FROM documents d
+        JOIN document_embeddings de ON d.id = de.document_id
+        WHERE #{where_conditions.join(' AND ')}
+        ORDER BY de.embedding <=> $1::vector
+        LIMIT 3
+      SQL
+
+      result = @db.exec_params(search_query, params)
+      
+      if result.ntuples == 0
+        puts "   No results found with the specified filters"
+        return
+      end
+
+      result.each_with_index do |row, index|
+        similarity = (row['cosine_similarity'].to_f * 100).round(1)
+        puts "   #{index + 1}. #{row['title']} (#{similarity}% similar)"
+        
+        metadata = JSON.parse(row['metadata'])
+        puts "      Difficulty: #{metadata['difficulty']}"
+        puts "      Tags: #{metadata['tags']&.join(', ')}"
+        puts "      #{row['content'][0..80]}..."
+        puts
+      end
+      
+    rescue Prescient::Error => e
+      puts "   ❌ Search failed: #{e.message}"
+    rescue PG::Error => e
+      puts "   ❌ Database error: #{e.message}"
+    end
+  end
+
+  def compare_distance_functions
+    query_text = "programming languages and development"
+    
+    puts "\n📏 Comparing distance functions for: '#{query_text}'"
+    
+    %w[cosine l2 inner_product].each do |distance_func|
+      puts "\n   #{distance_func.upcase} Distance:"
+      perform_similarity_search(query_text, limit: 2, distance_function: distance_func)
+    end
+  end
+
+  def cleanup
+    @db.close if @db
+  end
+end
+
+# Run the example
+begin
+  example = VectorSearchExample.new
+  example.run_example
+rescue StandardError => e
+  puts "❌ Example failed: #{e.message}"
+  puts e.backtrace.first(5).join("\n")
+ensure
+  example&.cleanup
+end
+
+puts "\n💡 Next steps:"
+puts "   - Try different embedding models (OpenAI, HuggingFace)"
+puts "   - Implement hybrid search (vector + keyword)"
+puts "   - Add document chunking for large texts"
+puts "   - Experiment with different similarity thresholds"
+puts "   - Add result re-ranking and filtering"

data/lib/prescient/base.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+class Prescient::Base
+  attr_reader :options
+
+  def initialize(**options)
+    @options = options
+    validate_configuration!
+  end
+
+  # Abstract methods that must be implemented by subclasses
+  def generate_embedding(text)
+    raise NotImplementedError, "#{self.class} must implement #generate_embedding"
+  end
+
+  def generate_response(prompt, context_items = [], **options)
+    raise NotImplementedError, "#{self.class} must implement #generate_response"
+  end
+
+  def health_check
+    raise NotImplementedError, "#{self.class} must implement #health_check"
+  end
+
+  def available?
+    health_check[:status] == 'healthy'
+  rescue StandardError
+    false
+  end
+
+  protected
+
+  def validate_configuration!
+    # Override in subclasses to validate required configuration
+  end
+
+  def handle_errors
+    yield
+  rescue Prescient::Error
+    # Re-raise Prescient errors without wrapping
+    raise
+  rescue Net::ReadTimeout, Net::OpenTimeout => e
+    raise Prescient::ConnectionError, "Request timeout: #{e.message}"
+  rescue Net::HTTPError => e
+    raise Prescient::ConnectionError, "HTTP error: #{e.message}"
+  rescue JSON::ParserError => e
+    raise Prescient::InvalidResponseError, "Invalid JSON response: #{e.message}"
+  rescue StandardError => e
+    raise Prescient::Error, "Unexpected error: #{e.message}"
+  end
+
+  def normalize_embedding(embedding, target_dimensions)
+    return nil unless embedding.is_a?(Array)
+    return embedding if embedding.length == target_dimensions
+
+    if embedding.length > target_dimensions
+      # Truncate
+      embedding.first(target_dimensions)
+    else
+      # Pad with zeros
+      embedding + Array.new(target_dimensions - embedding.length, 0.0)
+    end
+  end
+
+  def clean_text(text)
+    return '' if text.nil? || text.to_s.strip.empty?
+
+    cleaned = text.to_s
+      .strip
+      .gsub(/\s+/, ' ')
+
+    # Limit length for most models
+    cleaned.length > 8000 ? cleaned[0, 8000] : cleaned
+  end
+
+  # Default prompt templates - can be overridden in provider options
+  def default_prompt_templates
+    {
+      system_prompt:         'You are a helpful AI assistant. Answer questions clearly and accurately.',
+      no_context_template:   <<~TEMPLATE.strip,
+        %<system_prompt>s
+
+        Question: %<query>s
+
+        Please provide a helpful response based on your knowledge.
+      TEMPLATE
+      with_context_template: <<~TEMPLATE.strip,
+        %<system_prompt>s Use the following context to answer the question. If the context doesn't contain relevant information, say so clearly.
+
+        Context:
+        %<context>s
+
+        Question: %<query>s
+
+        Please provide a helpful response based on the context above.
+      TEMPLATE
+    }
+  end
+
+  # Build prompt using configurable templates
+  def build_prompt(query, context_items = [])
+    templates = default_prompt_templates.merge(@options[:prompt_templates] || {})
+    system_prompt = templates[:system_prompt]
+
+    if context_items.empty?
+      templates[:no_context_template] % {
+        system_prompt: system_prompt,
+        query:         query,
+      }
+    else
+      context_text = context_items.map.with_index(1) { |item, index|
+        "#{index}. #{format_context_item(item)}"
+      }.join("\n\n")
+
+      templates[:with_context_template] % {
+        system_prompt: system_prompt,
+        context:       context_text,
+        query:         query,
+      }
+    end
+  end
+
+  # Minimal default context configuration - users should define their own contexts
+  def default_context_configs
+    {
+      # Generic fallback configuration - works with any hash structure
+      'default' => {
+        fields:           [], # Will be dynamically determined from item keys
+        format:           nil, # Will use fallback formatting
+        embedding_fields: [], # Will use all string/text fields
+      },
+    }
+  end
+
+  # Extract text for embedding generation based on context configuration
+  def extract_embedding_text(item, context_type = nil)
+    return item.to_s unless item.is_a?(Hash)
+
+    config = resolve_context_config(item, context_type)
+    text_values = extract_configured_fields(item, config) || extract_text_values(item)
+    text_values.join(' ').strip
+  end
+
+  # Extract text values from hash, excluding non-textual fields
+  def extract_text_values(item)
+    # Common fields to exclude from embedding text
+    exclude_fields = ['id', '_id', 'uuid', 'created_at', 'updated_at', 'timestamp', 'version', 'status', 'active']
+
+    item.filter_map { |key, value|
+      next if exclude_fields.include?(key.to_s.downcase)
+      next unless value.is_a?(String) || value.is_a?(Numeric)
+      next if value.to_s.strip.empty?
+
+      value.to_s
+    }
+  end
+
+  # Generic context item formatting using configurable contexts
+  def format_context_item(item)
+    case item
+    when Hash then format_hash_item(item)
+    when String then item
+    else item.to_s
+    end
+  end
+
+  private
+
+  # Resolve context configuration for an item
+  def resolve_context_config(item, context_type)
+    context_configs = default_context_configs.merge(@options[:context_configs] || {})
+    return context_configs['default'] if context_configs.empty?
+
+    detected_type = context_type || detect_context_type(item)
+    context_configs[detected_type] || context_configs['default']
+  end
+
+  # Extract fields configured for embeddings
+  def extract_configured_fields(item, config)
+    return nil unless config[:embedding_fields]&.any?
+
+    config[:embedding_fields].filter_map { |field| item[field] || item[field.to_sym] }
+  end
+
+  # Format a hash item using context configuration
+  def format_hash_item(item)
+    config = resolve_context_config(item, nil)
+    return fallback_format_hash(item) unless config[:format]
+
+    format_data = build_format_data(item, config)
+    return fallback_format_hash(item) unless format_data.any?
+
+    apply_format_template(config[:format], format_data) || fallback_format_hash(item)
+  end
+
+  # Build format data from item fields
+  def build_format_data(item, config)
+    format_data = {}
+    fields_to_check = config[:fields].any? ? config[:fields] : item.keys.map(&:to_s)
+
+    fields_to_check.each do |field|
+      value = item[field] || item[field.to_sym]
+      format_data[field.to_sym] = value if value
+    end
+
+    format_data
+  end
+
+  # Apply format template with error handling
+  def apply_format_template(template, format_data)
+    template % format_data
+  rescue KeyError
+    nil
+  end
+
+  # Detect context type from item structure
+  def detect_context_type(item)
+    return 'default' unless item.is_a?(Hash)
+
+    # Check for explicit type fields (user-defined)
+    return item['type'].to_s if item['type']
+    return item['context_type'].to_s if item['context_type']
+    return item['model_type'].to_s.downcase if item['model_type']
+
+    # If no explicit type and user has configured contexts, try to match
+    context_configs = @options[:context_configs] || {}
+    return match_context_by_fields(item, context_configs) if context_configs.any?
+
+    # Default fallback
+    'default'
+  end
+
+  # Match context type based on configured field patterns
+  def match_context_by_fields(item, context_configs)
+    item_fields = item.keys.map(&:to_s)
+    best_match = find_best_field_match(item_fields, context_configs)
+    best_match || 'default'
+  end
+
+  # Find the best matching context configuration
+  def find_best_field_match(item_fields, context_configs)
+    best_match = nil
+    best_score = 0
+
+    context_configs.each do |context_type, config|
+      next unless config[:fields]&.any?
+
+      score = calculate_field_match_score(item_fields, config[:fields])
+      next unless score >= 0.5 && score > best_score
+
+      best_match = context_type
+      best_score = score
+    end
+
+    best_match
+  end
+
+  # Calculate field matching score
+  def calculate_field_match_score(item_fields, config_fields)
+    return 0 if config_fields.empty?
+
+    matching_fields = (item_fields & config_fields).size
+    matching_fields.to_f / config_fields.size
+  end
+
+  # Fallback formatting for hash items
+  def fallback_format_hash(item, format_data = nil)
+    # Fallback: join key-value pairs
+    (format_data || item).map { |k, v| "#{k}: #{v}" }.join(', ')
+  end
+end

data/lib/prescient/client.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Prescient
+  class Client
+    attr_reader :provider_name
+    attr_reader :provider
+
+    def initialize(provider_name = nil)
+      @provider_name = provider_name || Prescient.configuration.default_provider
+      @provider = Prescient.configuration.provider(@provider_name)
+
+      raise Prescient::Error, "Provider not found: #{@provider_name}" unless @provider
+    end
+
+    def generate_embedding(text, **options)
+      with_error_handling do
+        if options.any?
+          @provider.generate_embedding(text, **options)
+        else
+          @provider.generate_embedding(text)
+        end
+      end
+    end
+
+    def generate_response(prompt, context_items = [], **options)
+      with_error_handling do
+        if options.any?
+          @provider.generate_response(prompt, context_items, **options)
+        else
+          @provider.generate_response(prompt, context_items)
+        end
+      end
+    end
+
+    def health_check
+      @provider.health_check
+    end
+
+    def available?
+      @provider.available?
+    end
+
+    def provider_info
+      {
+        name:      @provider_name,
+        class:     @provider.class.name.split('::').last,
+        available: available?,
+        options:   sanitize_options(@provider.options),
+      }
+    end
+
+    def method_missing(method_name, ...)
+      if @provider.respond_to?(method_name)
+        @provider.send(method_name, ...)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      @provider.respond_to?(method_name, include_private) || super
+    end
+
+    private
+
+    def sanitize_options(options)
+      sensitive_keys = [:api_key, :password, :token, :secret]
+      options.reject { |key, _| sensitive_keys.include?(key.to_sym) }
+    end
+
+    def with_error_handling
+      retries = 0
+      begin
+        yield
+      rescue Prescient::RateLimitError => e
+        raise e unless retries < Prescient.configuration.retry_attempts
+
+        retries += 1
+        sleep(Prescient.configuration.retry_delay * retries)
+        retry
+      rescue Prescient::ConnectionError => e
+        raise e unless retries < Prescient.configuration.retry_attempts
+
+        retries += 1
+        sleep(Prescient.configuration.retry_delay)
+        retry
+      end
+    end
+  end
+
+  # Convenience methods for quick access
+  def self.client(provider_name = nil)
+    Client.new(provider_name)
+  end
+
+  def self.generate_embedding(text, provider: nil, **options)
+    client(provider).generate_embedding(text, **options)
+  end
+
+  def self.generate_response(prompt, context_items = [], provider: nil, **options)
+    client(provider).generate_response(prompt, context_items, **options)
+  end
+
+  def self.health_check(provider: nil)
+    client(provider).health_check
+  end
+end