hokipoki 0.3.4 → 0.5.0

data/lib/hokipoki/vector_engine.rb

@@ -0,0 +1,525 @@
+# frozen_string_literal: true
+
+require 'sqlite3'
+require 'json'
+require 'digest'
+
+module Hokipoki
+  # Core Vector Engine - The brain behind parasitic intelligence
+  # Uses template-based compression for 75% storage reduction
+  class VectorEngine
+    include Singleton
+
+    def initialize
+      @db_path = File.expand_path('~/.hokipoki/vectors.db')
+      @template_store = nil
+      @keyword_index = nil
+      @vector_cache = {}
+      @learning_patterns = {}
+      @stats = {
+        total_vectors: 0,
+        successful_retrievals: 0,
+        failed_retrievals: 0,
+        compression_ratio: 0.0
+      }
+
+      ensure_db_directory
+      initialize_database
+      display_startup_message
+    end
+
+    # Main API - Retrieve intelligent facts with template generation
+    def retrieve_facts(query, token_budget: 1500)
+      $stdout.puts "🧠 VECTOR ENGINE: Analyzing query intent..."
+
+      begin
+        # 1. Analyze query intent with multiple dimensions
+        intent = analyze_query_intent(query)
+        $stdout.puts "  🎯 Intent detected: #{intent}"
+
+        # 2. Extract technical keywords
+        keywords = extract_technical_keywords(query)
+        $stdout.puts "  📝 Keywords: #{keywords.join(', ')}"
+
+        # 3. Find matching template vectors
+        matching_vectors = find_matching_vectors(keywords, intent)
+        $stdout.puts "  🔍 Found #{matching_vectors.length} matching vectors"
+
+        if matching_vectors.any?
+          # 4. Generate content from templates
+          generated_content = generate_content_from_vectors(matching_vectors, intent, keywords)
+
+          # 5. Apply token budget management
+          final_content = apply_token_budget(generated_content, token_budget)
+
+          @stats[:successful_retrievals] += 1
+          $stdout.puts "  ✅ Content generated (#{estimate_tokens(final_content)} tokens)"
+
+          # Learn from successful retrieval
+          learn_from_success(query, keywords, intent, final_content)
+
+          return [final_content]
+        else
+          # Fallback to template-based generation
+          $stdout.puts "  ⚠️ No vectors found, using template fallback"
+          fallback_content = generate_fallback_content(query, intent, keywords)
+
+          @stats[:failed_retrievals] += 1
+          return [fallback_content]
+        end
+
+      rescue => e
+        $stdout.puts "  ❌ Vector engine error: #{e.message}"
+        @stats[:failed_retrievals] += 1
+        return generate_emergency_fallback(query, intent)
+      end
+    end
+
+    # Store content as template-based vector (75% compression)
+    def store_template_vector(content, source_file, metadata = {})
+      $stdout.puts "🦠 STORING VECTOR: #{File.basename(source_file || 'unknown')}"
+
+      # 1. Detect content template type
+      template_type = detect_template_type(content)
+      $stdout.puts "  📋 Template type: #{template_type}"
+
+      # 2. Extract atomic keywords
+      keywords = extract_atomic_keywords(content)
+      $stdout.puts "  🔑 Extracted #{keywords.length} keywords"
+
+      # 3. Calculate keyword weights (TF-IDF style)
+      keyword_weights = calculate_keyword_weights(keywords, content)
+
+      # 4. Determine generation parameters
+      generation_params = {
+        complexity: assess_content_complexity(content),
+        style: detect_content_style(content),
+        domain: detect_content_domain(content),
+        original_length: content.length
+      }
+
+      # 5. Store compressed representation
+      vector_id = store_vector_record(
+        template_type: template_type,
+        keywords: keywords,
+        keyword_weights: keyword_weights,
+        generation_params: generation_params,
+        source_file: source_file,
+        metadata: metadata
+      )
+
+      # 6. Update statistics
+      original_size = content.bytesize
+      compressed_size = calculate_compressed_size(keywords, template_type, generation_params)
+      compression_ratio = ((original_size - compressed_size).to_f / original_size * 100).round(2)
+
+      @stats[:total_vectors] += 1
+      @stats[:compression_ratio] = ((@stats[:compression_ratio] * (@stats[:total_vectors] - 1)) + compression_ratio) / @stats[:total_vectors]
+
+      $stdout.puts "  ✅ Stored with #{compression_ratio}% compression"
+
+      vector_id
+    end
+
+    # Get engine statistics
+    def statistics
+      success_rate = @stats[:successful_retrievals] + @stats[:failed_retrievals] > 0 ?
+        (@stats[:successful_retrievals].to_f / (@stats[:successful_retrievals] + @stats[:failed_retrievals]) * 100).round(1) : 0
+
+      {
+        total_vectors: @stats[:total_vectors],
+        success_rate: success_rate,
+        average_compression: @stats[:compression_ratio].round(2),
+        cache_size: @vector_cache.size,
+        learning_patterns: @learning_patterns.size,
+        database_size: File.exist?(@db_path) ? File.size(@db_path) : 0
+      }
+    end
+
+    # Display status for Claude visibility
+    def display_status
+      stats = statistics
+
+      message = "\n🧠 VECTOR ENGINE STATUS\n"
+      message += "=" * 40 + "\n"
+      message += "📊 Total Vectors: #{stats[:total_vectors]}\n"
+      message += "✅ Success Rate: #{stats[:success_rate]}%\n"
+      message += "🗜️  Avg Compression: #{stats[:average_compression]}%\n"
+      message += "💾 Cache Size: #{stats[:cache_size]} entries\n"
+      message += "🧠 Learning Patterns: #{stats[:learning_patterns]}\n"
+      message += "💿 Database Size: #{(stats[:database_size] / 1024.0).round(2)} KB\n"
+      message += "=" * 40 + "\n"
+
+      $stdout.puts message
+      puts message
+
+      message
+    end
+
+    # Scan and vectorize project files
+    def scan_project(project_path = Dir.pwd)
+      $stdout.puts "🔍 SCANNING PROJECT: #{File.basename(project_path)}"
+
+      # Find relevant files
+      file_patterns = %w[**/*.rb **/*.js **/*.erb **/*.yml **/*.md]
+      files = file_patterns.flat_map { |pattern| Dir.glob(File.join(project_path, pattern)) }
+
+      # Filter out excluded paths
+      excluded_patterns = %w[node_modules vendor log tmp .git]
+      files = files.reject { |file| excluded_patterns.any? { |pattern| file.include?(pattern) } }
+
+      $stdout.puts "📁 Found #{files.length} files to process"
+
+      processed = 0
+      files.each_with_index do |file, index|
+        begin
+          $stdout.puts "  [#{index + 1}/#{files.length}] Processing: #{File.basename(file)}"
+
+          content = File.read(file)
+          next if content.strip.empty?
+
+          store_template_vector(content, file, {
+            file_type: File.extname(file),
+            processed_at: Time.current.iso8601
+          })
+
+          processed += 1
+
+        rescue => e
+          $stdout.puts "    ❌ Error processing #{file}: #{e.message}"
+        end
+      end
+
+      $stdout.puts "✅ PROJECT SCAN COMPLETE: #{processed}/#{files.length} files processed"
+      processed
+    end
+
+    private
+
+    def ensure_db_directory
+      db_dir = File.dirname(@db_path)
+      FileUtils.mkdir_p(db_dir) unless Dir.exist?(db_dir)
+    end
+
+    def initialize_database
+      @db = SQLite3::Database.new(@db_path)
+      @db.results_as_hash = true
+
+      create_tables_if_needed
+    end
+
+    def create_tables_if_needed
+      @db.execute <<~SQL
+        CREATE TABLE IF NOT EXISTS vectors (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          template_type TEXT NOT NULL,
+          keywords TEXT NOT NULL,
+          keyword_weights TEXT NOT NULL,
+          generation_params TEXT NOT NULL,
+          source_file TEXT,
+          metadata TEXT,
+          created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+          updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+        )
+      SQL
+
+      @db.execute <<~SQL
+        CREATE INDEX IF NOT EXISTS idx_template_type ON vectors(template_type)
+      SQL
+
+      @db.execute <<~SQL
+        CREATE INDEX IF NOT EXISTS idx_keywords ON vectors(keywords)
+      SQL
+    end
+
+    def analyze_query_intent(query)
+      # Multi-dimensional intent analysis
+      intents = []
+
+      # Primary intent detection
+      case query.downcase
+      when /\b(implement|create|build|make|add|generate|write)\b/
+        intents << :implementation
+      when /\b(error|bug|fix|debug|troubleshoot|issue|problem)\b/
+        intents << :debugging
+      when /\b(how|what|why|when|where|explain|understand|learn)\b/
+        intents << :learning
+      when /\b(optimize|improve|enhance|better|faster|performance)\b/
+        intents << :optimization
+      when /\b(example|show|demo|sample|tutorial|guide)\b/
+        intents << :reference
+      when /\b(test|testing|spec|rspec|jest|unit|integration)\b/
+        intents << :testing
+      end
+
+      # Secondary intent detection
+      intents << :css if query.match?(/\b(css|style|stylesheet|tailwind|bootstrap)\b/i)
+      intents << :javascript if query.match?(/\b(js|javascript|node|npm|react|vue)\b/i)
+      intents << :ruby if query.match?(/\b(ruby|rails|gem|bundler|rake)\b/i)
+      intents << :database if query.match?(/\b(database|sql|migration|model|table)\b/i)
+
+      # Default to general if no specific intent detected
+      intents << :general if intents.empty?
+
+      intents.first || :general
+    end
+
+    def extract_technical_keywords(query)
+      # Remove stop words and extract meaningful terms
+      stop_words = %w[
+        the a an and or but in on at to for of with by from
+        how do i can you please help me show get make create
+        what is are was were will would should could might
+        this that these those here there where when why
+      ]
+
+      # Extract words, filter stop words, keep technical terms
+      words = query.downcase
+                  .gsub(/[^\w\s]/, ' ')
+                  .split(/\s+/)
+                  .reject { |word| stop_words.include?(word) || word.length < 2 }
+                  .select { |word| word.length > 2 || technical_term?(word) }
+
+      # Remove duplicates and limit to most relevant
+      words.uniq.first(10)
+    end
+
+    def technical_term?(word)
+      # Short technical terms that should be preserved
+      technical_terms = %w[css js sql api url ui ux db id]
+      technical_terms.include?(word.downcase)
+    end
+
+    def find_matching_vectors(keywords, intent)
+      # Build SQL query to find matching vectors
+      keyword_conditions = keywords.map { |keyword| "keywords LIKE '%#{keyword}%'" }.join(' OR ')
+
+      sql = if keyword_conditions.present?
+        "SELECT * FROM vectors WHERE (#{keyword_conditions}) ORDER BY created_at DESC LIMIT 10"
+      else
+        "SELECT * FROM vectors ORDER BY created_at DESC LIMIT 5"
+      end
+
+      @db.execute(sql)
+    end
+
+    def generate_content_from_vectors(vectors, intent, keywords)
+      template_store = get_template_store
+
+      generated_parts = vectors.map do |vector|
+        template_type = vector['template_type']
+        stored_keywords = JSON.parse(vector['keywords'])
+        generation_params = JSON.parse(vector['generation_params'])
+
+        template_store.generate_content(
+          template_type,
+          keywords: keywords + stored_keywords,
+          intent: intent,
+          params: generation_params
+        )
+      end
+
+      # Combine and deduplicate content
+      combined_content = generated_parts.compact.join(' | ')
+      combined_content.present? ? combined_content : generate_fallback_content(keywords.join(' '), intent, keywords)
+    end
+
+    def get_template_store
+      @template_store ||= TemplateStore.new
+    end
+
+    def apply_token_budget(content, budget)
+      estimated_tokens = estimate_tokens(content)
+
+      if estimated_tokens <= budget
+        content
+      else
+        # Truncate content to fit budget
+        chars_per_token = content.length.to_f / estimated_tokens
+        max_chars = (budget * chars_per_token * 0.9).to_i # 90% safety margin
+
+        content[0..max_chars].strip + "..."
+      end
+    end
+
+    def estimate_tokens(text)
+      return 0 if text.nil? || text.empty?
+      # Conservative estimation: ~4 characters per token
+      (text.length / 4.0).ceil
+    end
+
+    def generate_fallback_content(query, intent, keywords)
+      case intent
+      when :implementation
+        "Implementation context: #{keywords.join(', ')}. Consider Rails conventions, security best practices, and maintainable code patterns."
+      when :debugging
+        "Debugging context: #{keywords.join(', ')}. Check logs, verify configurations, test incrementally. Common issues: environment variables, dependencies, permissions."
+      when :learning
+        "Learning context: #{keywords.join(', ')}. Focus on fundamentals, official documentation, and proven patterns. Practice with simple examples first."
+      when :optimization
+        "Optimization context: #{keywords.join(', ')}. Profile before optimizing, focus on bottlenecks, consider caching, database queries, and algorithmic improvements."
+      else
+        "Context: #{keywords.join(', ')}. Use best practices, follow conventions, prioritize readability and maintainability."
+      end
+    end
+
+    def generate_emergency_fallback(query, intent)
+      ["Emergency context: Basic guidance available. Check documentation and verify system configuration."]
+    end
+
+    def detect_template_type(content)
+      # Analyze content to determine template type
+      case content
+      when /class\s+\w+.*< ApplicationRecord/, /belongs_to/, /has_many/, /validates/
+        'active_record_model'
+      when /class\s+\w+.*< ApplicationController/, /def\s+\w+/, /render/, /redirect_to/
+        'rails_controller'
+      when /def\s+\w+/, /class\s+\w+/, /module\s+\w+/
+        'ruby_class'
+      when /function\s+\w+/, /const\s+\w+/, /=>\s*/, /async\s+function/
+        'javascript_code'
+      when /\.css/, /background:/, /color:/, /font-size:/, /margin:/, /padding:/
+        'css_styles'
+      when /<%=/, /<% /, /<%# /, /<html/, /<div/, /<span/
+        'erb_template'
+      when /describe/, /it\s+["']/, /expect/, /test/, /spec/
+        'test_spec'
+      when /#\s+[A-Z]/, /##\s+/, /###\s+/, /\*\s+/, /-\s+/
+        'documentation'
+      when /migration/, /create_table/, /add_column/, /drop_table/
+        'database_migration'
+      when /config/, /settings/, /environment/, /secrets/
+        'configuration'
+      else
+        'general_code'
+      end
+    end
+
+    def extract_atomic_keywords(content)
+      keywords = []
+
+      # Extract Ruby methods
+      content.scan(/def\s+(\w+)/) { |match| keywords << "method_#{match[0]}" }
+
+      # Extract class names
+      content.scan(/class\s+(\w+)/) { |match| keywords << "class_#{match[0]}" }
+
+      # Extract constants
+      content.scan(/([A-Z][A-Z_]+)\s*=/) { |match| keywords << "constant_#{match[0]}" }
+
+      # Extract CSS classes
+      content.scan(/class=["\']([^"\']+)["\']/) { |match|
+        match[0].split(/\s+/).each { |cls| keywords << "css_#{cls}" }
+      }
+
+      # Extract technical terms
+      technical_terms = content.scan(/\b(rails|react|vue|angular|bootstrap|tailwind|postgres|mysql|redis|docker|kubernetes|aws|api|rest|graphql|jwt|oauth|json|xml|html|css|javascript|ruby|python|sql)\b/i)
+      technical_terms.flatten.each { |term| keywords << "tech_#{term.downcase}" }
+
+      # Extract file extensions and types
+      content.scan(/\.(\w+)$/) { |match| keywords << "filetype_#{match[0]}" }
+
+      keywords.uniq.first(20) # Limit to most relevant keywords
+    end
+
+    def calculate_keyword_weights(keywords, content)
+      weights = {}
+      total_words = content.split(/\s+/).length
+
+      keywords.each do |keyword|
+        # Simple TF-IDF approximation
+        term_frequency = content.scan(/#{Regexp.escape(keyword)}/i).length
+        # Inverse document frequency approximation (assuming 1000 documents)
+        inverse_doc_freq = Math.log(1000.0 / [@stats[:total_vectors] + 1, 1].max)
+
+        weights[keyword] = (term_frequency.to_f / total_words) * inverse_doc_freq
+      end
+
+      weights
+    end
+
+    def assess_content_complexity(content)
+      # Simple complexity assessment
+      factors = 0
+      factors += 1 if content.length > 1000
+      factors += 1 if content.scan(/def\s+\w+/).length > 5
+      factors += 1 if content.scan(/class\s+\w+/).length > 1
+      factors += 1 if content.include?('module')
+      factors += 1 if content.scan(/\b(async|await|promise|callback)\b/i).any?
+
+      case factors
+      when 0..1 then 'simple'
+      when 2..3 then 'moderate'
+      else 'complex'
+      end
+    end
+
+    def detect_content_style(content)
+      return 'tutorial' if content.match?(/step\s+\d+|first|then|next|finally/i)
+      return 'reference' if content.match?(/api|documentation|docs|spec/i)
+      return 'example' if content.match?(/example|demo|sample|usage/i)
+      return 'guide' if content.match?(/guide|how\s+to|walkthrough/i)
+      'standard'
+    end
+
+    def detect_content_domain(content)
+      return 'frontend' if content.match?/(css|html|javascript|react|vue|angular)/i
+      return 'backend' if content.match?/(rails|ruby|controller|model|database)/i
+      return 'devops' if content.match?/(docker|kubernetes|aws|deployment|server)/i
+      return 'testing' if content.match?/(test|spec|rspec|jest|unit|integration)/i
+      'general'
+    end
+
+    def store_vector_record(template_type:, keywords:, keyword_weights:, generation_params:, source_file:, metadata:)
+      vector_id = Digest::SHA256.hexdigest("#{template_type}_#{keywords.join('_')}_#{Time.current.to_f}")
+
+      @db.execute(
+        "INSERT INTO vectors (id, template_type, keywords, keyword_weights, generation_params, source_file, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        [
+          vector_id,
+          template_type,
+          JSON.generate(keywords),
+          JSON.generate(keyword_weights),
+          JSON.generate(generation_params),
+          source_file,
+          JSON.generate(metadata)
+        ]
+      )
+
+      vector_id
+    end
+
+    def calculate_compressed_size(keywords, template_type, generation_params)
+      # Estimate compressed representation size
+      keywords_size = keywords.join('').bytesize
+      template_size = template_type.bytesize
+      params_size = JSON.generate(generation_params).bytesize
+
+      keywords_size + template_size + params_size + 100 # Base overhead
+    end
+
+    def learn_from_success(query, keywords, intent, content)
+      # Store successful pattern for future improvement
+      pattern_key = "#{intent}_#{keywords.first(3).join('_')}"
+
+      @learning_patterns[pattern_key] ||= { successes: 0, failures: 0, last_success: nil }
+      @learning_patterns[pattern_key][:successes] += 1
+      @learning_patterns[pattern_key][:last_success] = Time.current
+
+      # Keep only recent patterns to avoid memory bloat
+      if @learning_patterns.size > 1000
+        oldest_patterns = @learning_patterns.sort_by { |k, v| v[:last_success] || Time.at(0) }.first(200)
+        oldest_patterns.each { |pattern, _| @learning_patterns.delete(pattern) }
+      end
+    end
+
+    def display_startup_message
+      $stdout.puts "\n🧠 VECTOR ENGINE: Initialized successfully"
+      $stdout.puts "💾 Database: #{@db_path}"
+      $stdout.puts "📊 Ready for parasitic intelligence operations"
+    end
+  end
+end
+
+# Load template store dependency
+require_relative 'template_store'

data/lib/hokipoki/version.rb

@@ -1,3 +1,3 @@
 module Hokipoki
-  VERSION = "0.3.4"
+  VERSION = "0.5.0"
 end