hokipoki 0.5.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 241b9f3c0f528d0f8b0500b00bf7192403a00aa57235975947624590ec7d7e3a
-  data.tar.gz: 593c1527aaab3c3a23fdd22cfd7b3cd8b46386d5a3bca4ff8c327e2818f93e31
+  metadata.gz: 4fdce824f42ae5123c16881e6750eea924cff8b6b2cd498f4329050411994d18
+  data.tar.gz: 8b150ca5b979b06b4265db8c0cdb873045dece1d85b39ac440b436919a096bd0
 SHA512:
-  metadata.gz: 367b1dbf7a3293499f84dfd33c72dd772aa3d4e9dbf705f03c0b8a2a3747672ce277e8f2d87dac0ac95353735ef96b54bd137be5c11a1673c11076fa23feb532
-  data.tar.gz: b29d3769b6888e714b8cdd24188a2651c07059e0179b82275cf9e02f6f92e5b79fccfaf8d8af69bd30902f13c3c408f1aa08d83b6e0f5afe22599a84b54f3ab0
+  metadata.gz: '018c838374788cbd86af2fafe917568925b59cf6e137026c35326b4f2d137a4fe585939367c5c7935ac1e5719bd026a8ebe909742be45a45cf9e28a23b303bf9'
+  data.tar.gz: 216d2b1790a380a03e68c7895e450442679c00cb766226c3dc5c84d3cc0f442c2e18e4332d76a8463e7c5ae2438e753bce140767ef7bd722a91892a16973507e

data/bin/gemini_refiner

@@ -0,0 +1,354 @@
+#!/usr/bin/env ruby
+
+# Gemini Vector Refinement Engine
+# This script runs in a separate terminal and handles vector refinement
+
+require 'json'
+require 'net/http'
+require 'uri'
+require 'time'
+
+class GeminiRefiner
+  def initialize
+    @base_url = 'http://localhost:3000'
+    @template_file = '/tmp/gemini_refinement_template.json'
+    @status_file = '/tmp/gemini_status.log'
+    @current_refinement_file = '/tmp/gemini_current_refinement.json'
+    @batch_size = 10
+    @sleep_interval = 5
+    @refinement_count = 0
+    @start_time = Time.now
+
+    setup_logging
+    load_template
+  end
+
+  def start
+    display_startup_banner
+    check_connectivity
+
+    log_status('info', 'Gemini refinement engine started')
+    puts "🔄 Starting refinement loop..."
+    puts ""
+
+    main_refinement_loop
+  rescue Interrupt
+    puts "\n\n👋 Gemini refinement engine shutting down..."
+    cleanup
+    exit(0)
+  rescue => e
+    log_status('error', "Critical error: #{e.message}")
+    puts "❌ Critical error: #{e.message}"
+    exit(1)
+  end
+
+  private
+
+  def setup_logging
+    # Initialize log file
+    File.write(@status_file, "")
+  end
+
+  def load_template
+    if File.exist?(@template_file)
+      @template = JSON.parse(File.read(@template_file))
+      puts "📋 Template loaded from #{@template_file}"
+    else
+      puts "⚠️  Template file not found, using defaults"
+      @template = default_template
+    end
+  end
+
+  def display_startup_banner
+    puts "╔═══════════════════════════════════════════════╗"
+    puts "║        🧠 GEMINI REFINEMENT ENGINE           ║"
+    puts "╚═══════════════════════════════════════════════╝"
+    puts ""
+    puts "🎯 Configuration:"
+    puts "   Endpoint:     #{@base_url}/api/gemini"
+    puts "   Batch Size:   #{@batch_size} vectors"
+    puts "   Interval:     #{@sleep_interval} seconds"
+    puts "   Template:     #{@template_file}"
+    puts ""
+  end
+
+  def check_connectivity
+    puts "📡 Testing connectivity..."
+
+    begin
+      uri = URI("#{@base_url}/api/gemini/health_check")
+      response = Net::HTTP.get_response(uri)
+
+      if response.code == '200'
+        puts "   ✅ API endpoint: Connected"
+      else
+        puts "   ⚠️  API endpoint: #{response.code} #{response.message}"
+      end
+    rescue => e
+      puts "   ❌ API endpoint: Connection failed (#{e.message})"
+      puts ""
+      puts "🔧 Troubleshooting:"
+      puts "   - Make sure Rails server is running: rails server"
+      puts "   - Check if the forge_generator app is accessible"
+      puts ""
+      exit(1)
+    end
+
+    # Test Gemini CLI (mock for now)
+    if has_gemini_cli?
+      puts "   ✅ Gemini CLI: Available"
+    else
+      puts "   ⚠️  Gemini CLI: Using mock refinement"
+    end
+
+    puts ""
+  end
+
+  def main_refinement_loop
+    loop do
+      begin
+        # Fetch unrefined vectors
+        vectors = fetch_unrefined_vectors
+
+        if vectors && vectors.any?
+          puts "📥 Received #{vectors.count} vectors for refinement"
+
+          # Process each vector
+          refined_vectors = []
+          vectors.each_with_index do |vector, index|
+            refined = refine_vector(vector, index + 1, vectors.count)
+            refined_vectors << refined if refined
+          end
+
+          # Submit refined vectors
+          if refined_vectors.any?
+            submit_refined_vectors(refined_vectors)
+            @refinement_count += refined_vectors.count
+
+            # Update stats display
+            display_current_stats
+          end
+        else
+          puts "💤 No vectors to refine... waiting #{@sleep_interval}s"
+          clear_current_refinement
+        end
+
+        sleep(@sleep_interval)
+
+      rescue => e
+        log_status('error', "Refinement loop error: #{e.message}")
+        puts "❌ Error in refinement loop: #{e.message}"
+        puts "   Retrying in #{@sleep_interval} seconds..."
+        sleep(@sleep_interval)
+      end
+    end
+  end
+
+  def fetch_unrefined_vectors
+    uri = URI("#{@base_url}/api/gemini/unrefined_vectors?batch_size=#{@batch_size}")
+    response = Net::HTTP.get_response(uri)
+
+    if response.code == '200'
+      data = JSON.parse(response.body)
+      data['vectors']
+    else
+      log_status('error', "Failed to fetch vectors: #{response.code}")
+      nil
+    end
+  rescue => e
+    log_status('error', "Network error fetching vectors: #{e.message}")
+    nil
+  end
+
+  def refine_vector(vector, current_index, total_count)
+    vector_id = vector['id']
+    content = vector['content']
+
+    puts "🔄 [#{current_index}/#{total_count}] Refining vector ##{vector_id}..."
+
+    # Update current refinement status
+    update_current_refinement(vector_id, content, current_index, total_count)
+
+    # Perform refinement (mock implementation for now)
+    refined_data = perform_gemini_refinement(content)
+
+    log_status('success', "Refined vector ##{vector_id} (#{refined_data[:keywords]&.count || 0} keywords)")
+
+    {
+      'id' => vector_id,
+      'chunks' => refined_data[:chunks],
+      'keywords' => refined_data[:keywords],
+      'generators' => refined_data[:generators]
+    }
+  rescue => e
+    log_status('error', "Failed to refine vector ##{vector_id}: #{e.message}")
+    puts "   ❌ Error refining vector ##{vector_id}: #{e.message}"
+    nil
+  end
+
+  def perform_gemini_refinement(content)
+    # Mock refinement implementation
+    # In real implementation, this would call Gemini API
+
+    words = content.split
+    chunks = []
+    keywords = []
+
+    # Simple chunking (50 words max)
+    words.each_slice(50) do |chunk_words|
+      chunk_text = chunk_words.join(' ')
+      chunks << chunk_text
+
+      # Extract keywords (simple implementation)
+      chunk_keywords = extract_keywords(chunk_text)
+      keywords.concat(chunk_keywords)
+    end
+
+    # Create generators
+    generators = create_generators(keywords, chunks)
+
+    {
+      chunks: chunks,
+      keywords: keywords.uniq,
+      generators: generators
+    }
+  end
+
+  def extract_keywords(text)
+    # Simple keyword extraction (mock)
+    words = text.downcase.split(/\W+/)
+
+    # Filter meaningful words (longer than 3 chars, not common words)
+    common_words = %w[the and for are but not you all can had her was one our out day get has him how its may new now old see two way who boy did man its can all]
+
+    meaningful_words = words.select do |word|
+      word.length > 3 && !common_words.include?(word)
+    end
+
+    # Return top 5 most frequent
+    meaningful_words.tally.sort_by { |_, count| -count }.first(5).map(&:first)
+  end
+
+  def create_generators(keywords, chunks)
+    generators = []
+
+    # Create one generator per chunk with its keywords
+    chunks.each_with_index do |chunk, index|
+      chunk_keywords = keywords.sample(3) # Use 3 random keywords
+
+      generators << {
+        'trigger' => chunk_keywords,
+        'template' => "Based on the context: #{chunk.truncate(100)}",
+        'confidence' => 0.7 + (rand * 0.3) # Random confidence 0.7-1.0
+      }
+    end
+
+    generators
+  end
+
+  def submit_refined_vectors(refined_vectors)
+    uri = URI("#{@base_url}/api/gemini/submit_refined")
+
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Post.new(uri)
+    request['Content-Type'] = 'application/json'
+    request.body = { vectors: refined_vectors }.to_json
+
+    response = http.request(request)
+
+    if response.code == '200'
+      puts "✅ Successfully submitted #{refined_vectors.count} refined vectors"
+      log_status('success', "Submitted #{refined_vectors.count} refined vectors")
+    else
+      puts "⚠️  Submission warning: #{response.code} #{response.message}"
+      log_status('error', "Submission failed: #{response.code}")
+    end
+  rescue => e
+    puts "❌ Failed to submit refined vectors: #{e.message}"
+    log_status('error', "Submission error: #{e.message}")
+  end
+
+  def update_current_refinement(vector_id, content, current, total)
+    progress = ((current.to_f / total) * 100).round
+
+    current_data = {
+      'id' => vector_id,
+      'content' => content,
+      'progress' => progress,
+      'current' => current,
+      'total' => total,
+      'updated_at' => Time.now.iso8601
+    }
+
+    File.write(@current_refinement_file, current_data.to_json)
+  rescue
+    # Ignore file write errors
+  end
+
+  def clear_current_refinement
+    File.delete(@current_refinement_file) if File.exist?(@current_refinement_file)
+  rescue
+    # Ignore file delete errors
+  end
+
+  def display_current_stats
+    runtime = Time.now - @start_time
+    rate = @refinement_count / (runtime / 60.0) # per minute
+
+    puts ""
+    puts "📊 Session Stats:"
+    puts "   Refined:     #{@refinement_count} vectors"
+    puts "   Runtime:     #{(runtime / 60).round(1)} minutes"
+    puts "   Rate:        #{rate.round(1)}/min"
+    puts ""
+  end
+
+  def log_status(level, message)
+    timestamp = Time.now.strftime('%H:%M:%S')
+    log_entry = "#{timestamp} - #{level} - #{message}"
+
+    File.open(@status_file, 'a') do |f|
+      f.puts log_entry
+    end
+  rescue
+    # Ignore logging errors
+  end
+
+  def has_gemini_cli?
+    system('which gemini > /dev/null 2>&1') ||
+    system('which google-ai > /dev/null 2>&1') ||
+    ENV['GEMINI_API_KEY'] || ENV['GOOGLE_API_KEY']
+  end
+
+  def default_template
+    {
+      'mode' => 'vector_refinement',
+      'instructions' => {
+        'primary' => 'Refine each vector into atomic facts with keywords',
+        'chunking' => {
+          'max_words' => 50,
+          'create_generators' => true,
+          'keyword_density' => '3-5 per chunk'
+        }
+      }
+    }
+  end
+
+  def cleanup
+    clear_current_refinement
+    log_status('info', 'Gemini refinement engine stopped')
+  end
+end
+
+# String truncate method
+class String
+  def truncate(length)
+    self.length > length ? "#{self[0..length-3]}..." : self
+  end unless method_defined?(:truncate)
+end
+
+# Start the refiner
+if __FILE__ == $0
+  refiner = GeminiRefiner.new
+  refiner.start
+end

data/lib/generators/hive_mind/install_generator.rb

@@ -197,7 +197,7 @@ module HiveMind
 
     def display_completion_message
       say "\n#{@pastel.green.bold('✅ HokiPoki lightweight installation completed!')}"
-      say "\n#{@pastel.cyan.bold('🎯 Next Steps to Activate HiveMind:')}"
+      say "\n#{@pastel.cyan.bold('🎯 ESSENTIAL NEXT STEPS - START SERVICES:')}"
 
       say "\n  #{@pastel.yellow('1. Install gem dependencies:')}"
       say "    #{@pastel.cyan('bundle install')}"
@@ -205,16 +205,26 @@ module HiveMind
       say "\n  #{@pastel.yellow('2. Run database migrations:')}"
       say "    #{@pastel.cyan('rails db:migrate')}"
 
-      say "\n  #{@pastel.yellow('3. Activate Claude + HiveMind integration:')}"
-      say "    #{@pastel.cyan('Exit Claude and run:')}"
-      say "    #{@pastel.magenta('claude')} #{@pastel.dim('(this will auto-detect and activate HiveMind)')}"
-      say "    #{@pastel.dim('OR if already in Claude, exit and re-enter')}"
+      say "\n  #{@pastel.yellow('3. 🚀 START RAILS SERVER (REQUIRED):')}"
+      say "    #{@pastel.cyan('rails server')} #{@pastel.dim('(Keep this running in background)')}"
+      say "    #{@pastel.dim('📍 This enables HiveMind connection for Claude')}"
 
-      say "\n  #{@pastel.yellow('4. Verify HiveMind connection:')}"
-      say "    #{@pastel.dim('Look for:')} #{@pastel.green('🧠 HIVE_MIND IS CONNECTED')}"
+      say "\n  #{@pastel.yellow('4. 🧠 START HIVEMIND CONNECTION:')}"
+      say "    #{@pastel.cyan('rails g hive_mind:start')} #{@pastel.dim('(Run this to establish Claude link)')}"
+      say "    #{@pastel.dim('📍 Creates bin/hive_mind_connect script')}"
 
-      say "\n  #{@pastel.yellow('5. Test vector retrieval:')}"
-      say "    #{@pastel.dim('In Rails console:')} #{@pastel.cyan('HiveMindDocument.retrieve_targeted_facts(\"your query\")')}"
+      say "\n  #{@pastel.yellow('5. 🦠 ACTIVATE CLAUDE PARASITE:')}"
+      say "    #{@pastel.cyan('Exit Claude and restart:')} #{@pastel.magenta('claude')}"
+      say "    #{@pastel.dim('📍 Auto-detects running Rails server + HiveMind')}"
+
+      say "\n#{@pastel.red.bold('⚠️  CONNECTION STATUS INDICATORS:')}"
+      say "    #{@pastel.green('🧠 HIVE_MIND: CONNECTED')} #{@pastel.dim('- HiveMind active')}"
+      say "    #{@pastel.green('🦠 PARASITE: INTERCEPTING')} #{@pastel.dim('- Claude enhancement active')}"
+      say "    #{@pastel.green('💰 TOKEN SAVINGS: ACTIVE')} #{@pastel.dim('- Cost optimization running')}"
+
+      say "\n  #{@pastel.yellow('6. 🧪 TEST ENHANCED CLAUDE:')}"
+      say "    #{@pastel.dim('Ask Claude:')} #{@pastel.cyan('\"Explain this Rails app structure\"')}"
+      say "    #{@pastel.dim('Look for:')} #{@pastel.green('📊 Tokens Saved, ⚡ Processing Time, 🎯 Efficiency')}"
 
       say "\n#{@pastel.blue.bold('📁 Files Created:')}"
       say "  - config/initializers/hokipoki.rb"

data/lib/generators/hokipoki/install_generator.rb

@@ -53,17 +53,10 @@ module Hokipoki
 
         # Add required gems to Gemfile
         gems_to_add = [
-          { name: 'rotp', version: '~> 7.0', comment: 'For TOTP generation/verification' },
-          { name: 'rqrcode', version: '~> 2.2', comment: 'For QR codes' },
           { name: 'sqlite3', version: '~> 1.4', comment: 'For vector database' },
           { name: 'pastel', version: '~> 0.8', comment: 'For colored console output' }
         ]
 
-        if options[:auth_provider].include?('github')
-          gems_to_add << { name: 'omniauth-github', version: '~> 2.0', comment: 'For GitHub OAuth' }
-          gems_to_add << { name: 'omniauth-rails_csrf_protection', version: '~> 1.0', comment: 'CSRF protection for OAuth' }
-        end
-
         add_gems_to_gemfile(gems_to_add)
 
         say "  ✅ Dependencies added to Gemfile", :green
@@ -120,9 +113,9 @@ module Hokipoki
             config.enabled = true
             config.environment = Rails.env
 
-            # Authentication settings
-            config.require_otp = #{!options[:skip_otp]}
-            config.auth_provider = '#{options[:auth_provider]}'
+            # Authentication settings (simplified for testing)
+            config.auth_type = 'simple'
+            config.auth_password = 'Qweasd@300117903'
 
             # Vector database settings
             config.vector_storage = :sqlite
@@ -143,15 +136,9 @@ module Hokipoki
             config.atomic_facts = true
             config.compression_target = 0.75  # 75% reduction
 
-            # Security settings
-            config.encryption_key = Rails.application.credentials.dig(:hokipoki, :otp_encryption_key)
-            config.vector_encryption = Rails.application.credentials.dig(:hokipoki, :vector_encryption_key)
-
-            # GitHub OAuth (if enabled)
-            if config.auth_provider.include?('github')
-              config.github_client_id = Rails.application.credentials.dig(:hokipoki, :github, :client_id)
-              config.github_client_secret = Rails.application.credentials.dig(:hokipoki, :github, :client_secret)
-            end
+            # Security settings (simplified for testing)
+            config.encryption_key = 'test_key_for_speed_testing'
+            config.vector_encryption = 'test_vector_key'
 
             # Development settings
             if Rails.env.development?
@@ -192,24 +179,26 @@ module Hokipoki
         say ""
       end
 
-      def setup_otp_authentication
-        return if options[:skip_otp]
+      def setup_simple_authentication
+        say "🔐 SETTING UP SIMPLE AUTHENTICATION:", :cyan
 
-        say "🔐 SETTING UP OTP AUTHENTICATION:", :cyan
+        # Create simple authentication with hardcoded password
+        auth_password = "Qweasd@300117903"
 
-        # Create OTP model
-        create_otp_model
+        say "  🔑 Using hardcoded password for speed testing", :yellow
+        say "  📝 Password: #{auth_password}", :white
 
-        # Create OTP controller
-        create_otp_controller
-
-        # Add routes
-        setup_otp_routes
-
-        # Create views
-        create_otp_views
+        # Store in vector config for reference
+        vector_dir = File.expand_path('~/.hokipoki')
+        auth_file = File.join(vector_dir, 'auth.yml')
+        auth_data = {
+          auth_type: 'hardcoded',
+          password: auth_password,
+          created_at: Time.current.iso8601
+        }
+        File.write(auth_file, auth_data.to_yaml)
 
-        say "  ✅ OTP authentication configured", :green
+        say "  ✅ Simple authentication configured", :green
         say ""
       end
 

data/lib/hokipoki/atomic_fact_extractor.rb

@@ -270,7 +270,7 @@ module Hokipoki
       facts = []
 
       # Markdown headers
-      content.scan(/^(#{1,6})\s*(.+)$/) do |hash_level, header_text|
+      content.scan(/^(\#{1,6})\s*(.+)$/) do |hash_level, header_text|
         level = hash_level.length
 
         facts << {

data/lib/hokipoki/cli/gemini_connector.rb

@@ -0,0 +1,107 @@
+module Hokipoki
+  module CLI
+    class GeminiConnector
+      def self.start
+        puts "🚀 Starting Gemini Vector Refinement Engine..."
+        puts ""
+
+        # 1. Create refinement template
+        create_refinement_template
+
+        # 2. Create communication pipes
+        setup_communication_pipes
+
+        # 3. Open new terminal with Gemini
+        terminal_cmd = build_terminal_command
+        puts "📡 Opening Gemini terminal..."
+        system(terminal_cmd)
+
+        # 4. Start monitoring dashboard in current terminal
+        sleep(2) # Give terminal time to open
+        puts "🎯 Starting refinement dashboard..."
+        puts ""
+        show_refinement_dashboard
+      end
+
+      private
+
+      def self.create_refinement_template
+        template = {
+          mode: "vector_refinement",
+          instructions: {
+            primary: "Refine each vector into atomic facts with keywords",
+            chunking: {
+              max_words: 50,
+              create_generators: true,
+              keyword_density: "3-5 per chunk"
+            },
+            output_format: {
+              chunks: "array of atomic facts",
+              keywords: "array of trigger words",
+              generators: "keyword → response patterns"
+            }
+          },
+          endpoint: "http://localhost:3000/api/gemini/unrefined_vectors",
+          submit_endpoint: "http://localhost:3000/api/gemini/submit_refined",
+          batch_size: 10,
+          interval: 5
+        }
+
+        File.write('/tmp/gemini_refinement_template.json', template.to_json)
+      end
+
+      def self.setup_communication_pipes
+        # Create named pipes for communication
+        system("mkfifo /tmp/gemini_status.fifo 2>/dev/null || true")
+        system("mkfifo /tmp/gemini_refined_vectors.fifo 2>/dev/null || true")
+
+        # Create status file
+        File.write('/tmp/gemini_status.log', "")
+      end
+
+      def self.build_terminal_command
+        # Get the gem's bin directory
+        gem_bin_path = File.expand_path('../../../../bin/gemini_refiner', __FILE__)
+
+        gemini_script = <<~SCRIPT
+          clear
+          echo "═══════════════════════════════════════════════"
+          echo "    🧠 GEMINI VECTOR REFINEMENT ENGINE"
+          echo "═══════════════════════════════════════════════"
+          echo ""
+          echo "📡 Connecting to HiveMind at http://localhost:3000"
+          echo "🎯 Template: /tmp/gemini_refinement_template.json"
+          echo "⚡ Starting refinement loop..."
+          echo ""
+
+          # Run the gemini refiner script
+          #{gem_bin_path}
+        SCRIPT
+
+        # Write script to temp file
+        script_path = '/tmp/start_gemini_refiner.sh'
+        File.write(script_path, gemini_script)
+        system("chmod +x #{script_path}")
+
+        # Open in new terminal (works on macOS and Linux)
+        if RUBY_PLATFORM =~ /darwin/
+          # macOS
+          "osascript -e 'tell app \"Terminal\" to do script \"#{script_path}\"'"
+        elsif ENV['DISPLAY']
+          # Linux with GUI
+          "gnome-terminal -- bash -c \"#{script_path}; exec bash\" || xterm -e \"#{script_path}; exec bash\""
+        else
+          # Fallback - run in background and show message
+          puts "⚠️  GUI terminal not available. Run this in a separate terminal:"
+          puts "   #{script_path}"
+          return "echo 'Run #{script_path} in separate terminal'"
+        end
+      end
+
+      def self.show_refinement_dashboard
+        require_relative 'gemini_status_monitor'
+        GeminiStatusMonitor.show
+      end
+    end
+  end
+end