hokipoki 0.8.5 → 0.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e6282c9c1b0a50ca6c78f7c19ef2268ab213738fb539ae497c10a9bb6ea14a1
-  data.tar.gz: ade8ac41a2d503c2a46bfd68e418ff4b233e4d3cd00a31a19f2311c18e102c6b
+  metadata.gz: b5a7fa3a37f4a74900d6a2c6e901ac496e53bc5f393d6af9623e00d7fbdbf2d7
+  data.tar.gz: 0c8a79d2d72f2f096d7328b924e68d77c7d4181449f34db9544ded4c388a0fcb
 SHA512:
-  metadata.gz: 81ed67ae4dcff72c7d43d169785bbc81320a6bc19c9cd89bb05a74ab95239bb0db521e46b92c7f0a39a4f88ce4f1ef4d4a845a7d76944c1f2b8bcaeae50da46a
-  data.tar.gz: 9f66a3118b48c98f06e6242fc116f4d7e633c4ca2ac5f0e2cd5a672346873415f04ee2891d4bc1fb38c1ec90a2768d8c3704e48637095273ac9659fefe1c94a7
+  metadata.gz: af5e8f308a673ce3160b5ce0adb627edcebf8c04e7e6574f2cddd7ed70b2010829fdbecec8194865e87a612671aa8b58c406465d2a85711c97fa14e6fba265aa
+  data.tar.gz: 39ca391c5e1f001eef3d2dd5052402fe3493ed31c31c4ed8d4d146db9a85b0f0530c0d55a712fa68bcfcfc80a7ec52d40c956149231c337ece1e8bd874d356b4

data/bin/gemini_refiner

@@ -15,7 +15,7 @@ class GeminiRefiner
     @status_file = '/tmp/gemini_status.log'
     @current_refinement_file = '/tmp/gemini_current_refinement.json'
     @batch_size = 10
-    @sleep_interval = 5
+    @sleep_interval = 120  # 2 minutes
     @refinement_count = 0
     @start_time = Time.now
 
@@ -67,7 +67,7 @@ class GeminiRefiner
     puts "🎯 Configuration:"
     puts "   Endpoint:     #{@base_url}/api/gemini"
     puts "   Batch Size:   #{@batch_size} vectors"
-    puts "   Interval:     #{@sleep_interval} seconds"
+    puts "   Interval:     #{@sleep_interval / 60} minutes"
     puts "   Template:     #{@template_file}"
     puts ""
   end
@@ -129,7 +129,7 @@ class GeminiRefiner
             display_current_stats
           end
         else
-          puts "💤 No vectors to refine... waiting #{@sleep_interval}s"
+          puts "💤 No vectors to refine... waiting #{@sleep_interval / 60} minutes"
           clear_current_refinement
         end
 
@@ -138,7 +138,7 @@ class GeminiRefiner
       rescue => e
         log_status('error', "Refinement loop error: #{e.message}")
         puts "❌ Error in refinement loop: #{e.message}"
-        puts "   Retrying in #{@sleep_interval} seconds..."
+        puts "   Retrying in #{@sleep_interval / 60} minutes..."
         sleep(@sleep_interval)
       end
     end

data/lib/generators/hive_mind/install_generator.rb

@@ -129,11 +129,31 @@ module HiveMind
       # Create smart retrieval engine
       template 'smart_retrieval_engine.rb', 'app/services/smart_retrieval_engine.rb'
       say @pastel.green("✓ Created app/services/smart_retrieval_engine.rb")
+
+      # Create Gemini API controller for vector refinement
+      directory 'api', 'app/controllers/api'
+      say @pastel.green("✓ Created app/controllers/api/gemini_controller.rb")
     end
 
     def setup_routing
-      say "\n#{@pastel.cyan('🛣️  Skipping routes for lightweight install')}"
-      say @pastel.green("✓ No additional routes needed")
+      say "\n#{@pastel.cyan('🛣️  Adding Gemini API routes')}"
+
+      # Add Gemini API routes
+      routes_content = <<~RUBY
+
+  # Gemini vector refinement API
+  namespace :api do
+    namespace :gemini do
+      get :health_check
+      get :unrefined_vectors
+      post :submit_refined
+      get :stats
+    end
+  end
+      RUBY
+
+      append_to_file 'config/routes.rb', routes_content
+      say @pastel.green("✓ Added Gemini API routes")
     end
 
     def create_cli_tools

data/lib/generators/hive_mind/templates/api/gemini_controller.rb

@@ -0,0 +1,283 @@
+class Api::GeminiController < ApplicationController
+  skip_before_action :verify_authenticity_token
+  before_action :log_gemini_activity, except: [:health_check]
+
+  # Health check endpoint
+  def health_check
+    render json: {
+      status: 'ok',
+      timestamp: Time.current,
+      system: 'hokipoki_gemini_api'
+    }
+  end
+
+  # Endpoint for Gemini to fetch unrefined vectors
+  def unrefined_vectors
+    begin
+      batch_size = params[:batch_size]&.to_i || 10
+      batch_size = [batch_size, 50].min # Cap at 50 for safety
+
+      vectors = HiveMindDocument.where(refined: false)
+                                .limit(batch_size)
+                                .order(:created_at)
+
+      response_data = {
+        vectors: vectors.map { |v| format_vector_for_gemini(v) },
+        refinement_template: refinement_template,
+        batch_info: {
+          requested: batch_size,
+          returned: vectors.count,
+          total_pending: HiveMindDocument.where(refined: false).count
+        },
+        timestamp: Time.current
+      }
+
+      log_gemini_status('info', "Served #{vectors.count} unrefined vectors to Gemini")
+
+      render json: response_data
+
+    rescue => e
+      log_gemini_status('error', "Failed to fetch unrefined vectors: #{e.message}")
+
+      render json: {
+        error: 'Failed to fetch vectors',
+        message: e.message,
+        timestamp: Time.current
+      }, status: 500
+    end
+  end
+
+  # Endpoint for Gemini to submit refined vectors
+  def submit_refined
+    begin
+      refined_data = JSON.parse(request.body.read)
+      processed_count = 0
+      errors = []
+
+      refined_data['vectors']&.each do |refined|
+        begin
+          vector = HiveMindDocument.find(refined['id'])
+
+          # Update with refined data
+          update_data = {
+            refined: true,
+            refined_at: Time.current,
+            refined_by: 'gemini'
+          }
+
+          # Add refined content if provided
+          if refined['chunks']
+            update_data[:chunks] = refined['chunks']
+          end
+
+          if refined['keywords']
+            update_data[:keywords] = refined['keywords']
+          end
+
+          if refined['generators']
+            update_data[:generators] = refined['generators']
+          end
+
+          # Store refined metadata
+          existing_metadata = vector.metadata || {}
+          update_data[:metadata] = existing_metadata.merge({
+            'refined_by' => 'gemini',
+            'refined_at' => Time.current.iso8601,
+            'original_content_length' => vector.content&.length,
+            'chunks_count' => refined['chunks']&.length,
+            'keywords_count' => refined['keywords']&.length
+          })
+
+          vector.update!(update_data)
+          processed_count += 1
+
+          log_gemini_status('success', "Refined vector ##{vector.id} (#{refined['keywords']&.length || 0} keywords)")
+
+        rescue => e
+          error_msg = "Failed to update vector #{refined['id']}: #{e.message}"
+          errors << error_msg
+          log_gemini_status('error', error_msg)
+        end
+      end
+
+      response_data = {
+        status: 'success',
+        processed: processed_count,
+        errors: errors,
+        timestamp: Time.current
+      }
+
+      if errors.any?
+        response_data[:status] = 'partial_success'
+        render json: response_data, status: 207 # Multi-status
+      else
+        render json: response_data
+      end
+
+    rescue JSON::ParserError => e
+      log_gemini_status('error', "Invalid JSON in refined data: #{e.message}")
+
+      render json: {
+        error: 'Invalid JSON format',
+        message: e.message,
+        timestamp: Time.current
+      }, status: 400
+
+    rescue => e
+      log_gemini_status('error', "Failed to process refined vectors: #{e.message}")
+
+      render json: {
+        error: 'Failed to process refined vectors',
+        message: e.message,
+        timestamp: Time.current
+      }, status: 500
+    end
+  end
+
+  # Stats endpoint for monitoring
+  def stats
+    begin
+      stats = {
+        total_vectors: HiveMindDocument.count,
+        refined_vectors: HiveMindDocument.where(refined: true).count,
+        pending_vectors: HiveMindDocument.where(refined: false).count,
+        refinement_rate: calculate_refinement_rate,
+        last_activity: get_last_activity,
+        timestamp: Time.current
+      }
+
+      render json: stats
+
+    rescue => e
+      render json: {
+        error: 'Failed to generate stats',
+        message: e.message,
+        timestamp: Time.current
+      }, status: 500
+    end
+  end
+
+  private
+
+  def format_vector_for_gemini(vector)
+    {
+      id: vector.id,
+      content: vector.content,
+      source_type: vector.source_type,
+      source_id: vector.source_id,
+      metadata: vector.metadata || {},
+      created_at: vector.created_at,
+      content_length: vector.content&.length || 0,
+      preview: vector.content&.truncate(100)
+    }
+  end
+
+  def refinement_template
+    {
+      instruction: "Refine each vector into atomic facts with keywords and generators",
+      chunking_rules: {
+        max_words_per_chunk: 50,
+        min_words_per_chunk: 10,
+        overlap_words: 5,
+        preserve_context: true
+      },
+      keyword_extraction: {
+        keywords_per_chunk: "3-5",
+        types: ["technical", "contextual", "semantic"],
+        importance_threshold: 0.6
+      },
+      generator_creation: {
+        pattern: "trigger_keywords → response_template",
+        confidence_threshold: 0.7,
+        max_generators_per_chunk: 3
+      },
+      output_format: {
+        chunks: "Array of atomic fact strings",
+        keywords: "Array of keyword strings",
+        generators: "Array of {trigger: [...], template: '...', confidence: 0.0-1.0}"
+      },
+      example: {
+        input: "Rails authentication with Devise requires configuration",
+        expected_output: {
+          chunks: ["Rails authentication with Devise requires configuration"],
+          keywords: ["rails", "authentication", "devise", "configuration"],
+          generators: [
+            {
+              trigger: ["rails", "authentication", "devise"],
+              template: "For Rails authentication, use Devise gem with proper configuration",
+              confidence: 0.85
+            }
+          ]
+        }
+      }
+    }
+  end
+
+  def calculate_refinement_rate
+    # Calculate vectors refined in last hour
+    one_hour_ago = 1.hour.ago
+    recent_refined = HiveMindDocument.where(refined: true)
+                                    .where('refined_at > ?', one_hour_ago)
+                                    .count
+
+    # Return per minute rate
+    (recent_refined / 60.0).round(2)
+  end
+
+  def get_last_activity
+    begin
+      last_refined = HiveMindDocument.where(refined: true)
+                                    .order(:refined_at)
+                                    .last
+
+      if last_refined&.refined_at
+        {
+          type: 'refinement',
+          timestamp: last_refined.refined_at,
+          vector_id: last_refined.id
+        }
+      else
+        {
+          type: 'none',
+          timestamp: nil,
+          vector_id: nil
+        }
+      end
+    rescue
+      { type: 'error', timestamp: nil, vector_id: nil }
+    end
+  end
+
+  def log_gemini_activity
+    activity_log = "#{Time.current.strftime('%H:%M:%S')} - #{action_name} - #{request.remote_ip}"
+
+    # Log to Gemini status file
+    File.open('/tmp/gemini_status.log', 'a') do |f|
+      f.puts "#{Time.current.strftime('%H:%M:%S')} - api - #{activity_log}"
+    end
+  rescue
+    # Ignore logging errors
+  end
+
+  def log_gemini_status(level, message)
+    timestamp = Time.current.strftime('%H:%M:%S')
+    log_entry = "#{timestamp} - #{level} - #{message}"
+
+    # Log to Rails logger
+    case level
+    when 'error'
+      Rails.logger.error "[GEMINI] #{message}"
+    when 'success'
+      Rails.logger.info "[GEMINI] #{message}"
+    else
+      Rails.logger.debug "[GEMINI] #{message}"
+    end
+
+    # Log to Gemini status file for dashboard
+    File.open('/tmp/gemini_status.log', 'a') do |f|
+      f.puts log_entry
+    end
+  rescue
+    # Ignore logging errors
+  end
+end

data/lib/hokipoki/cli/gemini_connector.rb

@@ -44,7 +44,7 @@ module Hokipoki
           endpoint: "http://localhost:3000/api/gemini/unrefined_vectors",
           submit_endpoint: "http://localhost:3000/api/gemini/submit_refined",
           batch_size: 10,
-          interval: 5
+          interval: 120
         }
 
         File.write('/tmp/gemini_refinement_template.json', template.to_json)

data/lib/hokipoki/version.rb

@@ -1,3 +1,3 @@
 module Hokipoki
-  VERSION = "0.8.5"
+  VERSION = "0.8.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hokipoki
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.7
 platform: ruby
 authors:
 - Rails Utilities
@@ -232,6 +232,7 @@ files:
 - bin/gemini_refiner
 - lib/generators/hive_mind/install_generator.rb
 - lib/generators/hive_mind/start_generator.rb
+- lib/generators/hive_mind/templates/api/gemini_controller.rb
 - lib/generators/hive_mind/templates/create_hive_mind_documents.rb
 - lib/generators/hive_mind/templates/embedding_service.rb
 - lib/generators/hive_mind/templates/hive_mind_document.rb