htm 0.0.18 → 0.0.30

data/lib/htm/observability.rb

@@ -58,7 +58,7 @@ class HTM
           query_timings: query_timing_stats,
           service_timings: service_timing_stats,
           memory_usage: memory_stats,
-          collected_at: Time.current
+          collected_at: Time.now
         }
       end
 
@@ -74,38 +74,42 @@ class HTM
       #   - :wait_timeout - Connection wait timeout (ms)
       #
       def connection_pool_stats
-        return { status: :unavailable, message: "ActiveRecord not connected" } unless connected?
-
-        pool = ActiveRecord::Base.connection_pool
-
-        size = pool.size
-        connections = pool.connections.size
-        in_use = pool.connections.count(&:in_use?)
-        available = connections - in_use
-
-        # Calculate utilization based on connections in use vs pool size
-        utilization = size > 0 ? in_use.to_f / size : 0.0
-
-        # Determine health status
-        status = case
-        when available == 0 && in_use >= size
-          :exhausted
-        when utilization >= POOL_CRITICAL_THRESHOLD
-          :critical
-        when utilization >= POOL_WARNING_THRESHOLD
-          :warning
+        return { status: :unavailable, message: "Database not connected" } unless connected?
+
+        pool = HTM.db.pool
+
+        # Sequel's TimedQueueConnectionPool API:
+        # - max_size: maximum pool size
+        # - size: number of connections currently in pool (pre-allocated up to max_size)
+        # - num_waiting: threads waiting for connections
+        #
+        # Unlike ActiveRecord, Sequel pre-allocates connections. The key health indicator
+        # is num_waiting - if threads are waiting, the pool is under stress.
+        max_size = pool.max_size
+        current_size = pool.size
+        waiting = pool.num_waiting
+
+        # For Sequel's TimedQueueConnectionPool:
+        # - Pool is healthy if no threads are waiting
+        # - Pool is critical only if threads are waiting for connections
+        # - size == max_size is normal (pre-allocated pool), not a problem
+        status = if waiting > 0
+          waiting > max_size / 2 ? :exhausted : :critical
         else
           :healthy
         end
 
+        # Utilization based on waiting threads (pool stress indicator)
+        utilization = waiting > 0 ? ((waiting.to_f / max_size) * 100).round(2) : 0.0
+
         stats = {
-          size: size,
-          connections: connections,
-          in_use: in_use,
-          available: available,
-          utilization: (utilization * 100).round(2),
-          status: status,
-          wait_timeout: pool.checkout_timeout * 1000 # Convert to ms
+          size: max_size,
+          connections: current_size,
+          in_use: 0,  # Sequel doesn't expose checked-out count; use waiting as stress indicator
+          available: max_size,  # All connections are available when not waiting
+          waiting: waiting,
+          utilization: utilization,
+          status: status
         }
 
         # Log warnings if pool is stressed
@@ -170,7 +174,7 @@ class HTM
           @query_timings << {
             duration_ms: duration_ms,
             query_type: query_type,
-            recorded_at: Time.current
+            recorded_at: Time.now
           }
 
           # Keep only recent samples
@@ -186,7 +190,7 @@ class HTM
         @metrics_mutex.synchronize do
           @embedding_timings << {
             duration_ms: duration_ms,
-            recorded_at: Time.current
+            recorded_at: Time.now
           }
           @embedding_timings.shift if @embedding_timings.size > @max_timing_samples
         end
@@ -200,7 +204,7 @@ class HTM
         @metrics_mutex.synchronize do
           @tag_extraction_timings << {
             duration_ms: duration_ms,
-            recorded_at: Time.current
+            recorded_at: Time.now
           }
           @tag_extraction_timings.shift if @tag_extraction_timings.size > @max_timing_samples
         end
@@ -287,7 +291,7 @@ class HTM
           healthy: issues.empty?,
           checks: checks,
           issues: issues,
-          checked_at: Time.current
+          checked_at: Time.now
         }
       end
 
@@ -313,22 +317,22 @@ class HTM
 
       private
 
-      # Check if ActiveRecord is connected
+      # Check if Sequel database is connected
       def connected?
-        return false unless defined?(ActiveRecord::Base)
-        ActiveRecord::Base.connected? && ActiveRecord::Base.connection.active?
+        return false unless defined?(HTM) && HTM.respond_to?(:db)
+        db = HTM.db
+        return false unless db
+        db.test_connection
       rescue StandardError
         false
       end
 
       # Check if a PostgreSQL extension is installed
       def extension_installed?(name)
-        result = ActiveRecord::Base.connection.select_value(
-          ActiveRecord::Base.sanitize_sql_array(
-            ["SELECT COUNT(*) FROM pg_extension WHERE extname = ?", name]
-          )
-        )
-        result.to_i > 0
+        result = HTM.db.fetch(
+          "SELECT COUNT(*) AS cnt FROM pg_extension WHERE extname = ?", name
+        ).first
+        result[:cnt].to_i > 0
       end
 
       # Calculate timing statistics from samples

data/lib/htm/proposition_service.rb

@@ -26,8 +26,20 @@ class HTM
   #   #     "The Apollo 11 mission occurred in 1969."]
   #
   class PropositionService
-    MIN_PROPOSITION_LENGTH = 10   # Minimum characters for a valid proposition
-    MAX_PROPOSITION_LENGTH = 1000 # Maximum characters for a valid proposition
+    # Patterns that indicate meta-responses (LLM asking for input instead of extracting)
+    META_RESPONSE_PATTERNS = [
+      /please provide/i,
+      /provide the text/i,
+      /provide me with/i,
+      /I need the text/i,
+      /I am ready/i,
+      /waiting for/i,
+      /send me the/i,
+      /what text would you/i,
+      /what would you like/i,
+      /cannot extract.*without/i,
+      /no text provided/i
+    ].freeze
 
     # Circuit breaker for proposition extraction API calls
     @circuit_breaker = nil
@@ -112,6 +124,45 @@ class HTM
       end
     end
 
+    # Get minimum character length from config
+    #
+    # @return [Integer] Minimum character count for valid propositions
+    #
+    def self.min_length
+      HTM.config.proposition.min_length || 10
+    rescue
+      10
+    end
+
+    # Get maximum character length from config
+    #
+    # @return [Integer] Maximum character count for valid propositions
+    #
+    def self.max_length
+      HTM.config.proposition.max_length || 1000
+    rescue
+      1000
+    end
+
+    # Get minimum words from config
+    #
+    # @return [Integer] Minimum word count for valid propositions
+    #
+    def self.min_words
+      HTM.config.proposition.min_words || 5
+    rescue
+      5
+    end
+
+    # Check if proposition is a meta-response (LLM asking for input)
+    #
+    # @param proposition [String] Proposition to check
+    # @return [Boolean] True if it's a meta-response
+    #
+    def self.meta_response?(proposition)
+      META_RESPONSE_PATTERNS.any? { |pattern| proposition.match?(pattern) }
+    end
+
     # Validate and filter propositions
     #
     # @param propositions [Array<String>] Parsed propositions
@@ -119,13 +170,16 @@ class HTM
     #
     def self.validate_and_filter_propositions(propositions)
       valid_propositions = []
+      min_char_length = min_length
+      max_char_length = max_length
+      min_word_count = min_words
 
       propositions.each do |proposition|
-        # Check minimum length
-        next if proposition.length < MIN_PROPOSITION_LENGTH
+        # Check minimum length (characters)
+        next if proposition.length < min_char_length
 
         # Check maximum length
-        if proposition.length > MAX_PROPOSITION_LENGTH
+        if proposition.length > max_char_length
           HTM.logger.warn "PropositionService: Proposition too long, skipping: #{proposition[0..50]}..."
           next
         end
@@ -135,6 +189,19 @@ class HTM
           next
         end
 
+        # Check minimum word count
+        word_count = proposition.split.size
+        if word_count < min_word_count
+          HTM.logger.debug "PropositionService: Proposition too short (#{word_count} words), skipping: #{proposition}"
+          next
+        end
+
+        # Filter out meta-responses (LLM asking for more input)
+        if meta_response?(proposition)
+          HTM.logger.warn "PropositionService: Filtered meta-response: #{proposition[0..50]}..."
+          next
+        end
+
         # Proposition is valid
         valid_propositions << proposition
       end
@@ -149,9 +216,11 @@ class HTM
     #
     def self.valid_proposition?(proposition)
       return false unless proposition.is_a?(String)
-      return false if proposition.length < MIN_PROPOSITION_LENGTH
-      return false if proposition.length > MAX_PROPOSITION_LENGTH
+      return false if proposition.length < min_length
+      return false if proposition.length > max_length
       return false unless proposition.match?(/[a-zA-Z]{3,}/)
+      return false if proposition.split.size < min_words
+      return false if meta_response?(proposition)
 
       true
     end

data/lib/htm/railtie.rb

@@ -57,8 +57,8 @@ class HTM
     config.after_initialize do
       if Rails.env.development?
         begin
-          HTM::ActiveRecordConfig.establish_connection! unless HTM::ActiveRecordConfig.connected?
-          HTM::ActiveRecordConfig.verify_extensions!
+          HTM::SequelConfig.establish_connection! unless HTM::SequelConfig.db
+          HTM::SequelConfig.verify_extensions!
           HTM.logger.info "HTM database connection verified"
         rescue StandardError => e
           HTM.logger.warn "HTM database connection check failed: #{e.message}"

data/lib/htm/robot_group.rb

@@ -180,7 +180,7 @@ class HTM
       # Clear working memory flags for this robot
       HTM::Models::RobotNode
         .where(robot_id: htm.robot_id, working_memory: true)
-        .update_all(working_memory: false)
+        .update(working_memory: false)
     end
 
     # Promotes a passive robot to active status.
@@ -368,7 +368,7 @@ class HTM
     # Queries the database for the union of all members' working memory,
     # returning nodes sorted by creation date (newest first).
     #
-    # @return [ActiveRecord::Relation<HTM::Models::Node>] Collection of nodes
+    # @return [Sequel::Dataset] Collection of nodes
     #
     # @example
     #   nodes = group.working_memory_contents
@@ -378,9 +378,9 @@ class HTM
       node_ids = HTM::Models::RobotNode
                  .where(robot_id: member_ids, working_memory: true)
                  .distinct
-                 .pluck(:node_id)
+                 .select_map(:node_id)
 
-      HTM::Models::Node.where(id: node_ids).order(created_at: :desc)
+      HTM::Models::Node.where(id: node_ids).order(Sequel.desc(:created_at))
     end
 
     # Clears shared working memory for all group members.
@@ -397,7 +397,7 @@ class HTM
     def clear_working_memory
       count = HTM::Models::RobotNode
               .where(robot_id: member_ids, working_memory: true)
-              .update_all(working_memory: false)
+              .update(working_memory: false)
 
       # Clear in-memory working memory for primary robot
       primary = @active_robots.values.first || @passive_robots.values.first
@@ -435,21 +435,25 @@ class HTM
       # Get all node_ids currently in any member's working memory
       shared_node_ids = HTM::Models::RobotNode
                         .where(robot_id: member_ids, working_memory: true)
-                        .where.not(robot_id: htm.robot_id)
+                        .exclude(robot_id: htm.robot_id)
                         .distinct
-                        .pluck(:node_id)
+                        .select_map(:node_id)
 
       synced = 0
       shared_node_ids.each do |node_id|
         # Create or update robot_node with working_memory=true
-        robot_node = HTM::Models::RobotNode.find_or_initialize_by(
+        robot_node = HTM::Models::RobotNode.first(
           robot_id: htm.robot_id,
           node_id: node_id
         )
-        next if robot_node.working_memory?
+        robot_node ||= HTM::Models::RobotNode.new(
+          robot_id: htm.robot_id,
+          node_id: node_id
+        )
+        next if robot_node.working_memory
 
         robot_node.working_memory = true
-        robot_node.save!
+        robot_node.save
         synced += 1
       end
 
@@ -501,7 +505,7 @@ class HTM
       working_memories = member_ids.map do |robot_id|
         HTM::Models::RobotNode
           .where(robot_id: robot_id, working_memory: true)
-          .pluck(:node_id)
+          .select_map(:node_id)
           .sort
       end
 
@@ -543,16 +547,20 @@ class HTM
       # Get source's working memory nodes
       source_node_ids = HTM::Models::RobotNode
                         .where(robot_id: from_htm.robot_id, working_memory: true)
-                        .pluck(:node_id)
+                        .select_map(:node_id)
 
       transferred = 0
       source_node_ids.each do |node_id|
-        robot_node = HTM::Models::RobotNode.find_or_initialize_by(
+        robot_node = HTM::Models::RobotNode.first(
+          robot_id: to_htm.robot_id,
+          node_id: node_id
+        )
+        robot_node ||= HTM::Models::RobotNode.new(
           robot_id: to_htm.robot_id,
           node_id: node_id
         )
         robot_node.working_memory = true
-        robot_node.save!
+        robot_node.save
         transferred += 1
       end
 
@@ -560,7 +568,7 @@ class HTM
       if clear_source
         HTM::Models::RobotNode
           .where(robot_id: from_htm.robot_id, working_memory: true)
-          .update_all(working_memory: false)
+          .update(working_memory: false)
       end
 
       transferred
@@ -674,7 +682,7 @@ class HTM
     end
 
     def sync_node_to_in_memory_caches(node_id, origin_robot_id)
-      node = HTM::Models::Node.find_by(id: node_id)
+      node = HTM::Models::Node.first(id: node_id)
       return unless node
 
       all_robots.each do |_name, htm|
@@ -709,12 +717,16 @@ class HTM
       member_ids.each do |robot_id|
         next if robot_id == exclude
 
-        robot_node = HTM::Models::RobotNode.find_or_initialize_by(
+        robot_node = HTM::Models::RobotNode.first(
+          robot_id: robot_id,
+          node_id: node_id
+        )
+        robot_node ||= HTM::Models::RobotNode.new(
           robot_id: robot_id,
           node_id: node_id
         )
         robot_node.working_memory = true
-        robot_node.save!
+        robot_node.save
       end
     end
   end

data/lib/htm/sequel_config.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'sequel'
+
+class HTM
+  # Sequel database configuration and model loading
+  #
+  # Uses HTM::Config for database settings. Configuration can come from:
+  # - Environment variables (HTM_DATABASE__URL, HTM_DATABASE__HOST, etc.)
+  # - Programmatic configuration via HTM.configure
+  #
+  # Sequel is fiber-safe by design, making it ideal for async/fiber-based
+  # concurrency patterns used in HTM's job processing.
+  #
+  class SequelConfig
+    class << self
+      # Database connection instance
+      # @return [Sequel::Database, nil]
+      attr_reader :db
+
+      # Establish database connection from HTM::Config
+      #
+      # @param load_models [Boolean] Whether to load models after connection (default: true)
+      #   Set to false when running migrations on a fresh database
+      # @return [Sequel::Database] The database connection
+      #
+      def establish_connection!(load_models: true)
+        return @db if @db
+
+        config = load_database_config
+        connection_string = build_connection_string(config)
+
+        # Configure Sequel with fiber-safe settings
+        @db = Sequel.connect(connection_string, {
+          max_connections: config[:pool] || 5,
+          pool_timeout: (config[:checkout_timeout] || 5).to_i,
+          # Fiber-safe settings - important for async gem compatibility
+          preconnect: :concurrently,
+          # Use threaded mode which works well with fibers
+          single_threaded: false
+        })
+
+        # Load PostgreSQL-specific extensions for JSONB and array handling
+        @db.extension :pg_json
+        @db.extension :pg_array
+
+        # Set search path and statement timeout
+        @db.run("SET search_path TO public")
+        @db.run("SET statement_timeout = #{config[:statement_timeout] || 30_000}")
+
+        # Load models after connection is established (unless disabled for migrations)
+        require_models if load_models && models_loadable?
+
+        @db
+      end
+
+      # Check if models can be loaded (tables exist)
+      #
+      # @return [Boolean]
+      #
+      def models_loadable?
+        return false unless @db
+        @db.table_exists?(:robots)
+      rescue Sequel::DatabaseError
+        false
+      end
+
+      # Ensure models are loaded
+      #
+      # Call this after migrations to ensure models are available
+      # @return [void]
+      #
+      def ensure_models_loaded!
+        require_models unless @models_loaded
+      end
+
+      # Load database configuration from HTM::Config
+      #
+      # @return [Hash] Database configuration hash
+      #
+      def load_database_config
+        HTM.config.database_config
+      end
+
+      # Build connection string from config hash
+      #
+      # @param config [Hash] Database configuration
+      # @return [String] PostgreSQL connection string
+      #
+      def build_connection_string(config)
+        # If we have a URL already, use it
+        if config[:url]
+          return config[:url]
+        end
+
+        user = config[:username] || config[:user]
+        password = config[:password]
+        host = config[:host] || 'localhost'
+        port = config[:port] || 5432
+        database = config[:database]
+
+        if password && !password.empty?
+          "postgres://#{user}:#{password}@#{host}:#{port}/#{database}"
+        elsif user
+          "postgres://#{user}@#{host}:#{port}/#{database}"
+        else
+          "postgres://#{host}:#{port}/#{database}"
+        end
+      end
+
+      # Check if connection is established and active
+      #
+      # @return [Boolean]
+      #
+      def connected?
+        return false unless @db
+
+        @db.test_connection
+      rescue Sequel::DatabaseError
+        false
+      end
+
+      # Close all database connections
+      #
+      # @return [void]
+      #
+      def disconnect!
+        @db&.disconnect
+        @db = nil
+      end
+
+      # Verify required extensions are available
+      #
+      # @raise [RuntimeError] if required extensions are missing
+      # @return [true]
+      #
+      def verify_extensions!
+        required_extensions = {
+          'vector' => 'pgvector extension',
+          'pg_trgm' => 'PostgreSQL trigram extension'
+        }
+
+        missing = []
+        required_extensions.each do |ext, name|
+          result = @db["SELECT COUNT(*) AS cnt FROM pg_extension WHERE extname = ?", ext].first
+          missing << name if result[:cnt].to_i.zero?
+        end
+
+        if missing.any?
+          raise "Missing required PostgreSQL extensions: #{missing.join(', ')}"
+        end
+
+        true
+      end
+
+      # Get connection pool statistics
+      #
+      # @return [Hash] Pool statistics
+      #
+      def connection_stats
+        pool = @db.pool
+        {
+          size: pool.max_size,
+          available: pool.available_connections.size,
+          allocated: pool.allocated.size
+        }
+      rescue NoMethodError
+        # Fallback for connection pools that don't support these methods
+        { size: @db.pool.max_size }
+      end
+
+      # Run raw SQL
+      #
+      # @param sql [String] SQL to execute
+      # @return [void]
+      #
+      def execute(sql)
+        @db.run(sql)
+      end
+
+      # Select a single value
+      #
+      # @param sql [String] SQL query
+      # @return [Object] The value
+      #
+      def select_value(sql)
+        @db[sql].first&.values&.first
+      end
+
+      private
+
+      # Require all model files
+      def require_models
+        return if @models_loaded
+
+        require_relative 'models/robot'
+        require_relative 'models/node'
+        require_relative 'models/robot_node'
+        require_relative 'models/tag'
+        require_relative 'models/node_tag'
+        require_relative 'models/file_source'
+
+        @models_loaded = true
+      end
+    end
+  end
+
+  # Convenience method to access the database connection
+  #
+  # @return [Sequel::Database]
+  #
+  def self.db
+    SequelConfig.db || SequelConfig.establish_connection!
+  end
+end