htm 0.0.1 → 0.0.10

data/lib/htm/embedding_service.rb

@@ -10,12 +10,43 @@ class HTM
   # - Dimension handling (padding/truncation)
   # - Error handling and logging
   # - Storage formatting
+  # - Circuit breaker protection for external LLM failures
   #
   # The actual LLM call is delegated to HTM.configuration.embedding_generator
   #
   class EmbeddingService
     MAX_DIMENSION = 2000  # Maximum dimension for pgvector HNSW index
 
+    # Circuit breaker for embedding API calls
+    @circuit_breaker = nil
+    @circuit_breaker_mutex = Mutex.new
+
+    class << self
+      # Get or create the circuit breaker for embedding service
+      #
+      # @return [HTM::CircuitBreaker] The circuit breaker instance
+      #
+      def circuit_breaker
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker ||= HTM::CircuitBreaker.new(
+            name: 'embedding_service',
+            failure_threshold: 5,
+            reset_timeout: 60
+          )
+        end
+      end
+
+      # Reset the circuit breaker (useful for testing)
+      #
+      # @return [void]
+      #
+      def reset_circuit_breaker!
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker&.reset!
+        end
+      end
+    end
+
     # Generate embedding with validation and processing
     #
     # @param text [String] Text to embed
@@ -26,12 +57,15 @@ class HTM
     #     storage_embedding: String,         # Formatted for database storage
     #     storage_dimension: Integer         # Padded dimension (2000)
     #   }
+    # @raise [CircuitBreakerOpenError] If circuit breaker is open
     #
     def self.generate(text)
       HTM.logger.debug "EmbeddingService: Generating embedding for #{text.length} chars"
 
-      # Call configured embedding generator
-      raw_embedding = HTM.configuration.embedding_generator.call(text)
+      # Use circuit breaker to protect against cascading failures
+      raw_embedding = circuit_breaker.call do
+        HTM.configuration.embedding_generator.call(text)
+      end
 
       # Validate response
       validate_embedding!(raw_embedding)
@@ -61,6 +95,9 @@ class HTM
         storage_dimension: MAX_DIMENSION
       }
 
+    rescue HTM::CircuitBreakerOpenError
+      # Re-raise circuit breaker errors without wrapping
+      raise
     rescue HTM::EmbeddingError
       raise
     rescue StandardError => e

data/lib/htm/errors.rb

@@ -1,34 +1,154 @@
 # frozen_string_literal: true
 
-# HTM error classes
+# HTM (Hierarchical Temporary Memory) error classes
+#
+# All HTM errors inherit from HTM::Error, allowing you to catch
+# all HTM-related errors with a single rescue clause.
+#
+# @example Catching all HTM errors
+#   begin
+#     htm.remember("some content")
+#   rescue HTM::Error => e
+#     logger.error "HTM error: #{e.message}"
+#   end
+#
+# @example Catching specific errors
+#   begin
+#     htm.forget(node_id, soft: false)
+#   rescue HTM::NotFoundError
+#     puts "Node not found"
+#   rescue HTM::ValidationError
+#     puts "Invalid input"
+#   end
+#
 class HTM
   # Base error class for all HTM errors
+  #
+  # All custom HTM errors inherit from this class, providing a common
+  # ancestor for error handling.
+  #
   class Error < StandardError; end
 
-  # Validation errors
+  # Raised when input validation fails
+  #
+  # Common causes:
+  # - Empty or nil content for remember()
+  # - Content exceeding maximum size limit
+  # - Invalid tag format
+  # - Invalid recall strategy
+  # - Invalid timeframe format
+  #
+  # @example
+  #   htm.remember("")  # => raises ValidationError
+  #   htm.remember("x", tags: ["INVALID!"])  # => raises ValidationError
+  #
   class ValidationError < Error; end
 
-  # Resource exhausted errors (memory, tokens, etc.)
+  # Raised when system resources are exhausted
+  #
+  # Common causes:
+  # - Working memory token limit exceeded
+  # - Database connection pool exhausted
+  # - Memory allocation failures
+  #
   class ResourceExhaustedError < Error; end
 
-  # Resource not found errors
+  # Raised when a requested resource cannot be found
+  #
+  # Common causes:
+  # - Node ID does not exist
+  # - Robot not registered
+  # - File source not found
+  #
+  # @example
+  #   htm.forget(999999)  # => raises NotFoundError if node doesn't exist
+  #
   class NotFoundError < Error; end
 
-  # Embedding service errors
+  # Raised when embedding generation fails
+  #
+  # Common causes:
+  # - LLM provider API errors
+  # - Invalid embedding response format
+  # - Network connectivity issues
+  # - Model not available
+  #
+  # Note: This error is distinct from CircuitBreakerOpenError.
+  # EmbeddingError indicates a single failure, while CircuitBreakerOpenError
+  # indicates repeated failures have triggered protective circuit breaking.
+  #
   class EmbeddingError < Error; end
 
-  # Tag service errors
+  # Raised when tag extraction fails
+  #
+  # Common causes:
+  # - LLM provider API errors
+  # - Invalid tag response format
+  # - Network connectivity issues
+  # - Model not available
+  #
+  # Note: This error is distinct from CircuitBreakerOpenError.
+  # TagError indicates a single failure, while CircuitBreakerOpenError
+  # indicates repeated failures have triggered protective circuit breaking.
+  #
   class TagError < Error; end
 
-  # Database operation errors
+  # Raised when database operations fail
+  #
+  # Common causes:
+  # - Connection failures
+  # - Query syntax errors
+  # - Constraint violations
+  # - Extension not installed (pgvector, pg_trgm)
+  #
   class DatabaseError < Error; end
 
-  # Query timeout errors
+  # Raised when a database query exceeds the configured timeout
+  #
+  # Default timeout is 30 seconds. Configure via db_query_timeout parameter
+  # when initializing HTM.
+  #
+  # @example Handling timeout
+  #   begin
+  #     htm.recall("complex query", strategy: :hybrid)
+  #   rescue HTM::QueryTimeoutError
+  #     # Retry with simpler query or smaller limit
+  #   end
+  #
   class QueryTimeoutError < DatabaseError; end
 
-  # Authorization errors
+  # Raised when an operation is not authorized
+  #
+  # Reserved for future multi-tenant scenarios where access control
+  # may restrict certain operations.
+  #
   class AuthorizationError < Error; end
 
-  # Circuit breaker errors
-  class CircuitBreakerOpenError < EmbeddingError; end
+  # Raised when circuit breaker is open due to repeated failures
+  #
+  # The circuit breaker pattern protects against cascading failures when
+  # external LLM services are unavailable. When too many consecutive
+  # failures occur, the circuit "opens" and subsequent calls fail fast
+  # without attempting the operation.
+  #
+  # Circuit states:
+  # - :closed - Normal operation, requests flow through
+  # - :open - Too many failures, requests fail immediately
+  # - :half_open - Testing if service recovered
+  #
+  # After a reset timeout (default: 60 seconds), the circuit transitions
+  # to half-open and tests if the service has recovered.
+  #
+  # @example Handling circuit breaker
+  #   begin
+  #     htm.remember("new content")
+  #   rescue HTM::CircuitBreakerOpenError
+  #     # LLM service unavailable, but node is still saved
+  #     # Embeddings/tags will be generated later when service recovers
+  #   end
+  #
+  # @see HTM::CircuitBreaker
+  # @see HTM::Observability.circuit_breaker_stats
+  #
+  class CircuitBreakerOpenError < Error; end
 end

data/lib/htm/{sinatra.rb → integrations/sinatra.rb}

@@ -1,5 +1,19 @@
 # frozen_string_literal: true
 
+# HTM Sinatra Integration
+#
+# Optional integration for using HTM with Sinatra web applications.
+# This file is NOT loaded automatically - require it explicitly:
+#
+#   require 'htm'
+#   require 'htm/integrations/sinatra'
+#
+# Provides:
+# - HTM::Sinatra::Helpers - Request helpers (init_htm, htm, remember, recall)
+# - HTM::Sinatra::Middleware - Connection pool management
+# - Sinatra::Base.register_htm - One-line setup
+#
+
 require 'sinatra/base'
 
 class HTM
@@ -8,20 +22,22 @@ class HTM
   # Provides convenient helper methods for using HTM in Sinatra applications.
   #
   # @example Basic usage
+  #   require 'htm/integrations/sinatra'
+  #
   #   class MyApp < Sinatra::Base
-  #     helpers HTM::Sinatra::Helpers
+  #     register_htm
   #
   #     before do
   #       init_htm(robot_name: session[:user_id] || 'guest')
   #     end
   #
   #     post '/remember' do
-  #       node_id = htm.remember(params[:content], source: 'user')
+  #       node_id = remember(params[:content])
   #       json status: 'ok', node_id: node_id
   #     end
   #
   #     get '/recall' do
-  #       memories = htm.recall(params[:topic], limit: 10)
+  #       memories = recall(params[:topic], limit: 10)
   #       json memories: memories
   #     end
   #   end
@@ -53,11 +69,11 @@ class HTM
       # Remember information (convenience method)
       #
       # @param content [String] Content to remember
-      # @param source [String] Source identifier (default: 'user')
+      # @param tags [Array<String>] Optional tags to assign
       # @return [Integer] Node ID
       #
-      def remember(content, source: 'user')
-        htm.remember(content, source: source)
+      def remember(content, tags: [])
+        htm.remember(content, tags: tags)
       end
 
       # Recall memories (convenience method)
@@ -91,16 +107,18 @@ class HTM
     #   end
     #
     class Middleware
+      # Class-level storage for connection configuration (shared across threads)
+      @@db_config = nil
+      @@config_mutex = Mutex.new
+
       def initialize(app, options = {})
         @app = app
         @options = options
       end
 
       def call(env)
-        # Establish connection if needed
-        unless HTM::ActiveRecordConfig.connected?
-          HTM::ActiveRecordConfig.establish_connection!
-        end
+        # Ensure connection is available in this thread
+        ensure_thread_connection!
 
         # Process request
         status, headers, body = @app.call(env)
@@ -108,8 +126,55 @@ class HTM
         # Return response
         [status, headers, body]
       ensure
-        # Return connections to pool
-        ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord)
+        # Return connections to pool after request completes
+        if defined?(ActiveRecord::Base) && ActiveRecord::Base.respond_to?(:connection_handler)
+          ActiveRecord::Base.connection_handler.clear_active_connections!
+        end
+      end
+
+      # Store the connection config at startup (called from register_htm)
+      def self.store_config!
+        @@config_mutex.synchronize do
+          return if @@db_config
+
+          @@db_config = HTM::ActiveRecordConfig.load_database_config
+          HTM.logger.debug "HTM database config stored for thread-safe access"
+        end
+      end
+
+      private
+
+      def ensure_thread_connection!
+        # Check if connection pool exists and has an active connection
+        pool_exists = begin
+          ActiveRecord::Base.connection_pool
+          true
+        rescue ActiveRecord::ConnectionNotDefined
+          false
+        end
+
+        if pool_exists
+          return if ActiveRecord::Base.connection_pool.active_connection?
+        end
+
+        # Re-establish connection using stored config
+        if @@db_config
+          ActiveRecord::Base.establish_connection(@@db_config)
+          HTM.logger.debug "HTM database connection established for request thread"
+        else
+          raise "HTM database config not stored - call register_htm at app startup"
+        end
+      rescue ActiveRecord::ConnectionNotDefined, ActiveRecord::ConnectionNotEstablished => e
+        # Pool doesn't exist, establish connection
+        if @@db_config
+          ActiveRecord::Base.establish_connection(@@db_config)
+          HTM.logger.debug "HTM database connection established for request thread"
+        else
+          raise "HTM database config not stored - call register_htm at app startup"
+        end
+      rescue StandardError => e
+        HTM.logger.error "Failed to ensure thread connection: #{e.class} - #{e.message}"
+        raise
       end
     end
   end
@@ -150,6 +215,16 @@ module ::Sinatra
         end
       end
 
+      # Store database config for thread-safe access and establish initial connection
+      begin
+        HTM::Sinatra::Middleware.store_config!
+        HTM::ActiveRecordConfig.establish_connection!
+        HTM.logger.info "HTM database connection established"
+      rescue StandardError => e
+        HTM.logger.error "Failed to establish HTM database connection: #{e.message}"
+        raise
+      end
+
       HTM.logger.info "HTM registered with Sinatra application"
       HTM.logger.debug "HTM job backend: #{HTM.configuration.job_backend}"
     end

data/lib/htm/job_adapter.rb

@@ -72,7 +72,10 @@ class HTM
 
         # Convert job class to Sidekiq worker if needed
         sidekiq_class = to_sidekiq_worker(job_class)
-        sidekiq_class.perform_async(**params)
+
+        # Sidekiq 7.x requires native JSON types - convert symbol keys to strings
+        json_params = params.transform_keys(&:to_s)
+        sidekiq_class.perform_async(json_params)
 
         HTM.logger.debug "Enqueued #{job_class.name} via Sidekiq with params: #{params.inspect}"
       end
@@ -135,12 +138,16 @@ class HTM
         return job_class if job_class.included_modules.include?(Sidekiq::Worker)
 
         # Create wrapper Sidekiq worker
+        # Note: Sidekiq 7.x requires JSON-compatible args, so we accept a hash
+        # and convert string keys back to symbols for the underlying job
         Class.new do
           include Sidekiq::Worker
           sidekiq_options queue: :htm, retry: 3
 
-          define_method(:perform) do |**params|
-            job_class.perform(**params)
+          define_method(:perform) do |params|
+            # Convert string keys back to symbols for the job class
+            symbolized_params = params.transform_keys(&:to_sym)
+            job_class.perform(**symbolized_params)
           end
 
           # Set descriptive name

data/lib/htm/jobs/generate_embedding_job.rb

@@ -43,13 +43,14 @@ class HTM
           result = HTM::EmbeddingService.generate(node.content)
 
           # Update node with processed embedding
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
 
           HTM.logger.info "GenerateEmbeddingJob: Successfully generated embedding for node #{node_id} (#{result[:dimension]} dimensions)"
 
+        rescue HTM::CircuitBreakerOpenError => e
+          # Circuit breaker is open - service is unavailable, will retry later
+          HTM.logger.warn "GenerateEmbeddingJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
+
         rescue HTM::EmbeddingError => e
           # Log embedding-specific errors
           HTM.logger.error "GenerateEmbeddingJob: Embedding generation failed for node #{node_id}: #{e.message}"

data/lib/htm/jobs/generate_tags_job.rb

@@ -63,6 +63,10 @@ class HTM
 
           HTM.logger.info "GenerateTagsJob: Successfully generated #{tag_names.length} tags for node #{node_id}: #{tag_names.join(', ')}"
 
+        rescue HTM::CircuitBreakerOpenError => e
+          # Circuit breaker is open - service is unavailable, will retry later
+          HTM.logger.warn "GenerateTagsJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
+
         rescue HTM::TagError => e
           # Log tag-specific errors
           HTM.logger.error "GenerateTagsJob: Tag generation failed for node #{node_id}: #{e.message}"