htm 0.0.2 → 0.0.10

data/lib/htm/database.rb

@@ -116,7 +116,7 @@ class HTM
 
         conn = PG.connect(config)
 
-        tables = ['nodes', 'node_tags', 'tags', 'robots', 'operations_log', 'schema_migrations']
+        tables = ['nodes', 'node_tags', 'tags', 'robots', 'robot_nodes', 'file_sources', 'schema_migrations']
 
         puts "Dropping HTM tables..."
         tables.each do |table|
@@ -282,7 +282,8 @@ class HTM
 
       # Generate database documentation using tbls
       #
-      # Creates comprehensive database documentation in dbdoc/ directory including:
+      # Uses .tbls.yml configuration file for output directory and settings.
+      # Creates comprehensive database documentation including:
       # - Entity-relationship diagrams
       # - Table schemas with comments
       # - Index information
@@ -292,23 +293,6 @@ class HTM
       # @return [void]
       #
       def generate_docs(db_url = nil)
-        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
-        raise "Database configuration not found" unless config
-
-        dbdoc_dir = File.expand_path('../../dbdoc', __dir__)
-
-        puts "Generating database documentation in #{dbdoc_dir}..."
-
-        # Create dbdoc directory if it doesn't exist
-        Dir.mkdir(dbdoc_dir) unless Dir.exist?(dbdoc_dir)
-
-        # Build PostgreSQL connection string for tbls
-        pg_url = if config[:password]
-          "postgresql://#{config[:user]}:#{config[:password]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
-        else
-          "postgresql://#{config[:user]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
-        end
-
         # Check if tbls is installed
         unless system('which tbls > /dev/null 2>&1')
           puts "✗ Error: 'tbls' is not installed"
@@ -322,9 +306,31 @@ class HTM
           exit 1
         end
 
-        # Run tbls doc command with --force to allow updates
+        # Find the project root (where .tbls.yml should be)
+        project_root = File.expand_path('../..', __dir__)
+        tbls_config = File.join(project_root, '.tbls.yml')
+
+        unless File.exist?(tbls_config)
+          puts "✗ Error: .tbls.yml not found at #{tbls_config}"
+          exit 1
+        end
+
+        # Get database URL
+        dsn = db_url || ENV['HTM_DBURL']
+        raise "Database configuration not found. Set HTM_DBURL environment variable." unless dsn
+
+        # Ensure sslmode is set for local development (tbls requires it)
+        unless dsn.include?('sslmode=')
+          separator = dsn.include?('?') ? '&' : '?'
+          dsn = "#{dsn}#{separator}sslmode=disable"
+        end
+
+        puts "Generating database documentation using #{tbls_config}..."
+
+        # Run tbls doc command with config file and DSN override
+        # The --dsn flag overrides the dsn in .tbls.yml but other settings are preserved
         require 'open3'
-        cmd = ['tbls', 'doc', '--force', pg_url, dbdoc_dir]
+        cmd = ['tbls', 'doc', '--config', tbls_config, '--dsn', dsn, '--force']
 
         stdout, stderr, status = Open3.capture3(*cmd)
 
@@ -336,15 +342,18 @@ class HTM
         end
 
         puts stdout if stdout && !stdout.empty?
+
+        # Read docPath from config to show correct output location
+        doc_path = 'docs/database'  # default from .tbls.yml
         puts "✓ Database documentation generated successfully"
         puts ""
         puts "Documentation files:"
-        puts "  #{dbdoc_dir}/README.md       - Main documentation"
-        puts "  #{dbdoc_dir}/schema.svg      - ER diagram (if generated)"
-        puts "  #{dbdoc_dir}/*.md            - Individual table documentation"
+        puts "  #{doc_path}/README.md       - Main documentation"
+        puts "  #{doc_path}/schema.svg      - ER diagram"
+        puts "  #{doc_path}/*.md            - Individual table documentation"
         puts ""
         puts "View documentation:"
-        puts "  open #{dbdoc_dir}/README.md"
+        puts "  open #{doc_path}/README.md"
       end
 
       # Show database info
@@ -382,7 +391,7 @@ class HTM
 
         # Table info
         puts "\nHTM Tables:"
-        tables = ['nodes', 'tags', 'robots', 'operations_log', 'schema_migrations']
+        tables = ['nodes', 'node_tags', 'tags', 'robots', 'robot_nodes', 'file_sources', 'schema_migrations']
         tables.each do |table|
           begin
             count = conn.exec("SELECT COUNT(*) FROM #{table}").first['count']
@@ -405,23 +414,41 @@ class HTM
 
       # Parse database connection URL
       #
-      # @param url [String] Connection URL
+      # @param url [String] Connection URL (e.g., postgresql://user:pass@host:port/dbname)
       # @return [Hash, nil] Connection configuration hash
+      # @raise [ArgumentError] If URL format is invalid
       #
       def parse_connection_url(url)
         return nil unless url
 
         uri = URI.parse(url)
+
+        # Validate URL format
+        unless uri.scheme&.match?(/\Apostgres(?:ql)?\z/i)
+          raise ArgumentError, "Invalid database URL scheme: #{uri.scheme}. Expected 'postgresql' or 'postgres'."
+        end
+
+        unless uri.host && !uri.host.empty?
+          raise ArgumentError, "Database URL must include a host"
+        end
+
+        dbname = uri.path&.slice(1..-1)  # Remove leading /
+        if dbname.nil? || dbname.empty?
+          raise ArgumentError, "Database URL must include a database name (path segment)"
+        end
+
         params = URI.decode_www_form(uri.query || '').to_h
 
         {
           host: uri.host,
-          port: uri.port,
-          dbname: uri.path[1..-1],  # Remove leading /
+          port: uri.port || 5432,
+          dbname: dbname,
           user: uri.user,
           password: uri.password,
           sslmode: params['sslmode'] || 'prefer'
         }
+      rescue URI::InvalidURIError => e
+        raise ArgumentError, "Invalid database URL format: #{e.message}"
       end
 
       # Build config from individual environment variables
@@ -432,12 +459,12 @@ class HTM
         return nil unless ENV['HTM_DBNAME']
 
         {
-          host: ENV['HTM_DBHOST'] || 'cw7rxj91bm.srbbwwxn56.tsdb.cloud.timescale.com',
-          port: (ENV['HTM_DBPORT'] || 37807).to_i,
+          host: ENV['HTM_DBHOST'] || 'localhost',
+          port: (ENV['HTM_DBPORT'] || 5432).to_i,
           dbname: ENV['HTM_DBNAME'],
           user: ENV['HTM_DBUSER'],
           password: ENV['HTM_DBPASS'],
-          sslmode: 'require'
+          sslmode: ENV['HTM_DBSSLMODE'] || 'prefer'
         }
       end
 
@@ -506,9 +533,11 @@ class HTM
           version = File.basename(file).split('_').first
           name = File.basename(file, '.rb')
 
-          # Check if already run
+          # Check if already run (use parameterized query to prevent SQL injection)
           already_run = conn.select_value(
-            "SELECT COUNT(*) FROM schema_migrations WHERE version = '#{version}'"
+            ActiveRecord::Base.sanitize_sql_array(
+              ["SELECT COUNT(*) FROM schema_migrations WHERE version = ?", version]
+            )
           ).to_i > 0
 
           if already_run
@@ -525,9 +554,11 @@ class HTM
             migration = migration_class.new
             migration.migrate(:up)
 
-            # Record in schema_migrations
+            # Record in schema_migrations (use parameterized query to prevent SQL injection)
             conn.execute(
-              "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
+              ActiveRecord::Base.sanitize_sql_array(
+                ["INSERT INTO schema_migrations (version) VALUES (?)", version]
+              )
             )
 
             puts "    ✓ Completed"

data/lib/htm/embedding_service.rb

@@ -10,12 +10,43 @@ class HTM
   # - Dimension handling (padding/truncation)
   # - Error handling and logging
   # - Storage formatting
+  # - Circuit breaker protection for external LLM failures
   #
   # The actual LLM call is delegated to HTM.configuration.embedding_generator
   #
   class EmbeddingService
     MAX_DIMENSION = 2000  # Maximum dimension for pgvector HNSW index
 
+    # Circuit breaker for embedding API calls
+    @circuit_breaker = nil
+    @circuit_breaker_mutex = Mutex.new
+
+    class << self
+      # Get or create the circuit breaker for embedding service
+      #
+      # @return [HTM::CircuitBreaker] The circuit breaker instance
+      #
+      def circuit_breaker
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker ||= HTM::CircuitBreaker.new(
+            name: 'embedding_service',
+            failure_threshold: 5,
+            reset_timeout: 60
+          )
+        end
+      end
+
+      # Reset the circuit breaker (useful for testing)
+      #
+      # @return [void]
+      #
+      def reset_circuit_breaker!
+        @circuit_breaker_mutex.synchronize do
+          @circuit_breaker&.reset!
+        end
+      end
+    end
+
     # Generate embedding with validation and processing
     #
     # @param text [String] Text to embed
@@ -26,12 +57,15 @@ class HTM
     #     storage_embedding: String,         # Formatted for database storage
     #     storage_dimension: Integer         # Padded dimension (2000)
     #   }
+    # @raise [CircuitBreakerOpenError] If circuit breaker is open
     #
     def self.generate(text)
       HTM.logger.debug "EmbeddingService: Generating embedding for #{text.length} chars"
 
-      # Call configured embedding generator
-      raw_embedding = HTM.configuration.embedding_generator.call(text)
+      # Use circuit breaker to protect against cascading failures
+      raw_embedding = circuit_breaker.call do
+        HTM.configuration.embedding_generator.call(text)
+      end
 
       # Validate response
       validate_embedding!(raw_embedding)
@@ -61,6 +95,9 @@ class HTM
         storage_dimension: MAX_DIMENSION
       }
 
+    rescue HTM::CircuitBreakerOpenError
+      # Re-raise circuit breaker errors without wrapping
+      raise
     rescue HTM::EmbeddingError
       raise
     rescue StandardError => e

data/lib/htm/errors.rb

@@ -1,34 +1,154 @@
 # frozen_string_literal: true
 
-# HTM error classes
+# HTM (Hierarchical Temporary Memory) error classes
+#
+# All HTM errors inherit from HTM::Error, allowing you to catch
+# all HTM-related errors with a single rescue clause.
+#
+# @example Catching all HTM errors
+#   begin
+#     htm.remember("some content")
+#   rescue HTM::Error => e
+#     logger.error "HTM error: #{e.message}"
+#   end
+#
+# @example Catching specific errors
+#   begin
+#     htm.forget(node_id, soft: false)
+#   rescue HTM::NotFoundError
+#     puts "Node not found"
+#   rescue HTM::ValidationError
+#     puts "Invalid input"
+#   end
+#
 class HTM
   # Base error class for all HTM errors
+  #
+  # All custom HTM errors inherit from this class, providing a common
+  # ancestor for error handling.
+  #
   class Error < StandardError; end
 
-  # Validation errors
+  # Raised when input validation fails
+  #
+  # Common causes:
+  # - Empty or nil content for remember()
+  # - Content exceeding maximum size limit
+  # - Invalid tag format
+  # - Invalid recall strategy
+  # - Invalid timeframe format
+  #
+  # @example
+  #   htm.remember("")  # => raises ValidationError
+  #   htm.remember("x", tags: ["INVALID!"])  # => raises ValidationError
+  #
   class ValidationError < Error; end
 
-  # Resource exhausted errors (memory, tokens, etc.)
+  # Raised when system resources are exhausted
+  #
+  # Common causes:
+  # - Working memory token limit exceeded
+  # - Database connection pool exhausted
+  # - Memory allocation failures
+  #
   class ResourceExhaustedError < Error; end
 
-  # Resource not found errors
+  # Raised when a requested resource cannot be found
+  #
+  # Common causes:
+  # - Node ID does not exist
+  # - Robot not registered
+  # - File source not found
+  #
+  # @example
+  #   htm.forget(999999)  # => raises NotFoundError if node doesn't exist
+  #
   class NotFoundError < Error; end
 
-  # Embedding service errors
+  # Raised when embedding generation fails
+  #
+  # Common causes:
+  # - LLM provider API errors
+  # - Invalid embedding response format
+  # - Network connectivity issues
+  # - Model not available
+  #
+  # Note: This error is distinct from CircuitBreakerOpenError.
+  # EmbeddingError indicates a single failure, while CircuitBreakerOpenError
+  # indicates repeated failures have triggered protective circuit breaking.
+  #
   class EmbeddingError < Error; end
 
-  # Tag service errors
+  # Raised when tag extraction fails
+  #
+  # Common causes:
+  # - LLM provider API errors
+  # - Invalid tag response format
+  # - Network connectivity issues
+  # - Model not available
+  #
+  # Note: This error is distinct from CircuitBreakerOpenError.
+  # TagError indicates a single failure, while CircuitBreakerOpenError
+  # indicates repeated failures have triggered protective circuit breaking.
+  #
   class TagError < Error; end
 
-  # Database operation errors
+  # Raised when database operations fail
+  #
+  # Common causes:
+  # - Connection failures
+  # - Query syntax errors
+  # - Constraint violations
+  # - Extension not installed (pgvector, pg_trgm)
+  #
   class DatabaseError < Error; end
 
-  # Query timeout errors
+  # Raised when a database query exceeds the configured timeout
+  #
+  # Default timeout is 30 seconds. Configure via db_query_timeout parameter
+  # when initializing HTM.
+  #
+  # @example Handling timeout
+  #   begin
+  #     htm.recall("complex query", strategy: :hybrid)
+  #   rescue HTM::QueryTimeoutError
+  #     # Retry with simpler query or smaller limit
+  #   end
+  #
   class QueryTimeoutError < DatabaseError; end
 
-  # Authorization errors
+  # Raised when an operation is not authorized
+  #
+  # Reserved for future multi-tenant scenarios where access control
+  # may restrict certain operations.
+  #
   class AuthorizationError < Error; end
 
-  # Circuit breaker errors
-  class CircuitBreakerOpenError < EmbeddingError; end
+  # Raised when circuit breaker is open due to repeated failures
+  #
+  # The circuit breaker pattern protects against cascading failures when
+  # external LLM services are unavailable. When too many consecutive
+  # failures occur, the circuit "opens" and subsequent calls fail fast
+  # without attempting the operation.
+  #
+  # Circuit states:
+  # - :closed - Normal operation, requests flow through
+  # - :open - Too many failures, requests fail immediately
+  # - :half_open - Testing if service recovered
+  #
+  # After a reset timeout (default: 60 seconds), the circuit transitions
+  # to half-open and tests if the service has recovered.
+  #
+  # @example Handling circuit breaker
+  #   begin
+  #     htm.remember("new content")
+  #   rescue HTM::CircuitBreakerOpenError
+  #     # LLM service unavailable, but node is still saved
+  #     # Embeddings/tags will be generated later when service recovers
+  #   end
+  #
+  # @see HTM::CircuitBreaker
+  # @see HTM::Observability.circuit_breaker_stats
+  #
+  class CircuitBreakerOpenError < Error; end
 end

data/lib/htm/jobs/generate_embedding_job.rb

@@ -43,13 +43,14 @@ class HTM
           result = HTM::EmbeddingService.generate(node.content)
 
           # Update node with processed embedding
-          node.update!(
-            embedding: result[:storage_embedding],
-            embedding_dimension: result[:dimension]
-          )
+          node.update!(embedding: result[:storage_embedding])
 
           HTM.logger.info "GenerateEmbeddingJob: Successfully generated embedding for node #{node_id} (#{result[:dimension]} dimensions)"
 
+        rescue HTM::CircuitBreakerOpenError => e
+          # Circuit breaker is open - service is unavailable, will retry later
+          HTM.logger.warn "GenerateEmbeddingJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
+
         rescue HTM::EmbeddingError => e
           # Log embedding-specific errors
           HTM.logger.error "GenerateEmbeddingJob: Embedding generation failed for node #{node_id}: #{e.message}"

data/lib/htm/jobs/generate_tags_job.rb

@@ -63,6 +63,10 @@ class HTM
 
           HTM.logger.info "GenerateTagsJob: Successfully generated #{tag_names.length} tags for node #{node_id}: #{tag_names.join(', ')}"
 
+        rescue HTM::CircuitBreakerOpenError => e
+          # Circuit breaker is open - service is unavailable, will retry later
+          HTM.logger.warn "GenerateTagsJob: Circuit breaker open for node #{node_id}, will retry when service recovers"
+
         rescue HTM::TagError => e
           # Log tag-specific errors
           HTM.logger.error "GenerateTagsJob: Tag generation failed for node #{node_id}: #{e.message}"