htm 0.0.17 → 0.0.18

data/lib/htm/mcp/cli.rb

@@ -8,7 +8,7 @@ class HTM
 
       def print_help
         puts <<~HELP
-          HTM MCP Server - Memory management for AI assistants
+          HTM MCP Server v#{HTM::VERSION} - Memory management for AI assistants
 
           USAGE:
             htm_mcp [COMMAND]
@@ -20,66 +20,81 @@ class HTM
             init      Alias for setup
             verify    Verify database connection and extensions
             stats     Show memory statistics
+            config    Output default configuration to STDOUT
             version   Show HTM version
             help      Show this help message
 
           ENVIRONMENT VARIABLES:
 
+            Note: Nested config uses double underscores (e.g., HTM_EMBEDDING__PROVIDER)
+
             Environment:
               HTM_ENV                       Environment name: development, test, production
                                             (priority: HTM_ENV > RAILS_ENV > RACK_ENV > 'development')
 
-            Database (required):
-              HTM_DBURL                     PostgreSQL connection URL
+            Database:
+              HTM_DATABASE__URL             PostgreSQL connection URL (preferred)
                                             Example: postgresql://user:pass@localhost:5432/htm_development
-
-            Database (alternative to HTM_DBURL):
-              HTM_DBNAME                    Database name
-              HTM_DBHOST                    Database host (default: localhost)
-              HTM_DBPORT                    Database port (default: 5432)
-              HTM_DBUSER                    Database username
-              HTM_DBPASS                    Database password
-              HTM_DBSSLMODE                 SSL mode (default: prefer)
-
-            LLM Providers:
-              HTM_EMBEDDING_PROVIDER        Embedding provider (default: ollama)
-              HTM_EMBEDDING_MODEL           Embedding model (default: nomic-embed-text:latest)
-              HTM_EMBEDDING_DIMENSIONS      Embedding dimensions (default: 768)
-              HTM_TAG_PROVIDER              Tag extraction provider (default: ollama)
-              HTM_TAG_MODEL                 Tag model (default: gemma3:latest)
-              HTM_PROPOSITION_PROVIDER      Proposition provider (default: ollama)
-              HTM_PROPOSITION_MODEL         Proposition model (default: gemma3:latest)
-              HTM_EXTRACT_PROPOSITIONS      Enable propositions (default: false)
-
-            Ollama (default provider):
-              HTM_OLLAMA_URL                Ollama server URL (default: http://localhost:11434)
-
-            Other Providers (set API keys as needed):
-              HTM_OPENAI_API_KEY            OpenAI API key
-              HTM_ANTHROPIC_API_KEY         Anthropic API key
-              HTM_GEMINI_API_KEY            Google Gemini API key
-              HTM_AZURE_API_KEY             Azure OpenAI API key
-              HTM_AZURE_ENDPOINT            Azure OpenAI endpoint
-
-            Timeouts:
-              HTM_EMBEDDING_TIMEOUT         Embedding timeout seconds (default: 120)
-              HTM_TAG_TIMEOUT               Tag timeout seconds (default: 180)
-              HTM_CONNECTION_TIMEOUT        Connection timeout seconds (default: 30)
+              HTM_DATABASE__HOST            Database host (default: localhost)
+              HTM_DATABASE__PORT            Database port (default: 5432)
+              HTM_DATABASE__NAME            Database name
+              HTM_DATABASE__USER            Database username
+              HTM_DATABASE__PASSWORD        Database password
+              HTM_DATABASE__SSLMODE         SSL mode (default: prefer)
+              HTM_DATABASE__POOL_SIZE       Connection pool size (default: 10)
+
+            Embedding:
+              HTM_EMBEDDING__PROVIDER       Provider (default: ollama)
+              HTM_EMBEDDING__MODEL          Model (default: nomic-embed-text:latest)
+              HTM_EMBEDDING__DIMENSIONS     Dimensions (default: 768)
+              HTM_EMBEDDING__TIMEOUT        Timeout seconds (default: 120)
+              HTM_EMBEDDING__MAX_DIMENSION  Max dimensions (default: 2000)
+
+            Tag Extraction:
+              HTM_TAG__PROVIDER             Provider (default: ollama)
+              HTM_TAG__MODEL                Model (default: gemma3:latest)
+              HTM_TAG__TIMEOUT              Timeout seconds (default: 180)
+              HTM_TAG__MAX_DEPTH            Max hierarchy depth (default: 4)
+
+            Proposition Extraction:
+              HTM_PROPOSITION__PROVIDER     Provider (default: ollama)
+              HTM_PROPOSITION__MODEL        Model (default: gemma3:latest)
+              HTM_PROPOSITION__TIMEOUT      Timeout seconds (default: 180)
+              HTM_PROPOSITION__ENABLED      Enable extraction (default: false)
 
             Chunking:
-              HTM_CHUNK_SIZE                Max chars per chunk (default: 1024)
-              HTM_CHUNK_OVERLAP             Chunk overlap chars (default: 64)
+              HTM_CHUNKING__SIZE            Max chars per chunk (default: 1024)
+              HTM_CHUNKING__OVERLAP         Chunk overlap chars (default: 64)
+
+            Job Backend:
+              HTM_JOB__BACKEND              Backend: inline, thread, active_job, sidekiq
+
+            Provider API Keys:
+              HTM_PROVIDERS__OLLAMA__URL           Ollama URL (default: http://localhost:11434)
+              HTM_PROVIDERS__OPENAI__API_KEY       OpenAI API key
+              HTM_PROVIDERS__ANTHROPIC__API_KEY    Anthropic API key
+              HTM_PROVIDERS__GEMINI__API_KEY       Google Gemini API key
+              HTM_PROVIDERS__AZURE__API_KEY        Azure OpenAI API key
+              HTM_PROVIDERS__AZURE__ENDPOINT       Azure OpenAI endpoint
 
             Other:
-              HTM_LOG_LEVEL                 Log level (default: INFO)
-              HTM_JOB_BACKEND               Job backend: inline, thread, active_job, sidekiq
+              HTM_LOG_LEVEL                 Log level (default: info)
+              HTM_CONNECTION_TIMEOUT        Connection timeout seconds (default: 30)
               HTM_TELEMETRY_ENABLED         Enable OpenTelemetry (default: false)
-              HTM_MAX_EMBEDDING_DIMENSION   Max vector dimensions (default: 2000)
-              HTM_MAX_TAG_DEPTH             Max tag hierarchy depth (default: 4)
+
+          OPTIONS:
+            -c, --config [PATH]   Without PATH: output default config to STDOUT
+                                  With PATH: load config from YAML file
 
           EXAMPLES:
+            # Generate a config file template
+            htm_mcp --config > my_config.yml
+
+            # Start server with custom config
+            htm_mcp --config my_config.yml
+
             # First-time setup
-            export HTM_DBURL="postgresql://postgres@localhost:5432/htm"
+            export HTM_DATABASE__URL="postgresql://postgres@localhost:5432/htm"
             htm_mcp setup
 
             # Verify connection
@@ -100,7 +115,7 @@ class HTM
                 "htm-memory": {
                   "command": "/path/to/htm_mcp",
                   "env": {
-                    "HTM_DBURL": "postgresql://postgres@localhost:5432/htm_development"
+                    "HTM_DATABASE__URL": "postgresql://postgres@localhost:5432/htm_development"
                   }
                 }
               }
@@ -109,9 +124,9 @@ class HTM
       end
 
       def check_database_config!
-        unless ENV['HTM_DBURL'] || ENV['HTM_DBNAME']
+        unless ENV['HTM_DATABASE__URL'] || ENV['HTM_DATABASE__NAME']
           warn "Error: Database not configured."
-          warn "Set HTM_DBURL or HTM_DBNAME environment variable."
+          warn "Set HTM_DATABASE__URL or HTM_DATABASE__NAME environment variable."
           warn "Run 'htm_mcp help' for details."
           exit 1
         end
@@ -123,12 +138,12 @@ class HTM
         warn ""
         if msg.include?("does not exist")
           warn "Suggestion: The database does not exist. Create it with:"
-          warn "  createdb #{extract_dbname(ENV['HTM_DBURL'] || ENV['HTM_DBNAME'])}"
+          warn "  createdb #{extract_dbname(ENV['HTM_DATABASE__URL'] || ENV['HTM_DATABASE__NAME'])}"
           warn "Then initialize the schema with:"
           warn "  htm_mcp setup"
         elsif msg.include?("password authentication failed") || msg.include?("no password supplied")
           warn "Suggestion: Check your database credentials."
-          warn "Verify HTM_DBURL has correct username and password:"
+          warn "Verify HTM_DATABASE__URL has correct username and password:"
           warn "  postgresql://USER:PASSWORD@localhost:5432/DATABASE"
         elsif msg.include?("connection refused") || msg.include?("could not connect")
           warn "Suggestion: PostgreSQL server is not running or not accepting connections."
@@ -261,6 +276,101 @@ class HTM
         pending_count
       end
 
+      def output_default_config
+        defaults_path = File.expand_path('../config/defaults.yml', __dir__)
+        if File.exist?(defaults_path)
+          puts File.read(defaults_path)
+        else
+          warn "Error: defaults.yml not found at #{defaults_path}"
+          exit 1
+        end
+      end
+
+      def load_config_file(path)
+        unless File.exist?(path)
+          warn "Error: Config file not found: #{path}"
+          exit 1
+        end
+
+        begin
+          require 'yaml'
+          config_data = YAML.safe_load(
+            File.read(path),
+            permitted_classes: [Symbol],
+            symbolize_names: true,
+            aliases: true
+          ) || {}
+
+          # Determine which section to use based on environment
+          env = HTM::Config.env.to_sym
+          base = config_data[:defaults] || {}
+          env_overrides = config_data[env] || {}
+
+          # Merge base with environment-specific overrides
+          merged = deep_merge(base, env_overrides)
+
+          apply_config(merged)
+
+          warn "Loaded configuration from: #{path}"
+          warn "Environment: #{env}"
+        rescue => e
+          warn "Error loading config file: #{e.message}"
+          warn e.backtrace.first(5).join("\n") if ENV['DEBUG']
+          exit 1
+        end
+      end
+
+      def deep_merge(base, override)
+        base.merge(override) do |_key, old_val, new_val|
+          if old_val.is_a?(Hash) && new_val.is_a?(Hash)
+            deep_merge(old_val, new_val)
+          else
+            new_val.nil? ? old_val : new_val
+          end
+        end
+      end
+
+      def apply_config(config)
+        HTM.configure do |c|
+          # Apply nested sections
+          apply_section(c, :database, config[:database])
+          apply_section(c, :service, config[:service])
+          apply_section(c, :embedding, config[:embedding])
+          apply_section(c, :tag, config[:tag])
+          apply_section(c, :proposition, config[:proposition])
+          apply_section(c, :chunking, config[:chunking])
+          apply_section(c, :circuit_breaker, config[:circuit_breaker])
+          apply_section(c, :relevance, config[:relevance])
+          apply_section(c, :job, config[:job])
+          apply_section(c, :providers, config[:providers])
+
+          # Apply top-level scalars
+          c.week_start = config[:week_start] if config[:week_start]
+          c.connection_timeout = config[:connection_timeout] if config[:connection_timeout]
+          c.telemetry_enabled = config[:telemetry_enabled] unless config[:telemetry_enabled].nil?
+          c.log_level = config[:log_level] if config[:log_level]
+        end
+      end
+
+      def apply_section(config, section_name, values)
+        return unless values.is_a?(Hash)
+
+        section = config.send(section_name)
+        values.each do |key, value|
+          next if value.nil?
+
+          if value.is_a?(Hash)
+            # Handle nested sections (like providers.openai)
+            subsection = section.send(key)
+            value.each do |subkey, subvalue|
+              subsection.send("#{subkey}=", subvalue) unless subvalue.nil?
+            end
+          else
+            section.send("#{key}=", value)
+          end
+        end
+      end
+
       def run_stats
         puts "HTM Memory Statistics"
         puts "====================="
@@ -298,35 +408,67 @@ class HTM
       end
 
       def run(args)
+        args = args.dup
+
+        # Handle -c / --config option first (can be combined with other commands)
+        config_loaded = handle_config_option(args)
+
+        # Process remaining command
         case args[0]&.downcase
         when 'help', '-h', '--help'
           print_help
-          exit 0
         when 'version', '-v', '--version'
           puts "HTM #{HTM::VERSION}"
-          exit 0
         when 'setup', 'init'
           run_setup
-          exit 0
         when 'verify'
           run_verify
-          exit 0
         when 'stats'
           run_stats
-          exit 0
+        when 'config'
+          output_default_config
         when 'server', 'stdio', nil
           # Return false to indicate server should start
           # 'stdio' is accepted for compatibility with MCP clients that pass it as an argument
-          false
+          return false
         when /^-/
-          warn "Unknown option: #{args[0]}"
-          warn "Run 'htm_mcp help' for usage."
+          $stderr.puts "Unknown option: #{args[0]}"
+          $stderr.puts "Run 'htm_mcp help' for usage."
           exit 1
         else
-          warn "Unknown command: #{args[0]}"
-          warn "Run 'htm_mcp help' for usage."
+          $stderr.puts "Unknown command: #{args[0]}"
+          $stderr.puts "Run 'htm_mcp help' for usage."
           exit 1
         end
+        true
+      end
+
+      # Handle -c / --config option, modifying args in place
+      # Returns true if config was loaded, nil otherwise
+      def handle_config_option(args)
+        config_idx = args.index('-c') || args.index('--config')
+        return nil unless config_idx
+
+        # Remove the -c/--config flag
+        args.delete_at(config_idx)
+
+        # Check if next arg is a path (not another flag or command)
+        next_arg = args[config_idx]
+
+        if next_arg.nil? || next_arg.start_with?('-') || command?(next_arg)
+          # No path provided - output default config and exit
+          output_default_config
+          exit 0
+        else
+          # Path provided - load config file
+          config_path = args.delete_at(config_idx)
+          load_config_file(config_path)
+          true
+        end
+      end
+
+      def command?(arg)
+        %w[help version setup init verify stats config server stdio].include?(arg.downcase)
       end
     end
   end

data/lib/htm/mcp/server.rb

@@ -28,9 +28,9 @@ class HTM
       end
 
       def check_database_config!
-        unless ENV['HTM_DBURL'] || ENV['HTM_DBNAME']
+        unless ENV['HTM_DATABASE__URL'] || ENV['HTM_DATABASE__NAME']
           warn "Error: Database not configured."
-          warn "Set HTM_DBURL or HTM_DBNAME environment variable."
+          warn "Set HTM_DATABASE__URL or HTM_DATABASE__NAME environment variable."
           warn "Run 'htm_mcp help' for details."
           exit 1
         end
@@ -70,7 +70,7 @@ class HTM
 
       def configure_htm!
         HTM.configure do |config|
-          config.job_backend = :inline  # Synchronous for MCP responses
+          config.job.backend = :inline  # Synchronous for MCP responses
           config.logger = @silent_logger    # Silent logging for MCP
         end
       end

data/lib/htm/proposition_service.rb

@@ -67,8 +67,6 @@ class HTM
     # @raise [PropositionError] If extraction fails
     #
     def self.extract(content)
-      HTM.logger.debug "PropositionService: Extracting propositions from #{content.length} chars"
-
       # Use circuit breaker to protect against cascading failures
       raw_propositions = circuit_breaker.call do
         HTM.configuration.proposition_extractor.call(content)
@@ -78,11 +76,7 @@ class HTM
       parsed_propositions = parse_propositions(raw_propositions)
 
       # Validate and filter propositions
-      valid_propositions = validate_and_filter_propositions(parsed_propositions)
-
-      HTM.logger.debug "PropositionService: Extracted #{valid_propositions.length} valid propositions"
-
-      valid_propositions
+      validate_and_filter_propositions(parsed_propositions)
 
     rescue HTM::CircuitBreakerOpenError
       # Re-raise circuit breaker errors without wrapping
@@ -128,10 +122,7 @@ class HTM
 
       propositions.each do |proposition|
         # Check minimum length
-        if proposition.length < MIN_PROPOSITION_LENGTH
-          HTM.logger.debug "PropositionService: Proposition too short, skipping: #{proposition}"
-          next
-        end
+        next if proposition.length < MIN_PROPOSITION_LENGTH
 
         # Check maximum length
         if proposition.length > MAX_PROPOSITION_LENGTH
@@ -141,7 +132,6 @@ class HTM
 
         # Check for actual content (not just punctuation/whitespace)
         unless proposition.match?(/[a-zA-Z]{3,}/)
-          HTM.logger.debug "PropositionService: Proposition lacks content, skipping: #{proposition}"
           next
         end
 

data/lib/htm/railtie.rb

@@ -32,14 +32,13 @@ class HTM
         config.logger = Rails.logger
 
         # Use ActiveJob for background jobs in Rails
-        config.job_backend = :active_job unless Rails.env.test?
+        config.job.backend = :active_job unless Rails.env.test?
 
         # Use inline execution in test environment for synchronous behavior
-        config.job_backend = :inline if Rails.env.test?
+        config.job.backend = :inline if Rails.env.test?
       end
 
       HTM.logger.info "HTM initialized for Rails application"
-      HTM.logger.debug "HTM job backend: #{HTM.configuration.job_backend}"
     end
 
     # Load Rake tasks
@@ -63,7 +62,7 @@ class HTM
           HTM.logger.info "HTM database connection verified"
         rescue StandardError => e
           HTM.logger.warn "HTM database connection check failed: #{e.message}"
-          HTM.logger.warn "Set HTM_DBURL environment variable or configure database.yml"
+          HTM.logger.warn "Set HTM_DATABASE__URL environment variable"
         end
       end
     end

data/lib/htm/tag_service.rb

@@ -65,9 +65,6 @@ class HTM
     # @raise [CircuitBreakerOpenError] If circuit breaker is open
     #
     def self.extract(content, existing_ontology: [])
-      HTM.logger.debug "TagService: Extracting tags from #{content.length} chars"
-      HTM.logger.debug "TagService: Using ontology with #{existing_ontology.size} existing tags"
-
       # Use circuit breaker to protect against cascading failures
       raw_tags = circuit_breaker.call do
         HTM.configuration.tag_extractor.call(content, existing_ontology)
@@ -77,11 +74,7 @@ class HTM
       parsed_tags = parse_tags(raw_tags)
 
       # Validate and filter tags
-      valid_tags = validate_and_filter_tags(parsed_tags)
-
-      HTM.logger.debug "TagService: Extracted #{valid_tags.length} valid tags: #{valid_tags.join(', ')}"
-
-      valid_tags
+      validate_and_filter_tags(parsed_tags)
 
     rescue HTM::CircuitBreakerOpenError
       # Re-raise circuit breaker errors without wrapping

data/lib/htm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class HTM
-  VERSION = '0.0.17'
+  VERSION = '0.0.18'
 end

data/lib/htm.rb

@@ -2,7 +2,7 @@
 
 require_relative "htm/version"
 require_relative "htm/errors"
-require_relative "htm/configuration"
+require_relative "htm/config"
 require_relative "htm/circuit_breaker"
 require_relative "htm/active_record_config"
 require_relative "htm/database"
@@ -70,7 +70,7 @@ class HTM
   #
   # @param working_memory_size [Integer] Maximum tokens for working memory (default: 128,000)
   # @param robot_name [String] Human-readable name for this robot (auto-generated if not provided)
-  # @param db_config [Hash] Database configuration (uses ENV['HTM_DBURL'] if not provided)
+  # @param db_config [Hash] Database configuration (uses ENV['HTM_DATABASE__URL'] if not provided)
   # @param db_pool_size [Integer] Database connection pool size (default: 5)
   # @param db_query_timeout [Integer] Database query timeout in milliseconds (default: 30000)
   # @param db_cache_size [Integer] Number of database query results to cache (default: 1000, use 0 to disable)
@@ -170,7 +170,7 @@ class HTM
       enqueue_tags_job(node_id, manual_tags: tags)
 
       # Enqueue proposition extraction if enabled and not already a proposition
-      if HTM.configuration.extract_propositions && !metadata[:is_proposition]
+      if HTM.config.extract_propositions && !metadata[:is_proposition]
         enqueue_propositions_job(node_id)
       end
     else
@@ -251,7 +251,6 @@ class HTM
     normalized_timeframe = if timeframe == :auto
       result = HTM::Timeframe.normalize(:auto, query: topic)
       search_query = result.query  # Use cleaned query for search
-      HTM.logger.debug "Auto-extracted timeframe: #{result.extracted.inspect}" if result.extracted
       result.timeframe
     else
       HTM::Timeframe.normalize(timeframe)
@@ -595,7 +594,6 @@ class HTM
         tag = HTM::Models::Tag.find_or_create_by!(name: tag_name)
         HTM::Models::NodeTag.find_or_create_by!(node_id: node_id, tag_id: tag.id)
       end
-      HTM.logger.debug "Added #{manual_tags.length} manual tags to node #{node_id}"
     end
 
     # Enqueue tag generation using configured job backend
@@ -679,4 +677,125 @@ class HTM
     end
   end
 
+  # ===========================================================================
+  # Class Methods
+  # ===========================================================================
+
+  class << self
+    # Get current configuration (singleton)
+    #
+    # @return [HTM::Config]
+    #
+    def config
+      @config ||= Config.new
+    end
+
+    # Alias for backward compatibility
+    alias configuration config
+
+    # Configure HTM
+    #
+    # @yield [config] Configuration object
+    # @yieldparam config [HTM::Config]
+    #
+    # @example Custom configuration
+    #   HTM.configure do |config|
+    #     config.embedding_generator = ->(text) { MyEmbedder.embed(text) }
+    #     config.tag_extractor = ->(text, ontology) { MyTagger.extract(text, ontology) }
+    #   end
+    #
+    # @example Default configuration
+    #   HTM.configure  # Uses RubyLLM defaults
+    #
+    def configure
+      yield(config) if block_given?
+      config.validate!
+      config
+    end
+
+    # Reset configuration to defaults
+    def reset_configuration!
+      @config = nil
+    end
+
+    # Get current environment
+    #
+    # @return [String] Current environment name
+    #
+    def env
+      Config.env
+    end
+
+    # Check if running in test environment
+    #
+    # @return [Boolean]
+    #
+    def test?
+      env == 'test'
+    end
+
+    # Check if running in development environment
+    #
+    # @return [Boolean]
+    #
+    def development?
+      env == 'development'
+    end
+
+    # Check if running in production environment
+    #
+    # @return [Boolean]
+    #
+    def production?
+      env == 'production'
+    end
+
+    # Generate embedding using EmbeddingService
+    #
+    # @param text [String] Text to embed
+    # @return [Array<Float>] Embedding vector (original, not padded)
+    #
+    def embed(text)
+      result = HTM::EmbeddingService.generate(text)
+      result[:embedding]
+    end
+
+    # Extract tags using TagService
+    #
+    # @param text [String] Text to analyze
+    # @param existing_ontology [Array<String>] Sample of existing tags for context
+    # @return [Array<String>] Extracted and validated tag names
+    #
+    def extract_tags(text, existing_ontology: [])
+      HTM::TagService.extract(text, existing_ontology: existing_ontology)
+    end
+
+    # Extract propositions using PropositionService
+    #
+    # @param text [String] Text to analyze
+    # @return [Array<String>] Extracted atomic propositions
+    #
+    def extract_propositions(text)
+      HTM::PropositionService.extract(text)
+    end
+
+    # Count tokens using configured counter
+    #
+    # @param text [String] Text to count tokens for
+    # @return [Integer] Token count
+    #
+    def count_tokens(text)
+      config.token_counter.call(text)
+    rescue StandardError => e
+      raise HTM::ValidationError, "Token counting failed: #{e.message}"
+    end
+
+    # Get configured logger
+    #
+    # @return [Logger] Configured logger instance
+    #
+    def logger
+      config.logger
+    end
+  end
 end