htm 0.0.20 → 0.0.30

data/examples/examples_helper.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+# Examples Helper - Ensures all examples use the htm_examples database
+#
+# This file must be required at the top of every example file before loading HTM.
+# It enforces database isolation to prevent examples from polluting development
+# or production databases.
+#
+# Usage:
+#   require_relative 'examples_helper'  # For files in examples/
+#   require_relative '../examples_helper'  # For files in examples/subdirectory/
+#
+# This sets:
+#   - HTM_ENV=examples
+#   - HTM_DATABASE__URL to point to htm_examples database
+#
+# Before running examples:
+#   1. Create the examples database: createdb htm_examples
+#   2. Set up schema: HTM_ENV=examples rake htm:db:setup
+#   3. Run example: ruby examples/basic_usage.rb
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+# Set examples environment BEFORE loading HTM
+# This is critical because HTM::Config reads environment at load time
+# Note: 'examples' is not in the gem's defaults.yml - it's configured here
+ENV['HTM_ENV'] = 'examples'
+
+# Build examples database name from service name + environment
+# Uses HTM_SERVICE__NAME env var if set, otherwise defaults to 'htm'
+service_name = ENV['HTM_SERVICE__NAME'] || 'htm'
+examples_db_name = "#{service_name}_examples"
+
+# Safety check: Refuse to run examples against non-examples databases
+# Database name must end with _examples (e.g., htm_examples, myapp_examples)
+if ENV['HTM_DATABASE__URL'] && !ENV['HTM_DATABASE__URL'].include?('_examples')
+  abort <<~ERROR
+    SAFETY CHECK FAILED: Examples must run against an examples database!
+
+    HTM_DATABASE__URL is set to: #{ENV['HTM_DATABASE__URL']}
+
+    This does not appear to be an examples database (must contain '_examples').
+    Running examples against development or production databases can corrupt data.
+
+    To fix, either:
+      1. Unset HTM_DATABASE__URL and let this helper configure it
+      2. Set: export HTM_DATABASE__URL="postgresql://#{ENV['USER']}@localhost:5432/#{examples_db_name}"
+
+  ERROR
+end
+
+# ALWAYS use the examples database - never allow examples to run against other databases
+examples_db_url = "postgresql://#{ENV['USER']}@localhost:5432/#{examples_db_name}"
+ENV['HTM_DATABASE__URL'] = examples_db_url
+
+# Load HTM first
+require "htm"
+
+# Configure HTM for examples environment
+# This keeps the 'examples' environment configuration out of the gem's bundled defaults
+HTM.configure do |config|
+  # Use inline job backend for CLI examples (clearer console output)
+  # Rails/Sinatra apps should override this in their initializers to use :fiber or :active_job
+  config.job.backend = :inline unless defined?(Rails)
+
+  # Set log level for examples
+  config.log_level = :info
+
+  # Disable telemetry for examples
+  config.telemetry_enabled = false
+end
+
+# Module with helper methods for examples
+module ExamplesHelper
+  # Check if database is available and configured
+  #
+  # @return [Boolean] true if database is ready
+  def self.database_ready?
+    return false unless HTM.config.database_configured?
+
+    begin
+      HTM::SequelConfig.establish_connection!
+      HTM::SequelConfig.connected?
+    rescue => e
+      false
+    end
+  end
+
+  # Verify database is ready or print helpful error message
+  #
+  # @return [void] exits with error if database not ready
+  def self.require_database!
+    unless database_ready?
+      abort <<~ERROR
+        ERROR: Examples database not available.
+
+        Please set up the examples database:
+          1. createdb htm_examples
+          2. HTM_ENV=examples rake htm:db:setup
+
+        Then run the example again.
+      ERROR
+    end
+  end
+
+  # Print a section header
+  #
+  # @param title [String] section title
+  def self.section(title)
+    border = "=" * (title.size + 6)
+    puts
+    puts border
+    puts "== #{title} =="
+    puts border
+    puts
+  end
+
+  # Print a success message
+  #
+  # @param message [String] success message
+  def self.success(message)
+    puts "[OK] #{message}"
+  end
+
+  # Print an info message
+  #
+  # @param message [String] info message
+  def self.info(message)
+    puts "[..] #{message}"
+  end
+
+  # Print environment info
+  def self.print_environment
+    puts "Environment: #{HTM.config.environment}"
+    puts "Database: #{HTM.config.actual_database_name}"
+    puts "Job Backend: #{HTM.config.job.backend}"
+  end
+end

data/lib/htm/config/builder.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+class HTM
+  class Config
+    module Builder
+      def build_default_logger
+        logger = Logger.new($stdout)
+        logger.level = log_level
+        logger.formatter = proc do |severity, datetime, _progname, msg|
+          "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity} -- HTM: #{msg}\n"
+        end
+        logger
+      end
+
+      def build_default_token_counter
+        lambda do |text|
+          require 'tiktoken_ruby' unless defined?(Tiktoken)
+          encoder = Tiktoken.encoding_for_model("gpt-3.5-turbo")
+          encoder.encode(text).length
+        end
+      end
+
+      def build_default_embedding_generator
+        lambda do |text|
+          require 'ruby_llm' unless defined?(RubyLLM)
+
+          configure_ruby_llm(embedding_provider)
+          refresh_ollama_models! if embedding_provider == :ollama
+
+          model = embedding_provider == :ollama ? normalize_ollama_model(embedding_model) : embedding_model
+          response = RubyLLM.embed(text, model: model)
+          embedding = extract_embedding_from_response(response)
+
+          unless embedding.is_a?(Array) && embedding.all? { |v| v.is_a?(Numeric) }
+            raise HTM::EmbeddingError, "Invalid embedding response format from #{embedding_provider}"
+          end
+
+          embedding
+        end
+      end
+
+      def build_default_tag_extractor
+        lambda do |text, existing_ontology = []|
+          require 'ruby_llm' unless defined?(RubyLLM)
+
+          configure_ruby_llm(tag_provider)
+          refresh_ollama_models! if tag_provider == :ollama
+
+          model = tag_provider == :ollama ? normalize_ollama_model(tag_model) : tag_model
+
+          prompt = build_tag_extraction_prompt(text, existing_ontology)
+          system_prompt = build_tag_system_prompt
+
+          chat = RubyLLM.chat(model: model)
+          chat.with_instructions(system_prompt)
+          response = chat.ask(prompt)
+
+          parse_tag_response(extract_text_from_response(response))
+        end
+      end
+
+      def build_default_proposition_extractor
+        lambda do |text|
+          require 'ruby_llm' unless defined?(RubyLLM)
+
+          configure_ruby_llm(proposition_provider)
+          refresh_ollama_models! if proposition_provider == :ollama
+
+          model = proposition_provider == :ollama ? normalize_ollama_model(proposition_model) : proposition_model
+
+          prompt = build_proposition_extraction_prompt(text)
+          system_prompt = build_proposition_system_prompt
+
+          chat = RubyLLM.chat(model: model)
+          chat.with_instructions(system_prompt)
+          response = chat.ask(prompt)
+
+          parse_proposition_response(extract_text_from_response(response))
+        end
+      end
+
+      # ==========================================================================
+      # Response Extraction Helpers
+      # ==========================================================================
+
+      def extract_embedding_from_response(response)
+        return nil unless response
+
+        case response
+        when Array
+          response
+        when ->(r) { r.respond_to?(:vectors) }
+          vectors = response.vectors
+          vectors.is_a?(Array) && vectors.first.is_a?(Array) ? vectors.first : vectors
+        when ->(r) { r.respond_to?(:to_a) }
+          response.to_a
+        when ->(r) { r.respond_to?(:embedding) }
+          response.embedding
+        else
+          if response.respond_to?(:instance_variable_get)
+            vectors = response.instance_variable_get(:@vectors)
+            return vectors.first if vectors.is_a?(Array) && vectors.first.is_a?(Array)
+            return vectors if vectors.is_a?(Array)
+          end
+          raise HTM::EmbeddingError, "Cannot extract embedding from response: #{response.class}"
+        end
+      end
+
+      def extract_text_from_response(response)
+        return '' unless response
+
+        case response
+        when String then response
+        when ->(r) { r.respond_to?(:content) } then response.content.to_s
+        when ->(r) { r.respond_to?(:text) } then response.text.to_s
+        else response.to_s
+        end
+      end
+
+      def parse_tag_response(text)
+        tags = text.to_s.split("\n").map(&:strip).reject(&:empty?)
+        valid_tags = tags.select { |tag| tag =~ /^[a-z0-9\-]+(:[a-z0-9\-]+)*$/ }
+        valid_tags.select { |tag| tag.count(':') < max_tag_depth }
+      end
+
+      def parse_proposition_response(text)
+        text.to_s
+          .split("\n")
+          .map(&:strip)
+          .map { |line| line.sub(/^[-*]\s*/, '') }
+          .map(&:strip)
+          .reject(&:empty?)
+      end
+
+      # ==========================================================================
+      # Prompt Builders
+      # ==========================================================================
+
+      def build_tag_extraction_prompt(text, existing_ontology)
+        taxonomy_context = if existing_ontology.any?
+          sample_tags = existing_ontology.sample([existing_ontology.size, 20].min)
+          tag.taxonomy_context_existing % { sample_tags: sample_tags.join(', ') }
+        else
+          tag.taxonomy_context_empty
+        end
+
+        tag.user_prompt_template % {
+          text: text,
+          max_depth: max_tag_depth,
+          taxonomy_context: taxonomy_context
+        }
+      end
+
+      def build_tag_system_prompt
+        tag.system_prompt.to_s.strip
+      end
+
+      def build_proposition_extraction_prompt(text)
+        proposition.user_prompt_template % { text: text }
+      end
+
+      def build_proposition_system_prompt
+        proposition.system_prompt.to_s.strip
+      end
+    end
+  end
+end

data/lib/htm/config/database.rb

@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+class HTM
+  class Config
+    module Database
+      # ==========================================================================
+      # Database Component Accessors
+      # ==========================================================================
+      #
+      # These methods provide convenient access to database components.
+      # Components are automatically reconciled at config load time:
+      #   - If database.url exists: components are extracted and populated
+      #   - If database.url is missing: it's built from components
+      #
+      # ==========================================================================
+
+      # @return [String, nil] the database host
+      def database_host
+        database.host
+      end
+
+      # @return [Integer, nil] the database port
+      def database_port
+        database.port
+      end
+
+      # @return [String, nil] the database name
+      def database_name
+        database.name
+      end
+
+      # @return [String, nil] the database user
+      def database_user
+        database.user
+      end
+
+      # @return [String, nil] the database password
+      def database_password
+        database.password
+      end
+
+      def database_config
+        url = database_url
+        return {} unless url
+
+        uri = URI.parse(url)
+
+        # Coercion now merges env vars with SCHEMA defaults, so pool_size/timeout
+        # are always available even when only HTM_DATABASE__URL is set
+        {
+          adapter: 'postgresql',
+          host: uri.host,
+          port: uri.port || 5432,
+          database: uri.path&.sub(%r{^/}, ''),
+          username: uri.user,
+          password: uri.password,
+          pool: database.pool_size.to_i,
+          timeout: database.timeout.to_i,
+          sslmode: database.sslmode,
+          encoding: 'unicode',
+          prepared_statements: false,
+          advisory_locks: false
+        }.compact
+      end
+
+      def database_configured?
+        url = database_url
+        (url && !url.empty?) || (database.name && !database.name.empty?)
+      end
+
+      # Database convenience methods
+      def database_url
+        url = database.url
+        return url if url && !url.empty?
+
+        build_database_url
+      end
+
+      # Validate that database is configured for the current environment
+      #
+      # @raise [HTM::ConfigurationError] if database is not configured
+      # @return [true] if database is configured
+      def validate_database!
+        validate_environment!
+
+        unless database_configured?
+          raise HTM::ConfigurationError,
+            "No database configured for environment '#{environment}'. " \
+            "Set HTM_DATABASE__URL or HTM_DATABASE__NAME, " \
+            "or add database.name to the '#{environment}:' section in your config."
+        end
+
+        true
+      end
+
+      # ==========================================================================
+      # Database Naming Convention
+      # ==========================================================================
+      #
+      # Database names MUST follow the convention: {service_name}_{environment}
+      #
+      # Examples:
+      #   - htm_development
+      #   - htm_test
+      #   - htm_production
+      #   - payroll_development
+      #   - payroll_test
+      #
+      # This ensures:
+      #   1. Database names are predictable and self-documenting
+      #   2. Environment mismatches are impossible (exact match required)
+      #   3. Service isolation (can't accidentally use another app's database)
+      #
+      # ==========================================================================
+
+      # Returns the expected database name based on service.name and environment
+      #
+      # @return [String] expected database name in format "{service_name}_{environment}"
+      #
+      # @example
+      #   config.service.name = "htm"
+      #   HTM_ENV = "test"
+      #   config.expected_database_name  # => "htm_test"
+      #
+      def expected_database_name
+        "#{service_name}_#{environment}"
+      end
+
+      # Extract the actual database name from URL or config
+      #
+      # @return [String, nil] the database name
+      def actual_database_name
+        url = database&.url
+        if url && !url.empty?
+          # Parse database name from URL: postgresql://user@host:port/dbname
+          uri = URI.parse(url) rescue nil
+          return uri&.path&.sub(%r{^/}, '')
+        end
+
+        database&.name
+      end
+
+      # Validate that the database name follows the naming convention
+      #
+      # Database names must be: {service_name}_{environment}
+      #
+      # @raise [HTM::ConfigurationError] if database name doesn't match expected
+      # @return [true] if database name is valid
+      #
+      # @example Valid configurations
+      #   HTM_ENV=test, service.name=htm, database=htm_test        # OK
+      #   HTM_ENV=production, service.name=payroll, database=payroll_production  # OK
+      #
+      # @example Invalid configurations (will raise)
+      #   HTM_ENV=test, service.name=htm, database=htm_production  # Wrong environment
+      #   HTM_ENV=test, service.name=htm, database=payroll_test    # Wrong service
+      #   HTM_ENV=test, service.name=htm, database=mydb            # Wrong format
+      #
+      def validate_database_name!
+        actual = actual_database_name
+        expected = expected_database_name
+
+        return true if actual == expected
+
+        raise HTM::ConfigurationError,
+          "Database name '#{actual}' does not match expected '#{expected}'.\n" \
+          "Database names must follow the convention: {service_name}_{environment}\n" \
+          "  Service name: #{service_name}\n" \
+          "  Environment:  #{environment}\n" \
+          "  Expected:     #{expected}\n" \
+          "  Actual:       #{actual}\n\n" \
+          "Either:\n" \
+          "  - Set HTM_DATABASE__URL to point to '#{expected}'\n" \
+          "  - Set HTM_DATABASE__NAME=#{expected}\n" \
+          "  - Change HTM_ENV to match the database suffix"
+      end
+
+      # Check if the database name matches the expected convention
+      #
+      # @return [Boolean] true if database name matches expected
+      def valid_database_name?
+        actual_database_name == expected_database_name
+      end
+
+      # ==========================================================================
+      # Database URL/Components Parsing
+      # ==========================================================================
+
+      # Parse database URL into component hash
+      #
+      # @return [Hash, nil] parsed components or nil if no URL
+      def parse_database_url
+        url = database&.url
+        return nil if url.nil? || url.empty?
+
+        uri = URI.parse(url) rescue nil
+        return nil unless uri
+
+        # Parse query string for sslmode
+        query_params = URI.decode_www_form(uri.query || '').to_h
+        sslmode = query_params['sslmode']
+
+        {
+          host: uri.host,
+          port: uri.port,
+          name: uri.path&.sub(%r{^/}, ''),
+          user: uri.user,
+          password: uri.password,
+          sslmode: sslmode
+        }.compact
+      end
+
+      private
+
+      def build_database_url
+        return nil unless database.name && !database.name.empty?
+
+        auth = if database.user && !database.user.empty?
+          database.password && !database.password.empty? ? "#{database.user}:#{database.password}@" : "#{database.user}@"
+        else
+          ''
+        end
+
+        url = "postgresql://#{auth}#{database.host}:#{database.port}/#{database.name}"
+
+        # Add sslmode as query parameter if set
+        if database.sslmode && !database.sslmode.empty?
+          url += "?sslmode=#{database.sslmode}"
+        end
+
+        url
+      end
+
+      # ==========================================================================
+      # Database Configuration Reconciliation
+      # ==========================================================================
+      #
+      # Ensures database.url and database.* components are synchronized:
+      #
+      # 1. If database.url exists:
+      #    - Extract all components from the URL
+      #    - For each component: if config has a different value → ERROR
+      #    - For each component: if config is missing → populate from URL
+      #
+      # 2. If database.url is missing but components exist:
+      #    - Verify minimum required components (at least database.name)
+      #    - Build and set database.url from components
+      #    - If insufficient components → ERROR
+      #
+      # This runs automatically at config load time via on_load callback.
+      #
+      # ==========================================================================
+
+      def reconcile_database_config
+        url = database&.url
+        has_url = url && !url.empty?
+
+        if has_url
+          reconcile_from_url
+        else
+          reconcile_from_components
+        end
+      end
+
+      def reconcile_from_url
+        url_components = parse_database_url
+        return unless url_components
+
+        # URL is the source of truth - populate all components from it
+        # This overwrites any values from config files (they're just defaults)
+        %i[host port name user password sslmode].each do |component|
+          url_value = url_components[component]
+          next if url_value.nil?
+
+          database.send("#{component}=", url_value)
+        end
+      end
+
+      def reconcile_from_components
+        # Check what components we have
+        name = database&.name
+        has_name = name && !name.empty?
+
+        # If no database config at all, that's fine - might not need database
+        # Just return without error; validate_database! will catch if needed later
+        return unless has_name || has_any_database_component?
+
+        # If name is missing, auto-generate from service.name and environment
+        # Format: {service_name}_{environment} (e.g., "htm_development")
+        unless has_name
+          database.name = expected_database_name
+        end
+
+        # Use defaults for host/port if not set
+        database.host = 'localhost' if database.host.nil? || database.host.empty?
+        database.port = 5432 if database.port.nil?
+
+        # Build and set the URL
+        database.url = build_database_url
+      end
+
+      def has_any_database_component?
+        %i[host port user password].any? do |comp|
+          val = database.send(comp)
+          next false if val.nil?
+          next false if val.respond_to?(:empty?) && val.empty?
+          # Skip defaults
+          next false if comp == :host && val == 'localhost'
+          next false if comp == :port && val == 5432
+          true
+        end
+      end
+    end
+  end
+end

data/lib/htm/config/defaults.yml

@@ -91,8 +91,9 @@ defaults:
       RULES:
       1. Each concept belongs to ONE path only
       2. Use lowercase, hyphens for multi-word terms
-      3. Return 2-5 tags that best classify this text
-      4. Match existing taxonomy paths when applicable
+      3. ALWAYS use SINGULAR forms, never plurals (user NOT users, framework NOT frameworks, model NOT models)
+      4. Return 2-5 tags that best classify this text
+      5. Match existing taxonomy paths when applicable
 
       TEXT: %{text}
 
@@ -113,26 +114,53 @@ defaults:
     model: gemma3:latest
     timeout: 180
     enabled: true
+    min_length: 10
+    max_length: 1000
+    min_words: 5
 
     system_prompt: |
-      You are an atomic fact extraction system. Your goal is maximum decomposition.
-      Break every statement into its smallest possible factual units.
+      You are an atomic fact extraction system. Extract factual propositions from text.
       Output ONLY propositions, one per line, prefixed with a dash (-).
+      NEVER ask for clarification or more text - extract what you can from the given text.
+      If the text contains no extractable facts, output nothing.
 
     user_prompt_template: |
-      Extract all ATOMIC factual propositions from the following text.
+      Extract ATOMIC factual propositions from the following text.
 
-      An atomic proposition expresses exactly ONE relationship or fact.
+      An atomic proposition:
+      - Expresses exactly ONE fact or relationship
+      - Is understandable WITHOUT any additional context
+      - Uses FULL NAMES, never pronouns (he, she, it, they, this, that)
+      - Contains at least 5 words for sufficient context
 
-      Rules:
+      PRONOUN REPLACEMENT EXAMPLES:
+      BAD:  "It is important to know the enemy."
+      GOOD: "Knowing the enemy is important for spiritual warfare."
+
+      BAD:  "They support thousands of users."
+      GOOD: "The Every.to products support thousands of daily users."
+
+      BAD:  "This results in a calculation."
+      GOOD: "The age calculation depends on whether the birthday has occurred."
+
+      CONTEXT ENRICHMENT EXAMPLES:
+      BAD:  "Wiring costs $1,000."
+      GOOD: "Solar panel wiring costs $1,000 for the Oklahoma barndominium project."
+
+      BAD:  "The driveway is gravel."
+      GOOD: "Dewayne's barndominium driveway is gravel."
+
+      RULES:
       1. Split compound statements into separate atomic facts
       2. Each proposition = exactly one fact
-      3. Use full names, never pronouns
-      4. Make each proposition understandable in isolation
+      3. Replace ALL pronouns with the actual names/nouns they refer to
+      4. Include enough context to understand the fact in isolation
+      5. If unsure what a pronoun refers to, omit that proposition
 
       TEXT: %{text}
 
       Return ONLY atomic propositions, one per line. Use a dash (-) prefix for each.
+      If you cannot extract any facts, return nothing. Do NOT ask for more text.
 
   # ---------------------------------------------------------------------------
   # Chunking Configuration (for file loading)