htm 0.0.18 → 0.0.30

data/lib/htm/database.rb

@@ -6,7 +6,7 @@ require 'set'
 
 class HTM
   # Database setup and configuration for HTM
-  # Handles schema creation and database initialization
+  # Handles schema creation and database initialization using Sequel
   class Database
     class << self
       # Set up the HTM database schema
@@ -17,19 +17,22 @@ class HTM
       # @return [void]
       #
       def setup(db_url = nil, run_migrations: true, dump_schema: false)
-        require 'active_record'
-        require_relative 'active_record_config'
+        require 'sequel'
+        require_relative 'sequel_config'
 
-        # Establish ActiveRecord connection
-        HTM::ActiveRecordConfig.establish_connection!
+        # Establish Sequel connection (don't load models yet - tables may not exist)
+        HTM::SequelConfig.establish_connection!(load_models: false)
 
-        # Run migrations using ActiveRecord
+        # Run migrations using Sequel
         if run_migrations
-          puts "Running ActiveRecord migrations..."
-          run_activerecord_migrations
+          puts "Running Sequel migrations..."
+          run_sequel_migrations
         end
 
-        puts "✓ HTM database schema created successfully"
+        # Now that tables exist, load models
+        HTM::SequelConfig.ensure_models_loaded!
+
+        puts "HTM database schema created successfully"
 
         # Optionally dump schema
         if dump_schema
@@ -44,15 +47,18 @@ class HTM
       # @return [void]
       #
       def migrate(db_url = nil)
-        require 'active_record'
-        require_relative 'active_record_config'
+        require 'sequel'
+        require_relative 'sequel_config'
+
+        # Establish Sequel connection (don't load models - tables may not exist)
+        HTM::SequelConfig.establish_connection!(load_models: false)
 
-        # Establish ActiveRecord connection
-        HTM::ActiveRecordConfig.establish_connection!
+        run_sequel_migrations
 
-        run_activerecord_migrations
+        # Load models now that tables exist
+        HTM::SequelConfig.ensure_models_loaded!
 
-        puts "✓ Database migrations completed"
+        puts "Database migrations completed"
       end
 
       # Show migration status
@@ -61,26 +67,26 @@ class HTM
       # @return [void]
       #
       def migration_status(db_url = nil)
-        require 'active_record'
-        require_relative 'active_record_config'
+        require 'sequel'
+        require_relative 'sequel_config'
 
-        # Establish ActiveRecord connection
-        HTM::ActiveRecordConfig.establish_connection!
+        # Establish Sequel connection (don't load models - we just need the DB)
+        HTM::SequelConfig.establish_connection!(load_models: false)
 
         migrations_path = File.expand_path('../../db/migrate', __dir__)
 
         # Get available migrations from files
         available_migrations = Dir.glob(File.join(migrations_path, '*.rb')).map do |file|
           {
-            version: File.basename(file).split('_').first,
+            version: File.basename(file).split('_').first.to_i,
             name: File.basename(file, '.rb')
           }
         end.sort_by { |m| m[:version] }
 
         # Get applied migrations from database
         applied_versions = begin
-          ActiveRecord::Base.connection.select_values('SELECT version FROM schema_migrations ORDER BY version')
-        rescue ActiveRecord::StatementInvalid
+          HTM.db[:schema_migrations].select_map(:version).map(&:to_i)
+        rescue Sequel::DatabaseError
           []
         end
 
@@ -92,7 +98,7 @@ class HTM
         else
           available_migrations.each do |migration|
             status = applied_versions.include?(migration[:version])
-            status_mark = status ? "✓" : "✗"
+            status_mark = status ? "[x]" : "[ ]"
 
             puts "#{status_mark} #{migration[:name]}"
           end
@@ -125,38 +131,36 @@ class HTM
         tables.each do |table|
           begin
             conn.exec("DROP TABLE IF EXISTS #{table} CASCADE")
-            puts "  ✓ Dropped #{table}"
+            puts "  Dropped #{table}"
           rescue PG::Error => e
-            puts "  ✗ Error dropping #{table}: #{e.message}"
+            puts "  Error dropping #{table}: #{e.message}"
           end
         end
 
         # Drop functions and triggers
         begin
           conn.exec("DROP FUNCTION IF EXISTS extract_ontology_topics() CASCADE")
-          puts "  ✓ Dropped ontology functions and triggers"
+          puts "  Dropped ontology functions and triggers"
         rescue PG::Error => e
-          puts "  ✗ Error dropping functions: #{e.message}"
+          puts "  Error dropping functions: #{e.message}"
         end
 
         # Drop views
         begin
           conn.exec("DROP VIEW IF EXISTS ontology_structure CASCADE")
           conn.exec("DROP VIEW IF EXISTS topic_relationships CASCADE")
-          puts "  ✓ Dropped ontology views"
+          puts "  Dropped ontology views"
         rescue PG::Error => e
-          puts "  ✗ Error dropping views: #{e.message}"
+          puts "  Error dropping views: #{e.message}"
         end
 
         conn.close
-        puts "✓ All HTM tables dropped"
+        puts "All HTM tables dropped"
       end
 
       # Seed database with sample data
       #
       # Loads and executes db/seeds.rb file following Rails conventions.
-      # All seeding logic is contained in db/seeds.rb and reads data
-      # from markdown files in db/seed_data/ directory.
       #
       # @param db_url [String] Database connection URL (uses ENV['HTM_DATABASE__URL'] if not provided)
       # @return [void]
@@ -165,7 +169,7 @@ class HTM
         seeds_file = File.expand_path('../../db/seeds.rb', __dir__)
 
         unless File.exist?(seeds_file)
-          puts "✗ Error: Seeds file not found at #{seeds_file}"
+          puts "Error: Seeds file not found at #{seeds_file}"
           puts "  Please create db/seeds.rb with your seeding logic"
           exit 1
         end
@@ -189,12 +193,6 @@ class HTM
 
         puts "Dumping schema to #{schema_file}..."
 
-        # Build pg_dump command
-        # --schema-only: only dump schema, not data
-        # --no-owner: don't set ownership
-        # --no-privileges: don't dump access privileges
-        # --no-tablespaces: don't dump tablespace assignments
-        # --exclude-schema=_timescaledb_*: exclude TimescaleDB internal schemas
         env = {
           'PGPASSWORD' => config[:password]
         }
@@ -214,23 +212,19 @@ class HTM
           '-d', config[:dbname]
         ]
 
-        # Execute pg_dump and capture output
         require 'open3'
         stdout, stderr, status = Open3.capture3(env, *cmd)
 
         unless status.success?
-          puts "✗ Error dumping schema:"
+          puts "Error dumping schema:"
           puts stderr
           exit 1
         end
 
-        # Clean up the output
         cleaned_schema = clean_schema_dump(stdout)
-
-        # Write to file
         File.write(schema_file, cleaned_schema)
 
-        puts "✓ Schema dumped successfully to #{schema_file}"
+        puts "Schema dumped successfully to #{schema_file}"
         puts "  Size: #{File.size(schema_file)} bytes"
       end
 
@@ -248,14 +242,13 @@ class HTM
         schema_file = File.expand_path('../../db/schema.sql', __dir__)
 
         unless File.exist?(schema_file)
-          puts "✗ Schema file not found: #{schema_file}"
+          puts "Schema file not found: #{schema_file}"
           puts "  Run 'rake htm:db:schema:dump' first to create it"
           exit 1
         end
 
         puts "Loading schema from #{schema_file}..."
 
-        # Build psql command
         env = {
           'PGPASSWORD' => config[:password]
         }
@@ -270,35 +263,26 @@ class HTM
           '--quiet'
         ]
 
-        # Execute psql
         require 'open3'
         stdout, stderr, status = Open3.capture3(env, *cmd)
 
         unless status.success?
-          puts "✗ Error loading schema:"
+          puts "Error loading schema:"
           puts stderr
           exit 1
         end
 
-        puts "✓ Schema loaded successfully"
+        puts "Schema loaded successfully"
       end
 
       # Generate database documentation using tbls
       #
-      # Uses .tbls.yml configuration file for output directory and settings.
-      # Creates comprehensive database documentation including:
-      # - Entity-relationship diagrams
-      # - Table schemas with comments
-      # - Index information
-      # - Relationship diagrams
-      #
       # @param db_url [String] Database connection URL (uses ENV['HTM_DATABASE__URL'] if not provided)
       # @return [void]
       #
       def generate_docs(db_url = nil)
-        # Check if tbls is installed
         unless system('which tbls > /dev/null 2>&1')
-          puts "✗ Error: 'tbls' is not installed"
+          puts "Error: 'tbls' is not installed"
           puts ""
           puts "Install tbls:"
           puts "  brew install k1LoW/tap/tbls"
@@ -309,20 +293,17 @@ class HTM
           exit 1
         end
 
-        # Find the project root (where .tbls.yml should be)
         project_root = File.expand_path('../..', __dir__)
         tbls_config = File.join(project_root, '.tbls.yml')
 
         unless File.exist?(tbls_config)
-          puts "✗ Error: .tbls.yml not found at #{tbls_config}"
+          puts "Error: .tbls.yml not found at #{tbls_config}"
           exit 1
         end
 
-        # Get database URL
         dsn = db_url || ENV['HTM_DATABASE__URL']
         raise "Database configuration not found. Set HTM_DATABASE__URL environment variable." unless dsn
 
-        # Ensure sslmode is set for local development (tbls requires it)
         unless dsn.include?('sslmode=')
           separator = dsn.include?('?') ? '&' : '?'
           dsn = "#{dsn}#{separator}sslmode=disable"
@@ -330,15 +311,13 @@ class HTM
 
         puts "Generating database documentation using #{tbls_config}..."
 
-        # Run tbls doc command with config file and DSN override
-        # The --dsn flag overrides the dsn in .tbls.yml but other settings are preserved
         require 'open3'
         cmd = ['tbls', 'doc', '--config', tbls_config, '--dsn', dsn, '--force']
 
         stdout, stderr, status = Open3.capture3(*cmd)
 
         unless status.success?
-          puts "✗ Error generating documentation:"
+          puts "Error generating documentation:"
           puts stderr
           puts stdout
           exit 1
@@ -346,9 +325,8 @@ class HTM
 
         puts stdout if stdout && !stdout.empty?
 
-        # Read docPath from config to show correct output location
-        doc_path = 'docs/database'  # default from .tbls.yml
-        puts "✓ Database documentation generated successfully"
+        doc_path = 'docs/database'
+        puts "Database documentation generated successfully"
         puts ""
         puts "Documentation files:"
         puts "  #{doc_path}/README.md       - Main documentation"
@@ -373,7 +351,6 @@ class HTM
         puts "\nHTM Database Information (#{HTM.env})"
         puts "=" * 80
 
-        # Connection info
         puts "\nConnection:"
         puts "  Environment: #{HTM.env}"
         puts "  Host: #{config[:host]}"
@@ -381,19 +358,16 @@ class HTM
         puts "  Database: #{config[:dbname]}"
         puts "  User: #{config[:user]}"
 
-        # PostgreSQL version
         version = conn.exec("SELECT version()").first['version']
         puts "\nPostgreSQL Version:"
         puts "  #{version.split(',').first}"
 
-        # Extensions
         puts "\nExtensions:"
         extensions = conn.exec("SELECT extname, extversion FROM pg_extension ORDER BY extname").to_a
         extensions.each do |ext|
           puts "  #{ext['extname']} (#{ext['extversion']})"
         end
 
-        # Table info
         puts "\nHTM Tables:"
         tables = ['nodes', 'node_tags', 'tags', 'robots', 'robot_nodes', 'file_sources', 'schema_migrations']
         tables.each do |table|
@@ -405,7 +379,6 @@ class HTM
           end
         end
 
-        # Database size
         db_size = conn.exec(
           "SELECT pg_size_pretty(pg_database_size($1)) AS size",
           [config[:dbname]]
@@ -427,7 +400,6 @@ class HTM
 
         uri = URI.parse(url)
 
-        # Validate URL format
         unless uri.scheme&.match?(/\Apostgres(?:ql)?\z/i)
           raise ArgumentError, "Invalid database URL scheme: #{uri.scheme}. Expected 'postgresql' or 'postgres'."
         end
@@ -436,7 +408,7 @@ class HTM
           raise ArgumentError, "Database URL must include a host"
         end
 
-        dbname = uri.path&.slice(1..-1)  # Remove leading /
+        dbname = uri.path&.slice(1..-1)
         if dbname.nil? || dbname.empty?
           raise ArgumentError, "Database URL must include a database name (path segment)"
         end
@@ -482,16 +454,16 @@ class HTM
         htm_config = HTM.config
 
         if htm_config.database_configured?
-          ar_config = htm_config.database_config
+          db_config = htm_config.database_config
 
-          # Convert ActiveRecord config keys to PG-style keys
+          # Convert to PG-style keys
           {
-            host: ar_config[:host],
-            port: ar_config[:port],
-            dbname: ar_config[:database],
-            user: ar_config[:username],
-            password: ar_config[:password],
-            sslmode: ar_config[:sslmode] || 'prefer'
+            host: db_config[:host],
+            port: db_config[:port],
+            dbname: db_config[:database],
+            user: db_config[:username],
+            password: db_config[:password],
+            sslmode: db_config[:sslmode] || 'prefer'
           }
         elsif ENV['HTM_DATABASE__URL']
           parse_connection_url(ENV['HTM_DATABASE__URL'])
@@ -506,38 +478,46 @@ class HTM
         # Check pgvector
         pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
         if pgvector
-          puts "✓ pgvector version: #{pgvector['extversion']}"
+          puts "pgvector version: #{pgvector['extversion']}"
         else
-          puts "⚠ Warning: pgvector extension not found"
+          puts "Warning: pgvector extension not found"
         end
 
         # Check pg_trgm
         pg_trgm = conn.exec("SELECT extversion FROM pg_extension WHERE extname='pg_trgm'").first
         if pg_trgm
-          puts "✓ pg_trgm version: #{pg_trgm['extversion']}"
+          puts "pg_trgm version: #{pg_trgm['extversion']}"
+        else
+          puts "Warning: pg_trgm extension not found"
+        end
+
+        # Check pg_search (BM25 full-text search)
+        pg_search = conn.exec("SELECT extversion FROM pg_extension WHERE extname='pg_search'").first
+        if pg_search
+          puts "pg_search version: #{pg_search['extversion']}"
         else
-          puts "⚠ Warning: pg_trgm extension not found"
+          puts "Warning: pg_search extension not found"
         end
       end
 
-      # Run ActiveRecord migrations from db/migrate/
+      # Run Sequel migrations from db/migrate/
       #
       # @return [void]
       #
-      def run_activerecord_migrations
+      def run_sequel_migrations
         migrations_path = File.expand_path('../../db/migrate', __dir__)
 
         unless Dir.exist?(migrations_path)
-          puts "⚠ No migrations directory found at #{migrations_path}"
+          puts "No migrations directory found at #{migrations_path}"
           return
         end
 
-        conn = ActiveRecord::Base.connection
+        db = HTM.db
 
         # Create schema_migrations table if it doesn't exist
-        unless conn.table_exists?('schema_migrations')
-          conn.create_table(:schema_migrations, id: false) do |t|
-            t.string :version, null: false, primary_key: true
+        unless db.table_exists?(:schema_migrations)
+          db.create_table(:schema_migrations) do
+            String :version, primary_key: true, null: false
           end
         end
 
@@ -550,17 +530,13 @@ class HTM
           version = File.basename(file).split('_').first
           name = File.basename(file, '.rb')
 
-          # Check if already run (use parameterized query to prevent SQL injection)
-          already_run = conn.select_value(
-            ActiveRecord::Base.sanitize_sql_array(
-              ["SELECT COUNT(*) FROM schema_migrations WHERE version = ?", version]
-            )
-          ).to_i > 0
+          # Check if already run
+          already_run = db[:schema_migrations].where(version: version).count > 0
 
           if already_run
-            puts "  ✓ #{name} (already migrated)"
+            puts "  [x] #{name} (already migrated)"
           else
-            puts "  → Running #{name}..."
+            puts "  --> Running #{name}..."
             require file
 
             # Get the migration class
@@ -568,21 +544,17 @@ class HTM
             migration_class = Object.const_get(class_name)
 
             # Run the migration
-            migration = migration_class.new
-            migration.migrate(:up)
+            migration = migration_class.new(db)
+            migration.up
 
-            # Record in schema_migrations (use parameterized query to prevent SQL injection)
-            conn.execute(
-              ActiveRecord::Base.sanitize_sql_array(
-                ["INSERT INTO schema_migrations (version) VALUES (?)", version]
-              )
-            )
+            # Record in schema_migrations
+            db[:schema_migrations].insert(version: version)
 
-            puts "    ✓ Completed"
+            puts "      Completed"
           end
         end
 
-        puts "✓ All migrations completed"
+        puts "All migrations completed"
       end
 
       # Clean up pg_dump output to make it more readable
@@ -594,18 +566,15 @@ class HTM
         lines = schema_dump.split("\n")
         cleaned = []
 
-        # Add header
         cleaned << "-- HTM Database Schema"
         cleaned << "-- Auto-generated from database using pg_dump"
         cleaned << "-- DO NOT EDIT THIS FILE MANUALLY"
         cleaned << "-- Run 'rake htm:db:schema:dump' to regenerate"
         cleaned << ""
 
-        # Skip pg_dump header comments
         skip_until_content = true
 
         lines.each do |line|
-          # Skip header comments
           if skip_until_content
             if line =~ /^(SET|CREATE|ALTER|--\s*Name:|COMMENT)/
               skip_until_content = false
@@ -614,29 +583,18 @@ class HTM
             end
           end
 
-          # Skip SET commands (session-specific settings)
           next if line =~ /^SET /
-
-          # Skip SELECT pg_catalog.set_config
           next if line =~ /^SELECT pg_catalog\.set_config/
-
-          # Skip extension comments (we keep extension creation)
           next if line =~ /^COMMENT ON EXTENSION/
 
-          # Keep everything else
           cleaned << line
         end
 
-        # Remove multiple blank lines
         result = cleaned.join("\n")
         result.gsub!(/\n{3,}/, "\n\n")
 
         result
       end
-
-      # Old methods removed - now using ActiveRecord migrations
-      # def run_schema(conn) - REMOVED
-      # def run_migrations_if_needed(conn) - REMOVED (see run_activerecord_migrations above)
     end
   end
 end

data/lib/htm/errors.rb

@@ -44,6 +44,20 @@ class HTM
   #
   class ValidationError < Error; end
 
+  # Raised when configuration is invalid or incomplete
+  #
+  # Common causes:
+  # - Invalid HTM_ENV value (not defined in config)
+  # - Missing database configuration for environment
+  # - HTM_CONF pointing to non-existent file
+  # - Invalid YAML syntax in config file
+  #
+  # @example
+  #   HTM_ENV=staginr rake htm:db:create  # => raises ConfigurationError
+  #   HTM_ENV=defaults rake htm:db:setup  # => raises ConfigurationError
+  #
+  class ConfigurationError < Error; end
+
   # Raised when system resources are exhausted
   #
   # Common causes:

data/lib/htm/integrations/sinatra.rb

@@ -100,6 +100,7 @@ class HTM
     # Rack middleware for HTM connection management
     #
     # Ensures database connections are properly managed across requests.
+    # With Sequel's fiber-safe connection pooling, this is largely automatic.
     #
     # @example Use in Sinatra app
     #   class MyApp < Sinatra::Base
@@ -107,70 +108,38 @@ class HTM
     #   end
     #
     class Middleware
-      # Class-level storage for connection configuration (shared across threads)
-      @@db_config = nil
-      @@config_mutex = Mutex.new
-
       def initialize(app, options = {})
         @app = app
         @options = options
       end
 
       def call(env)
-        # Ensure connection is available in this thread
-        ensure_thread_connection!
+        # Ensure connection is available
+        ensure_connection!
 
         # Process request
         status, headers, body = @app.call(env)
 
         # Return response
         [status, headers, body]
-      ensure
-        # Return connections to pool after request completes
-        if defined?(ActiveRecord::Base) && ActiveRecord::Base.respond_to?(:connection_handler)
-          ActiveRecord::Base.connection_handler.clear_active_connections!
-        end
       end
 
       # Store the connection config at startup (called from register_htm)
       def self.store_config!
-        @@config_mutex.synchronize do
-          return if @@db_config
-
-          @@db_config = HTM::ActiveRecordConfig.load_database_config
-        end
+        # With Sequel, connection is established globally via HTM::SequelConfig
+        # No additional per-request config storage needed
       end
 
       private
 
-      def ensure_thread_connection!
-        # Check if connection pool exists and has an active connection
-        pool_exists = begin
-          ActiveRecord::Base.connection_pool
-          true
-        rescue ActiveRecord::ConnectionNotDefined
-          false
-        end
-
-        if pool_exists
-          return if ActiveRecord::Base.connection_pool.active_connection?
-        end
-
-        # Re-establish connection using stored config
-        if @@db_config
-          ActiveRecord::Base.establish_connection(@@db_config)
-        else
-          raise "HTM database config not stored - call register_htm at app startup"
-        end
-      rescue ActiveRecord::ConnectionNotDefined, ActiveRecord::ConnectionNotEstablished
-        # Pool doesn't exist, establish connection
-        if @@db_config
-          ActiveRecord::Base.establish_connection(@@db_config)
-        else
-          raise "HTM database config not stored - call register_htm at app startup"
+      def ensure_connection!
+        # Sequel handles connection pooling automatically
+        # Just verify the connection is available
+        unless HTM.db
+          HTM::SequelConfig.establish_connection!
         end
       rescue StandardError => e
-        HTM.logger.error "Failed to ensure thread connection: #{e.class} - #{e.message}"
+        HTM.logger.error "Failed to ensure connection: #{e.class} - #{e.message}"
         raise
       end
     end
@@ -212,10 +181,10 @@ module ::Sinatra
         end
       end
 
-      # Store database config for thread-safe access and establish initial connection
+      # Establish initial connection (Sequel handles pooling automatically)
       begin
         HTM::Sinatra::Middleware.store_config!
-        HTM::ActiveRecordConfig.establish_connection!
+        HTM::SequelConfig.establish_connection!
         HTM.logger.info "HTM database connection established"
       rescue StandardError => e
         HTM.logger.error "Failed to establish HTM database connection: #{e.message}"