htm 0.0.1

data/lib/htm/database.rb

@@ -0,0 +1,594 @@
+# frozen_string_literal: true
+
+require 'pg'
+require 'uri'
+require 'set'
+
+class HTM
+  # Database setup and configuration for HTM
+  # Handles schema creation and database initialization
+  class Database
+    class << self
+      # Set up the HTM database schema
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @param run_migrations [Boolean] Whether to run migrations (default: true)
+      # @param dump_schema [Boolean] Whether to dump schema to db/schema.sql after setup (default: false)
+      # @return [void]
+      #
+      def setup(db_url = nil, run_migrations: true, dump_schema: false)
+        require 'active_record'
+        require_relative 'active_record_config'
+
+        # Establish ActiveRecord connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        # Run migrations using ActiveRecord
+        if run_migrations
+          puts "Running ActiveRecord migrations..."
+          run_activerecord_migrations
+        end
+
+        puts "✓ HTM database schema created successfully"
+
+        # Optionally dump schema
+        if dump_schema
+          puts ""
+          self.dump_schema(db_url)
+        end
+      end
+
+      # Run pending database migrations
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def migrate(db_url = nil)
+        require 'active_record'
+        require_relative 'active_record_config'
+
+        # Establish ActiveRecord connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        run_activerecord_migrations
+
+        puts "✓ Database migrations completed"
+      end
+
+      # Show migration status
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def migration_status(db_url = nil)
+        require 'active_record'
+        require_relative 'active_record_config'
+
+        # Establish ActiveRecord connection
+        HTM::ActiveRecordConfig.establish_connection!
+
+        migrations_path = File.expand_path('../../db/migrate', __dir__)
+
+        # Get available migrations from files
+        available_migrations = Dir.glob(File.join(migrations_path, '*.rb')).map do |file|
+          {
+            version: File.basename(file).split('_').first,
+            name: File.basename(file, '.rb')
+          }
+        end.sort_by { |m| m[:version] }
+
+        # Get applied migrations from database
+        applied_versions = begin
+          ActiveRecord::Base.connection.select_values('SELECT version FROM schema_migrations ORDER BY version')
+        rescue ActiveRecord::StatementInvalid
+          []
+        end
+
+        puts "\nMigration Status"
+        puts "=" * 100
+
+        if available_migrations.empty?
+          puts "No migration files found in db/migrate/"
+        else
+          available_migrations.each do |migration|
+            status = applied_versions.include?(migration[:version])
+            status_mark = status ? "✓" : "✗"
+
+            puts "#{status_mark} #{migration[:name]}"
+          end
+        end
+
+        applied_count = applied_versions.length
+        pending_count = available_migrations.length - applied_count
+
+        puts "\nSummary: #{applied_count} applied, #{pending_count} pending"
+        puts "=" * 100
+      end
+
+      # Drop all HTM tables
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def drop(db_url = nil)
+        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
+        raise "Database configuration not found" unless config
+
+        conn = PG.connect(config)
+
+        tables = ['nodes', 'node_tags', 'tags', 'robots', 'operations_log', 'schema_migrations']
+
+        puts "Dropping HTM tables..."
+        tables.each do |table|
+          begin
+            conn.exec("DROP TABLE IF EXISTS #{table} CASCADE")
+            puts "  ✓ Dropped #{table}"
+          rescue PG::Error => e
+            puts "  ✗ Error dropping #{table}: #{e.message}"
+          end
+        end
+
+        # Drop functions and triggers
+        begin
+          conn.exec("DROP FUNCTION IF EXISTS extract_ontology_topics() CASCADE")
+          puts "  ✓ Dropped ontology functions and triggers"
+        rescue PG::Error => e
+          puts "  ✗ Error dropping functions: #{e.message}"
+        end
+
+        # Drop views
+        begin
+          conn.exec("DROP VIEW IF EXISTS ontology_structure CASCADE")
+          conn.exec("DROP VIEW IF EXISTS topic_relationships CASCADE")
+          puts "  ✓ Dropped ontology views"
+        rescue PG::Error => e
+          puts "  ✗ Error dropping views: #{e.message}"
+        end
+
+        conn.close
+        puts "✓ All HTM tables dropped"
+      end
+
+      # Seed database with sample data
+      #
+      # Loads and executes db/seeds.rb file following Rails conventions.
+      # All seeding logic is contained in db/seeds.rb and reads data
+      # from markdown files in db/seed_data/ directory.
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def seed(db_url = nil)
+        seeds_file = File.expand_path('../../db/seeds.rb', __dir__)
+
+        unless File.exist?(seeds_file)
+          puts "✗ Error: Seeds file not found at #{seeds_file}"
+          puts "  Please create db/seeds.rb with your seeding logic"
+          exit 1
+        end
+
+        # Load and execute seeds.rb
+        load seeds_file
+      end
+
+      # Dump current database schema to db/schema.sql
+      #
+      # Uses pg_dump to create a clean SQL schema file without data
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def dump_schema(db_url = nil)
+        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
+        raise "Database configuration not found" unless config
+
+        schema_file = File.expand_path('../../db/schema.sql', __dir__)
+
+        puts "Dumping schema to #{schema_file}..."
+
+        # Build pg_dump command
+        # --schema-only: only dump schema, not data
+        # --no-owner: don't set ownership
+        # --no-privileges: don't dump access privileges
+        # --no-tablespaces: don't dump tablespace assignments
+        # --exclude-schema=_timescaledb_*: exclude TimescaleDB internal schemas
+        env = {
+          'PGPASSWORD' => config[:password]
+        }
+
+        cmd = [
+          'pg_dump',
+          '--schema-only',
+          '--no-owner',
+          '--no-privileges',
+          '--no-tablespaces',
+          '--exclude-schema=_timescaledb_*',
+          '--exclude-schema=information_schema',
+          '--exclude-schema=pg_catalog',
+          '-h', config[:host],
+          '-p', config[:port].to_s,
+          '-U', config[:user],
+          '-d', config[:dbname]
+        ]
+
+        # Execute pg_dump and capture output
+        require 'open3'
+        stdout, stderr, status = Open3.capture3(env, *cmd)
+
+        unless status.success?
+          puts "✗ Error dumping schema:"
+          puts stderr
+          exit 1
+        end
+
+        # Clean up the output
+        cleaned_schema = clean_schema_dump(stdout)
+
+        # Write to file
+        File.write(schema_file, cleaned_schema)
+
+        puts "✓ Schema dumped successfully to #{schema_file}"
+        puts "  Size: #{File.size(schema_file)} bytes"
+      end
+
+      # Load schema from db/schema.sql
+      #
+      # Uses psql to load the schema file
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def load_schema(db_url = nil)
+        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
+        raise "Database configuration not found" unless config
+
+        schema_file = File.expand_path('../../db/schema.sql', __dir__)
+
+        unless File.exist?(schema_file)
+          puts "✗ Schema file not found: #{schema_file}"
+          puts "  Run 'rake htm:db:schema:dump' first to create it"
+          exit 1
+        end
+
+        puts "Loading schema from #{schema_file}..."
+
+        # Build psql command
+        env = {
+          'PGPASSWORD' => config[:password]
+        }
+
+        cmd = [
+          'psql',
+          '-h', config[:host],
+          '-p', config[:port].to_s,
+          '-U', config[:user],
+          '-d', config[:dbname],
+          '-f', schema_file,
+          '--quiet'
+        ]
+
+        # Execute psql
+        require 'open3'
+        stdout, stderr, status = Open3.capture3(env, *cmd)
+
+        unless status.success?
+          puts "✗ Error loading schema:"
+          puts stderr
+          exit 1
+        end
+
+        puts "✓ Schema loaded successfully"
+      end
+
+      # Generate database documentation using tbls
+      #
+      # Creates comprehensive database documentation in dbdoc/ directory including:
+      # - Entity-relationship diagrams
+      # - Table schemas with comments
+      # - Index information
+      # - Relationship diagrams
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def generate_docs(db_url = nil)
+        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
+        raise "Database configuration not found" unless config
+
+        dbdoc_dir = File.expand_path('../../dbdoc', __dir__)
+
+        puts "Generating database documentation in #{dbdoc_dir}..."
+
+        # Create dbdoc directory if it doesn't exist
+        Dir.mkdir(dbdoc_dir) unless Dir.exist?(dbdoc_dir)
+
+        # Build PostgreSQL connection string for tbls
+        pg_url = if config[:password]
+          "postgresql://#{config[:user]}:#{config[:password]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
+        else
+          "postgresql://#{config[:user]}@#{config[:host]}:#{config[:port]}/#{config[:dbname]}?sslmode=#{config[:sslmode] || 'prefer'}"
+        end
+
+        # Check if tbls is installed
+        unless system('which tbls > /dev/null 2>&1')
+          puts "✗ Error: 'tbls' is not installed"
+          puts ""
+          puts "Install tbls:"
+          puts "  brew install k1LoW/tap/tbls"
+          puts "  # or"
+          puts "  go install github.com/k1LoW/tbls@latest"
+          puts ""
+          puts "See: https://github.com/k1LoW/tbls"
+          exit 1
+        end
+
+        # Run tbls doc command with --force to allow updates
+        require 'open3'
+        cmd = ['tbls', 'doc', '--force', pg_url, dbdoc_dir]
+
+        stdout, stderr, status = Open3.capture3(*cmd)
+
+        unless status.success?
+          puts "✗ Error generating documentation:"
+          puts stderr
+          puts stdout
+          exit 1
+        end
+
+        puts stdout if stdout && !stdout.empty?
+        puts "✓ Database documentation generated successfully"
+        puts ""
+        puts "Documentation files:"
+        puts "  #{dbdoc_dir}/README.md       - Main documentation"
+        puts "  #{dbdoc_dir}/schema.svg      - ER diagram (if generated)"
+        puts "  #{dbdoc_dir}/*.md            - Individual table documentation"
+        puts ""
+        puts "View documentation:"
+        puts "  open #{dbdoc_dir}/README.md"
+      end
+
+      # Show database info
+      #
+      # @param db_url [String] Database connection URL (uses ENV['HTM_DBURL'] if not provided)
+      # @return [void]
+      #
+      def info(db_url = nil)
+        config = parse_connection_url(db_url || ENV['HTM_DBURL'])
+        raise "Database configuration not found" unless config
+
+        conn = PG.connect(config)
+
+        puts "\nHTM Database Information"
+        puts "=" * 80
+
+        # Connection info
+        puts "\nConnection:"
+        puts "  Host: #{config[:host]}"
+        puts "  Port: #{config[:port]}"
+        puts "  Database: #{config[:dbname]}"
+        puts "  User: #{config[:user]}"
+
+        # PostgreSQL version
+        version = conn.exec("SELECT version()").first['version']
+        puts "\nPostgreSQL Version:"
+        puts "  #{version.split(',').first}"
+
+        # Extensions
+        puts "\nExtensions:"
+        extensions = conn.exec("SELECT extname, extversion FROM pg_extension ORDER BY extname").to_a
+        extensions.each do |ext|
+          puts "  #{ext['extname']} (#{ext['extversion']})"
+        end
+
+        # Table info
+        puts "\nHTM Tables:"
+        tables = ['nodes', 'tags', 'robots', 'operations_log', 'schema_migrations']
+        tables.each do |table|
+          begin
+            count = conn.exec("SELECT COUNT(*) FROM #{table}").first['count']
+            puts "  #{table}: #{count} rows"
+          rescue PG::UndefinedTable
+            puts "  #{table}: not created"
+          end
+        end
+
+        # Database size
+        db_size = conn.exec(
+          "SELECT pg_size_pretty(pg_database_size($1)) AS size",
+          [config[:dbname]]
+        ).first['size']
+        puts "\nDatabase Size: #{db_size}"
+
+        conn.close
+        puts "=" * 80
+      end
+
+      # Parse database connection URL
+      #
+      # @param url [String] Connection URL
+      # @return [Hash, nil] Connection configuration hash
+      #
+      def parse_connection_url(url)
+        return nil unless url
+
+        uri = URI.parse(url)
+        params = URI.decode_www_form(uri.query || '').to_h
+
+        {
+          host: uri.host,
+          port: uri.port,
+          dbname: uri.path[1..-1],  # Remove leading /
+          user: uri.user,
+          password: uri.password,
+          sslmode: params['sslmode'] || 'prefer'
+        }
+      end
+
+      # Build config from individual environment variables
+      #
+      # @return [Hash, nil] Connection configuration hash
+      #
+      def parse_connection_params
+        return nil unless ENV['HTM_DBNAME']
+
+        {
+          host: ENV['HTM_DBHOST'] || 'cw7rxj91bm.srbbwwxn56.tsdb.cloud.timescale.com',
+          port: (ENV['HTM_DBPORT'] || 37807).to_i,
+          dbname: ENV['HTM_DBNAME'],
+          user: ENV['HTM_DBUSER'],
+          password: ENV['HTM_DBPASS'],
+          sslmode: 'require'
+        }
+      end
+
+      # Get default database configuration
+      #
+      # @return [Hash, nil] Connection configuration hash
+      #
+      def default_config
+        # Prefer HTM_DBURL if available
+        if ENV['HTM_DBURL']
+          parse_connection_url(ENV['HTM_DBURL'])
+        elsif ENV['HTM_DBNAME']
+          parse_connection_params
+        else
+          nil
+        end
+      end
+
+      private
+
+      def verify_extensions(conn)
+        # Check pgvector
+        pgvector = conn.exec("SELECT extversion FROM pg_extension WHERE extname='vector'").first
+        if pgvector
+          puts "✓ pgvector version: #{pgvector['extversion']}"
+        else
+          puts "⚠ Warning: pgvector extension not found"
+        end
+
+        # Check pg_trgm
+        pg_trgm = conn.exec("SELECT extversion FROM pg_extension WHERE extname='pg_trgm'").first
+        if pg_trgm
+          puts "✓ pg_trgm version: #{pg_trgm['extversion']}"
+        else
+          puts "⚠ Warning: pg_trgm extension not found"
+        end
+      end
+
+      # Run ActiveRecord migrations from db/migrate/
+      #
+      # @return [void]
+      #
+      def run_activerecord_migrations
+        migrations_path = File.expand_path('../../db/migrate', __dir__)
+
+        unless Dir.exist?(migrations_path)
+          puts "⚠ No migrations directory found at #{migrations_path}"
+          return
+        end
+
+        conn = ActiveRecord::Base.connection
+
+        # Create schema_migrations table if it doesn't exist
+        unless conn.table_exists?('schema_migrations')
+          conn.create_table(:schema_migrations, id: false) do |t|
+            t.string :version, null: false, primary_key: true
+          end
+        end
+
+        # Get list of migration files
+        migration_files = Dir.glob("#{migrations_path}/*.rb").sort
+        puts "Found #{migration_files.length} migration files"
+
+        # Run each migration
+        migration_files.each do |file|
+          version = File.basename(file).split('_').first
+          name = File.basename(file, '.rb')
+
+          # Check if already run
+          already_run = conn.select_value(
+            "SELECT COUNT(*) FROM schema_migrations WHERE version = '#{version}'"
+          ).to_i > 0
+
+          if already_run
+            puts "  ✓ #{name} (already migrated)"
+          else
+            puts "  → Running #{name}..."
+            require file
+
+            # Get the migration class
+            class_name = name.split('_')[1..].map(&:capitalize).join
+            migration_class = Object.const_get(class_name)
+
+            # Run the migration
+            migration = migration_class.new
+            migration.migrate(:up)
+
+            # Record in schema_migrations
+            conn.execute(
+              "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
+            )
+
+            puts "    ✓ Completed"
+          end
+        end
+
+        puts "✓ All migrations completed"
+      end
+
+      # Clean up pg_dump output to make it more readable
+      #
+      # @param schema_dump [String] Raw pg_dump output
+      # @return [String] Cleaned schema
+      #
+      def clean_schema_dump(schema_dump)
+        lines = schema_dump.split("\n")
+        cleaned = []
+
+        # Add header
+        cleaned << "-- HTM Database Schema"
+        cleaned << "-- Auto-generated from database using pg_dump"
+        cleaned << "-- DO NOT EDIT THIS FILE MANUALLY"
+        cleaned << "-- Run 'rake htm:db:schema:dump' to regenerate"
+        cleaned << ""
+
+        # Skip pg_dump header comments
+        skip_until_content = true
+
+        lines.each do |line|
+          # Skip header comments
+          if skip_until_content
+            if line =~ /^(SET|CREATE|ALTER|--\s*Name:|COMMENT)/
+              skip_until_content = false
+            else
+              next
+            end
+          end
+
+          # Skip SET commands (session-specific settings)
+          next if line =~ /^SET /
+
+          # Skip SELECT pg_catalog.set_config
+          next if line =~ /^SELECT pg_catalog\.set_config/
+
+          # Skip extension comments (we keep extension creation)
+          next if line =~ /^COMMENT ON EXTENSION/
+
+          # Keep everything else
+          cleaned << line
+        end
+
+        # Remove multiple blank lines
+        result = cleaned.join("\n")
+        result.gsub!(/\n{3,}/, "\n\n")
+
+        result
+      end
+
+      # Old methods removed - now using ActiveRecord migrations
+      # def run_schema(conn) - REMOVED
+      # def run_migrations_if_needed(conn) - REMOVED (see run_activerecord_migrations above)
+    end
+  end
+end

data/lib/htm/embedding_service.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+class HTM
+  # Embedding Service - Processes and validates vector embeddings
+  #
+  # This service wraps the configured embedding generator and provides:
+  # - Response validation
+  # - Dimension handling (padding/truncation)
+  # - Error handling and logging
+  # - Storage formatting
+  #
+  # The actual LLM call is delegated to HTM.configuration.embedding_generator
+  #
+  class EmbeddingService
+    MAX_DIMENSION = 2000  # Maximum dimension for pgvector HNSW index
+
+    # Generate embedding with validation and processing
+    #
+    # @param text [String] Text to embed
+    # @return [Hash] Processed embedding with metadata
+    #   {
+    #     embedding: Array<Float>,           # Original embedding
+    #     dimension: Integer,                # Original dimension
+    #     storage_embedding: String,         # Formatted for database storage
+    #     storage_dimension: Integer         # Padded dimension (2000)
+    #   }
+    #
+    def self.generate(text)
+      HTM.logger.debug "EmbeddingService: Generating embedding for #{text.length} chars"
+
+      # Call configured embedding generator
+      raw_embedding = HTM.configuration.embedding_generator.call(text)
+
+      # Validate response
+      validate_embedding!(raw_embedding)
+
+      # Get actual dimension
+      actual_dimension = raw_embedding.length
+
+      # Check dimension limit
+      if actual_dimension > MAX_DIMENSION
+        HTM.logger.warn "EmbeddingService: Embedding dimension #{actual_dimension} exceeds max #{MAX_DIMENSION}, truncating"
+        raw_embedding = raw_embedding[0...MAX_DIMENSION]
+        actual_dimension = MAX_DIMENSION
+      end
+
+      # Pad to 2000 dimensions for consistent storage
+      storage_embedding = pad_embedding(raw_embedding)
+
+      # Format for database storage
+      storage_string = format_for_storage(storage_embedding)
+
+      HTM.logger.debug "EmbeddingService: Generated #{actual_dimension}D embedding (padded to #{MAX_DIMENSION})"
+
+      {
+        embedding: raw_embedding,
+        dimension: actual_dimension,
+        storage_embedding: storage_string,
+        storage_dimension: MAX_DIMENSION
+      }
+
+    rescue HTM::EmbeddingError
+      raise
+    rescue StandardError => e
+      HTM.logger.error "EmbeddingService: Failed to generate embedding: #{e.message}"
+      raise HTM::EmbeddingError, "Embedding generation failed: #{e.message}"
+    end
+
+    # Validate embedding response format
+    #
+    # @param embedding [Object] Raw embedding from generator
+    # @raise [HTM::EmbeddingError] if invalid
+    #
+    def self.validate_embedding!(embedding)
+      unless embedding.is_a?(Array)
+        raise HTM::EmbeddingError, "Embedding must be an Array, got #{embedding.class}"
+      end
+
+      if embedding.empty?
+        raise HTM::EmbeddingError, "Embedding array is empty"
+      end
+
+      unless embedding.all? { |v| v.is_a?(Numeric) }
+        raise HTM::EmbeddingError, "Embedding must contain only numeric values"
+      end
+
+      # Check for NaN or Infinity
+      if embedding.any? { |v| v.respond_to?(:nan?) && v.nan? || v.respond_to?(:infinite?) && v.infinite? }
+        raise HTM::EmbeddingError, "Embedding contains NaN or Infinity values"
+      end
+    end
+
+    # Pad embedding to MAX_DIMENSION with zeros
+    #
+    # @param embedding [Array<Float>] Original embedding
+    # @return [Array<Float>] Padded embedding
+    #
+    def self.pad_embedding(embedding)
+      return embedding if embedding.length >= MAX_DIMENSION
+
+      embedding + Array.new(MAX_DIMENSION - embedding.length, 0.0)
+    end
+
+    # Format embedding for database storage
+    #
+    # @param embedding [Array<Float>] Padded embedding
+    # @return [String] PostgreSQL array format
+    #
+    def self.format_for_storage(embedding)
+      "[#{embedding.join(',')}]"
+    end
+  end
+end