dbviewer 0.5.1 → 0.5.3

data/lib/dbviewer/datatable/query_operations.rb

@@ -0,0 +1,330 @@
+module Dbviewer
+  module Datatable
+    # QueryOperations handles CRUD operations and data querying for database tables
+    # It provides methods to fetch, filter and manipulate data in tables
+    class QueryOperations
+      attr_reader :connection, :adapter_name
+
+      # Initialize with dependencies
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] The database connection
+      # @param dynamic_model_factory [Dbviewer::Database::DynamicModelFactory] Factory for creating dynamic AR models
+      # @param query_executor [Dbviewer::Query::Executor] Executor for raw SQL queries
+      # @param table_metadata_manager [Dbviewer::Database::MetadataManager] Manager for table metadata
+      def initialize(connection, dynamic_model_factory, query_executor, table_metadata_manager)
+        @connection = connection
+        @adapter_name = connection.adapter_name.downcase
+        @dynamic_model_factory = dynamic_model_factory
+        @query_executor = query_executor
+        @table_metadata_manager = table_metadata_manager
+        @query_analyzer = ::Dbviewer::Query::Analyzer.new(connection)
+      end
+
+      # Get the number of columns in a table
+      # @param table_name [String] Name of the table
+      # @return [Integer] Number of columns
+      def column_count(table_name)
+        table_columns(table_name).size
+      end
+
+      # Get records from a table with pagination and sorting
+      # @param table_name [String] Name of the table
+      # @param params [Dbviewer::Datatable::QueryParams] Query parameters object
+      # @return [ActiveRecord::Result] Result set with columns and rows
+      def table_records(table_name, params)
+        model = get_model_for(table_name)
+        query = model.all
+
+        # Apply column filters if provided
+        query = apply_column_filters(query, table_name, params.column_filters)
+
+        # Apply sorting if provided
+        if params.order_by.present? && column_exists?(table_name, params.order_by)
+          query = query.order("#{connection.quote_column_name(params.order_by)} #{params.direction}")
+        end
+
+        # Apply pagination
+        records = query.limit(params.per_page).offset((params.page - 1) * params.per_page)
+
+        # Get column names for consistent ordering
+        column_names = table_columns(table_name).map { |c| c[:name] }
+
+        # Format results
+        @query_executor.to_result_set(records, column_names)
+      rescue => e
+        Rails.logger.error("[DBViewer] Error executing table query: #{e.message}")
+        raise e
+      end
+
+      # Get the total count of records in a table
+      # @param table_name [String] Name of the table
+      # @return [Integer] Number of records
+      def table_count(table_name)
+        get_model_for(table_name).count
+      rescue => e
+        Rails.logger.error("[DBViewer] Error counting records in table #{table_name}: #{e.message}")
+        0
+      end
+
+      # Get the number of records in a table (alias for table_count)
+      # @param table_name [String] Name of the table
+      # @return [Integer] Number of records
+      def record_count(table_name)
+        table_count(table_name)
+      end
+
+      # Get the number of records in a table with filters applied
+      # @param table_name [String] Name of the table
+      # @param column_filters [Hash] Hash of column_name => filter_value for filtering
+      # @return [Integer] Number of filtered records
+      def filtered_record_count(table_name, column_filters = {})
+        return table_count(table_name) unless column_filters.present?
+
+        model = get_model_for(table_name)
+        query = model.all
+
+        # Apply filters in the same way as table_records
+        query = apply_column_filters(query, table_name, column_filters)
+        query.count
+      rescue => e
+        Rails.logger.error("[DBViewer] Error counting filtered records in table #{table_name}: #{e.message}")
+        0
+      end
+
+      # Get column histogram/value distribution data for a specific column
+      # @param table_name [String] Name of the table
+      # @param column_name [String] Name of the column
+      # @param limit [Integer] Maximum number of distinct values to return
+      # @return [Array<Hash>] Array of value distribution data with labels and counts
+      def fetch_column_distribution(table_name, column_name, limit = 20)
+        return [] unless column_exists?(table_name, column_name)
+
+        query = "SELECT #{column_name} as label, COUNT(*) as count FROM #{table_name}
+                 WHERE #{column_name} IS NOT NULL
+                 GROUP BY #{column_name}
+                 ORDER BY count DESC LIMIT #{limit}"
+
+        begin
+          result = @connection.execute(query)
+          adapter = @connection.adapter_name.downcase
+
+          # Format depends on adapter
+          if adapter =~ /mysql/
+            result.to_a.map { |row| { label: row[0], value: row[1] } }
+          elsif adapter =~ /sqlite/
+            result.map { |row| { label: row["label"], value: row["count"] } }
+          else # postgresql
+            result.map { |row| { label: row["label"], value: row["count"] } }
+          end
+        rescue => e
+          Rails.logger.error("Error fetching column distribution: #{e.message}")
+          []
+        end
+      end
+
+      # Get timestamp aggregation data for charts
+      # @param table_name [String] Name of the table
+      # @param grouping [String] Grouping type (hourly, daily, weekly, monthly)
+      # @param column [String] Timestamp column name (defaults to created_at)
+      # @return [Array<Hash>] Array of timestamp data with labels and counts
+      def fetch_timestamp_data(table_name, grouping = "daily", column = "created_at")
+        return [] unless column_exists?(table_name, column)
+
+        adapter = @connection.adapter_name.downcase
+
+        date_format = case grouping
+        when "hourly"
+          if adapter =~ /mysql/
+            "DATE_FORMAT(#{column}, '%Y-%m-%d %H:00')"
+          elsif adapter =~ /sqlite/
+            "strftime('%Y-%m-%d %H:00', #{column})"
+          else # postgresql
+            "TO_CHAR(#{column}, 'YYYY-MM-DD HH24:00')"
+          end
+        when "weekly"
+          if adapter =~ /mysql/
+            "DATE_FORMAT(#{column}, '%Y-%v')"
+          elsif adapter =~ /sqlite/
+            "strftime('%Y-%W', #{column})"
+          else # postgresql
+            "TO_CHAR(#{column}, 'YYYY-IW')"
+          end
+        when "monthly"
+          if adapter =~ /mysql/
+            "DATE_FORMAT(#{column}, '%Y-%m')"
+          elsif adapter =~ /sqlite/
+            "strftime('%Y-%m', #{column})"
+          else # postgresql
+            "TO_CHAR(#{column}, 'YYYY-MM')"
+          end
+        else # daily is default
+          if adapter =~ /mysql/
+            "DATE(#{column})"
+          elsif adapter =~ /sqlite/
+            "date(#{column})"
+          else # postgresql
+            "DATE(#{column})"
+          end
+        end
+
+        # Query works the same for all database adapters
+        query = "SELECT #{date_format} as label, COUNT(*) as count FROM #{table_name}
+                 WHERE #{column} IS NOT NULL
+                 GROUP BY label
+                 ORDER BY MIN(#{column}) DESC LIMIT 30"
+
+        begin
+          result = @connection.execute(query)
+
+          # Format depends on adapter
+          if adapter =~ /mysql/
+            result.to_a.map { |row| { label: row[0], value: row[1] } }
+          elsif adapter =~ /sqlite/
+            result.map { |row| { label: row["label"], value: row["count"] } }
+          else # postgresql
+            result.map { |row| { label: row["label"], value: row["count"] } }
+          end
+        rescue => e
+          Rails.logger.error("Error fetching timestamp data: #{e.message}")
+          []
+        end
+      end
+
+      # Execute a raw SQL query after validating for safety
+      # @param sql [String] SQL query to execute
+      # @return [ActiveRecord::Result] Result set with columns and rows
+      # @raise [StandardError] If the query is invalid or unsafe
+      def execute_query(sql)
+        @query_executor.execute_query(sql)
+      end
+
+      # Execute a SQLite PRAGMA command without adding a LIMIT clause
+      # @param pragma [String] PRAGMA command to execute (without the "PRAGMA" keyword)
+      # @return [ActiveRecord::Result] Result set with the PRAGMA value
+      # @raise [StandardError] If the query is invalid or cannot be executed
+      def execute_sqlite_pragma(pragma)
+        @query_executor.execute_sqlite_pragma(pragma)
+      end
+
+      # Analyze query patterns and return performance recommendations
+      # @param table_name [String] Name of the table
+      # @param query_params [Dbviewer::Datatable::QueryParams] Query parameters
+      # @return [Hash] Analysis results
+      def analyze_query(table_name, query_params)
+        @query_analyzer.analyze_query(table_name, query_params)
+      end
+
+      private
+
+      # Apply column filters to a query
+      # @param query [ActiveRecord::Relation] The query to apply filters to
+      # @param table_name [String] Name of the table
+      # @param column_filters [Hash] Hash of column_name => filter_value pairs
+      # @return [ActiveRecord::Relation] The filtered query
+      def apply_column_filters(query, table_name, column_filters)
+        return query unless column_filters.present?
+
+        # Create a copy of column_filters to modify without affecting the original
+        filters = column_filters.dup
+
+        # First check if we have a datetime range filter for created_at
+        if filters["created_at"].present? &&
+           filters["created_at_end"].present? &&
+           column_exists?(table_name, "created_at")
+
+          # Handle datetime range for created_at
+          begin
+            start_datetime = Time.parse(filters["created_at"].to_s)
+            end_datetime = Time.parse(filters["created_at_end"].to_s)
+
+            # Make sure end_datetime is at the end of the day/minute if it doesn't have time component
+            if end_datetime.to_s.match(/00:00:00/)
+              end_datetime = end_datetime.end_of_day
+            end
+
+            Rails.logger.info("[DBViewer] Applying date range filter on #{table_name}.created_at: #{start_datetime} to #{end_datetime}")
+
+            # Use qualified column name for tables with schema
+            column_name = "#{table_name}.created_at"
+
+            # Different databases may require different SQL for datetime comparison
+            adapter_name = connection.adapter_name.downcase
+
+            if adapter_name =~ /mysql/
+              query = query.where("#{column_name} BETWEEN ? AND ?", start_datetime, end_datetime)
+            elsif adapter_name =~ /sqlite/
+              # SQLite needs special handling for datetime format
+              query = query.where("datetime(#{column_name}) BETWEEN datetime(?) AND datetime(?)", start_datetime, end_datetime)
+            else # postgresql
+              query = query.where("#{column_name} >= ? AND #{column_name} <= ?", start_datetime, end_datetime)
+            end
+
+            # Remove these from filters as they're handled above
+            filters.delete("created_at")
+            filters.delete("created_at_end")
+
+          rescue => e
+            Rails.logger.error("[DBViewer] Error parsing date range: #{e.message}")
+            # Remove the problematic filters and continue
+            filters.delete("created_at")
+            filters.delete("created_at_end")
+          end
+        end
+
+        # Apply remaining simple column filters
+        filters.each do |column, value|
+          next unless column_exists?(table_name, column)
+          next if value.blank?
+
+          column_sym = column.to_sym
+          Rails.logger.debug("[DBViewer] Applying filter: #{column} = #{value}")
+
+          # Handle different types of filtering
+          if value.to_s.include?("%") || value.to_s.include?("*")
+            # Pattern matching (LIKE operation)
+            pattern = value.to_s.gsub("*", "%")
+            query = query.where("#{column} LIKE ?", pattern)
+          elsif value.to_s.start_with?(">=", "<=", ">", "<", "!=")
+            # Comparison operators
+            operator = value.to_s.match(/^(>=|<=|>|<|!=)/)[1]
+            comparison_value = value.to_s.gsub(/^(>=|<=|>|<|!=)\s*/, "")
+
+            case operator
+            when ">="
+              query = query.where("#{column} >= ?", comparison_value)
+            when "<="
+              query = query.where("#{column} <= ?", comparison_value)
+            when ">"
+              query = query.where("#{column} > ?", comparison_value)
+            when "<"
+              query = query.where("#{column} < ?", comparison_value)
+            when "!="
+              query = query.where("#{column} != ?", comparison_value)
+            end
+          else
+            # Exact match
+            query = query.where(column_sym => value)
+          end
+        end
+
+        query
+      end
+
+      # Helper methods delegated to managers
+      def get_model_for(table_name)
+        @dynamic_model_factory.get_model_for(table_name)
+      end
+
+      def table_columns(table_name)
+        @table_metadata_manager.table_columns(table_name)
+      end
+
+      def column_exists?(table_name, column_name)
+        @table_metadata_manager.column_exists?(table_name, column_name)
+      end
+
+      def primary_key(table_name)
+        @table_metadata_manager.primary_key(table_name)
+      end
+    end
+  end
+end

data/lib/dbviewer/datatable/query_params.rb

@@ -0,0 +1,41 @@
+module Dbviewer
+  module Datatable
+    # QueryParams encapsulates parameters for table querying operations
+    class QueryParams
+      attr_reader :page, :order_by, :direction, :per_page, :column_filters, :max_records
+
+      # Initialize query parameters with defaults
+      # @param page [Integer] Page number (1-based)
+      # @param order_by [String, nil] Column to sort by
+      # @param direction [String] Sort direction ('ASC' or 'DESC')
+      # @param per_page [Integer, nil] Number of records per page
+      # @param column_filters [Hash, nil] Hash of column filters
+      # @param max_records [Integer] Maximum number of records to fetch
+      def initialize(
+        page: 1,
+        order_by: nil,
+        direction: "ASC",
+        per_page: nil,
+        column_filters: nil,
+        max_records: 1000
+      )
+        @page = [ 1, page.to_i ].max
+        @order_by = order_by
+        @direction = normalize_direction(direction)
+        @per_page = normalize_per_page(per_page || 25, max_records)
+        @column_filters = column_filters || {}
+        @max_records = max_records
+      end
+
+      private
+
+      def normalize_direction(dir)
+        %w[ASC DESC].include?(dir.to_s.upcase) ? dir.to_s.upcase : "ASC"
+      end
+
+      def normalize_per_page(per_page_value, max)
+        [ per_page_value.to_i, max ].min
+      end
+    end
+  end
+end

data/lib/dbviewer/engine.rb

@@ -2,9 +2,10 @@ module Dbviewer
   class Engine < ::Rails::Engine
     isolate_namespace Dbviewer
 
-    # Ensure lib directory is in the autoload path
-    config.autoload_paths << File.expand_path("../../", __FILE__)
-    config.eager_load_paths << File.expand_path("../../", __FILE__)
+    # Autoload lib directory
+    lib_path = File.expand_path("..", __dir__)
+    config.autoload_paths << lib_path
+    config.eager_load_paths << lib_path
 
     # Register generators
     config.app_generators do |g|
@@ -18,12 +19,14 @@ module Dbviewer
     end
 
     initializer "dbviewer.notifications" do
+      return unless Rails.env.development?
+
       ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
         event = ActiveSupport::Notifications::Event.new(*args)
 
         next if skip_internal_query?(event)
 
-        Logger.instance.add(event)
+        Dbviewer::Query::Logger.instance.add(event)
       end
     end
 
@@ -33,10 +36,10 @@ module Dbviewer
       caller_locations = caller_locations(1)
       return false unless caller_locations
 
-      # Look for dbviewer in the call stack
-      caller_locations.any? { |l| l.path.include?("dbviewer") }
-    rescue
-      false
+      excluded_caller_locations = caller_locations.filter do |caller_location|
+        !caller_location.path.include?("lib/dbviewer/engine.rb")
+      end
+      excluded_caller_locations.any? { |l| l.path.include?("dbviewer") }
     end
   end
 end

data/lib/dbviewer/query/analyzer.rb

@@ -0,0 +1,250 @@
+module Dbviewer
+  module Query
+    # Analyzer handles analysis of query patterns and statistics
+    class Analyzer
+      # Calculate statistics for a collection of queries
+      def self.generate_stats(queries)
+        {
+          total_count: queries.size,
+          total_duration_ms: queries.sum { |q| q[:duration_ms] },
+          avg_duration_ms: calculate_average_duration(queries),
+          max_duration_ms: queries.map { |q| q[:duration_ms] }.max || 0,
+          tables_queried: extract_queried_tables(queries),
+          potential_n_plus_1: detect_potential_n_plus_1(queries),
+          slowest_queries: get_slowest_queries(queries)
+        }.merge(calculate_request_stats(queries))
+      end
+
+      # Instance methods for query analysis
+      attr_reader :connection, :adapter_name
+
+      # Initialize the analyzer for instance methods
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+      def initialize(connection)
+        @connection = connection
+        @adapter_name = connection.adapter_name.downcase
+      end
+
+      # Check if a table has an index on a column
+      # @param table_name [String] Name of the table
+      # @param column_name [String] Name of the column
+      # @return [Boolean] True if column has an index, false otherwise
+      def has_index_on?(table_name, column_name)
+        indexes = connection.indexes(table_name)
+        indexes.any? do |index|
+          index.columns.include?(column_name) ||
+          (index.columns.length == 1 && index.columns[0] == column_name)
+        end
+      end
+
+      # Analyze a query and provide performance statistics and recommendations
+      # @param table_name [String] Name of the table
+      # @param query_params [Dbviewer::Datatable::QueryParams] Query parameters with filters
+      # @return [Hash] Analysis results with statistics and recommendations
+      def analyze_query(table_name, query_params)
+        results = {
+          table: table_name,
+          filters: query_params.column_filters.keys,
+          analysis: [],
+          recommendations: []
+        }
+
+        # Check created_at filter performance
+        if query_params.column_filters.key?("created_at")
+          analyze_timestamp_query(table_name, "created_at", results)
+        end
+
+        # Check updated_at filter performance
+        if query_params.column_filters.key?("updated_at")
+          analyze_timestamp_query(table_name, "updated_at", results)
+        end
+
+        # Check other filters for potential indexes
+        query_params.column_filters.each do |column_name, _value|
+          next if [ "created_at", "updated_at" ].include?(column_name)
+
+          unless has_index_on?(table_name, column_name)
+            results[:recommendations] << {
+              type: "missing_index",
+              message: "Consider adding an index on '#{column_name}' for faster filtering",
+              sql: index_creation_sql(table_name, column_name)
+            }
+          end
+        end
+
+        # Add database-specific recommendations
+        add_database_specific_recommendations(results)
+
+        results
+      end
+
+      private
+
+      # Analyze timestamp-based queries
+      def analyze_timestamp_query(table_name, column_name, results)
+        # Check if column exists
+        unless connection.column_exists?(table_name, column_name)
+          results[:analysis] << "Column '#{column_name}' not found in table '#{table_name}'"
+          return
+        end
+
+        # Check if there's an index on the timestamp column
+        unless has_index_on?(table_name, column_name)
+          results[:recommendations] << {
+            type: "missing_index",
+            message: "Consider adding an index on '#{column_name}' for faster filtering",
+            sql: index_creation_sql(table_name, column_name)
+          }
+        end
+
+        # Estimate data distribution if possible
+        if adapter_supports_statistics?(adapter_name)
+          add_data_distribution_stats(table_name, column_name, results)
+        end
+      rescue => e
+        results[:analysis] << "Error analyzing timestamp query: #{e.message}"
+      end
+
+      # Check if adapter supports statistics gathering
+      def adapter_supports_statistics?(adapter_name)
+        adapter_name.include?("postgresql")
+      end
+
+      # Add data distribution statistics
+      def add_data_distribution_stats(table_name, column_name, results)
+        case adapter_name
+        when /postgresql/
+          begin
+            # Get approximate date range and distribution
+            range_query = "SELECT min(#{column_name}), max(#{column_name}) FROM #{table_name}"
+            range_result = connection.execute(range_query).first
+
+            if range_result["min"] && range_result["max"]
+              min_date = range_result["min"]
+              max_date = range_result["max"]
+
+              results[:analysis] << {
+                type: "date_range",
+                min_date: min_date,
+                max_date: max_date,
+                span_days: ((Time.parse(max_date) - Time.parse(min_date)) / 86400).round
+              }
+            end
+          rescue => e
+            results[:analysis] << "Error getting date distribution: #{e.message}"
+          end
+        end
+      end
+
+      # Generate SQL for index creation
+      def index_creation_sql(table_name, column_name)
+        index_name = "index_#{table_name.gsub('.', '_')}_on_#{column_name}"
+        "CREATE INDEX #{index_name} ON #{table_name} (#{column_name})"
+      end
+
+      # Add database-specific recommendations
+      def add_database_specific_recommendations(results)
+        case adapter_name
+        when /mysql/
+          results[:recommendations] << {
+            type: "performance",
+            message: "For MySQL, consider optimizing the query with appropriate indexes and use EXPLAIN to verify query plan"
+          }
+        when /postgresql/
+          results[:recommendations] << {
+            type: "performance",
+            message: "For PostgreSQL, consider using EXPLAIN ANALYZE to verify query execution plan"
+          }
+        when /sqlite/
+          results[:recommendations] << {
+            type: "performance",
+            message: "For SQLite, consider using EXPLAIN QUERY PLAN to verify query execution strategy"
+          }
+        end
+      end
+
+      def self.calculate_average_duration(queries)
+        queries.any? ? (queries.sum { |q| q[:duration_ms] } / queries.size.to_f).round(2) : 0
+      end
+
+      def self.calculate_request_stats(queries)
+        # Calculate request groups statistics
+        requests = queries.group_by { |q| q[:request_id] }
+        {
+          request_count: requests.size,
+          avg_queries_per_request: queries.any? ? (queries.size.to_f / requests.size).round(2) : 0,
+          max_queries_per_request: requests.map { |_id, reqs| reqs.size }.max || 0
+        }
+      end
+
+      def self.extract_queried_tables(queries)
+        tables = Hash.new(0)
+
+        queries.each do |query|
+          extracted = ::Dbviewer::Query::Parser.extract_tables(query[:sql])
+          extracted.each { |table| tables[table] += 1 }
+        end
+
+        tables.sort_by { |_table, count| -count }.first(10).to_h
+      end
+
+      def self.detect_potential_n_plus_1(queries)
+        return [] if queries.empty?
+
+        potential_issues = []
+
+        # Group queries by request ID
+        requests = queries.group_by { |q| q[:request_id] }
+
+        requests.each do |request_id, request_queries|
+          analyze_request_patterns(request_id, request_queries, potential_issues)
+        end
+
+        potential_issues.sort_by { |issue| -issue[:total_duration_ms] }.first(10)
+      end
+
+      def self.get_slowest_queries(queries)
+        queries.sort_by { |q| -q[:duration_ms] }.first(10).map do |query|
+          {
+            sql: query[:sql],
+            duration_ms: query[:duration_ms],
+            request_id: query[:request_id],
+            timestamp: query[:timestamp]
+          }
+        end
+      end
+
+      def self.analyze_request_patterns(request_id, request_queries, potential_issues)
+        # Group similar queries within this request
+        patterns = {}
+
+        request_queries.each do |query|
+          # Normalize the query to detect patterns
+          normalized = ::Dbviewer::Query::Parser.normalize(query[:sql])
+          patterns[normalized] ||= []
+          patterns[normalized] << query
+        end
+
+        # Look for patterns with many repetitions
+        patterns.each do |pattern, pattern_queries|
+          next if pattern_queries.size < 5  # Only interested in repeated patterns
+
+          # Check if these queries target the same table
+          tables = ::Dbviewer::Query::Parser.extract_tables(pattern_queries.first[:sql])
+          target_table = tables.size == 1 ? tables.first : nil
+
+          # Add to potential issues
+          total_time = pattern_queries.sum { |q| q[:duration_ms] }
+          potential_issues << {
+            request_id: request_id,
+            pattern: pattern_queries.first[:sql],
+            count: pattern_queries.size,
+            table: target_table,
+            total_duration_ms: total_time.round(2),
+            avg_duration_ms: (total_time / pattern_queries.size).round(2)
+          }
+        end
+      end
+    end
+  end
+end