pg_reports 0.3.1 → 0.4.0

data/lib/pg_reports/modules/queries.rb

@@ -3,109 +3,17 @@
 module PgReports
   module Modules
     # Query analysis module - analyzes pg_stat_statements data
+    # Most report methods are generated from YAML definitions in lib/pg_reports/definitions/queries/
     module Queries
       extend self
 
-      # Slow queries - queries with high mean execution time
-      # @return [Report] Report with slow queries
-      def slow_queries(limit: 20)
-        data = executor.execute_from_file(:queries, :slow_queries)
-        threshold = PgReports.config.slow_query_threshold_ms
-
-        filtered = data.select { |row| row["mean_time_ms"].to_f >= threshold }
-          .first(limit)
-
-        enriched = enrich_with_annotations(filtered)
-
-        Report.new(
-          title: "Slow Queries (mean time >= #{threshold}ms)",
-          data: enriched,
-          columns: %w[query source calls mean_time_ms total_time_ms rows_per_call]
-        )
-      end
-
-      # Heavy queries - queries called most frequently
-      # @return [Report] Report with heavy queries
-      def heavy_queries(limit: 20)
-        data = executor.execute_from_file(:queries, :heavy_queries)
-        threshold = PgReports.config.heavy_query_threshold_calls
-
-        filtered = data.select { |row| row["calls"].to_i >= threshold }
-          .first(limit)
-
-        enriched = enrich_with_annotations(filtered)
-
-        Report.new(
-          title: "Heavy Queries (calls >= #{threshold})",
-          data: enriched,
-          columns: %w[query source calls total_time_ms mean_time_ms cache_hit_ratio]
-        )
-      end
-
-      # Expensive queries - queries consuming most total time
-      # @return [Report] Report with expensive queries
-      def expensive_queries(limit: 20)
-        data = executor.execute_from_file(:queries, :expensive_queries)
-        threshold = PgReports.config.expensive_query_threshold_ms
-
-        filtered = data.select { |row| row["total_time_ms"].to_f >= threshold }
-          .first(limit)
-
-        enriched = enrich_with_annotations(filtered)
-
-        Report.new(
-          title: "Expensive Queries (total time >= #{threshold}ms)",
-          data: enriched,
-          columns: %w[query source calls total_time_ms percent_of_total mean_time_ms]
-        )
-      end
-
-      # Queries missing indexes - sequential scans on large tables
-      # @return [Report] Report with queries likely missing indexes
-      def missing_index_queries(limit: 20)
-        data = executor.execute_from_file(:queries, :missing_index_queries)
-          .first(limit)
-
-        enriched = enrich_with_annotations(data)
-
-        Report.new(
-          title: "Queries Potentially Missing Indexes",
-          data: enriched,
-          columns: %w[query source calls seq_scan_count rows_examined table_name]
-        )
-      end
-
-      # Queries with low cache hit ratio
-      # @return [Report] Report with queries having poor cache utilization
-      def low_cache_hit_queries(limit: 20, min_calls: 100)
-        data = executor.execute_from_file(:queries, :low_cache_hit_queries)
-
-        filtered = data.select { |row| row["calls"].to_i >= min_calls }
-          .first(limit)
-
-        enriched = enrich_with_annotations(filtered)
-
-        Report.new(
-          title: "Queries with Low Cache Hit Ratio (min #{min_calls} calls)",
-          data: enriched,
-          columns: %w[query source calls cache_hit_ratio shared_blks_hit shared_blks_read]
-        )
-      end
-
-      # All query statistics ordered by total time
-      # @return [Report] Report with all query statistics
-      def all_queries(limit: 50)
-        data = executor.execute_from_file(:queries, :all_queries)
-          .first(limit)
-
-        enriched = enrich_with_annotations(data)
-
-        Report.new(
-          title: "All Query Statistics (top #{limit})",
-          data: enriched,
-          columns: %w[query source calls total_time_ms mean_time_ms rows]
-        )
-      end
+      # The following methods are auto-generated from YAML:
+      # - slow_queries(limit: 20)
+      # - heavy_queries(limit: 20)
+      # - expensive_queries(limit: 20)
+      # - missing_index_queries(limit: 20)
+      # - low_cache_hit_queries(limit: 20, min_calls: 100)
+      # - all_queries(limit: 50)
 
       # Reset pg_stat_statements statistics
       def reset_statistics!
@@ -120,6 +28,7 @@ module PgReports
       end
 
       # Enrich query data with parsed annotations (Marginalia, Rails QueryLogs, etc.)
+      # Used by YAML-based reports via enrichment hook
       def enrich_with_annotations(data)
         data.map do |row|
           query = row["query"].to_s

data/lib/pg_reports/modules/schema_analysis.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module PgReports
+  module Modules
+    # Schema analysis module - validates database schema consistency with application code
+    module SchemaAnalysis
+      extend self
+
+      # Missing validations - unique indexes without corresponding validations
+      # @return [Report] Report with unique indexes missing validations
+      def missing_validations
+        unique_indexes = executor.execute_from_file(:schema_analysis, :unique_indexes)
+        results = []
+
+        unique_indexes.each do |index|
+          schema_name = index["schema_name"]
+          table_name = index["table_name"]
+          index_name = index["index_name"]
+          column_names = parse_array(index["column_names"])
+          index_type = index["index_type"]
+
+          # Skip primary keys (they don't need validation)
+          next if index_type == "primary_key"
+
+          # Try to find the model for this table
+          model = find_model_for_table(table_name)
+
+          if model.nil?
+            results << {
+              "schema" => schema_name,
+              "table_name" => table_name,
+              "index_name" => index_name,
+              "columns" => column_names.join(", "),
+              "index_type" => index_type,
+              "status" => "no_model",
+              "validation_status" => "Model not found",
+              "suggestion" => "Create a model for this table or add validates :#{column_names.first}, uniqueness: true"
+            }
+            next
+          end
+
+          # Check if validation exists
+          has_validation = check_uniqueness_validation(model, column_names)
+
+          unless has_validation
+            results << {
+              "schema" => schema_name,
+              "table_name" => table_name,
+              "model_name" => model.name,
+              "index_name" => index_name,
+              "columns" => column_names.join(", "),
+              "index_type" => index_type,
+              "status" => "missing_validation",
+              "validation_status" => "Missing uniqueness validation",
+              "suggestion" => build_validation_suggestion(column_names)
+            }
+          end
+        end
+
+        Report.new(
+          title: "Unique Indexes Missing Validations",
+          data: results,
+          columns: %w[table_name model_name columns status validation_status]
+        )
+      end
+
+      private
+
+      def executor
+        @executor ||= Executor.new
+      end
+
+      # Parse PostgreSQL array string to Ruby array
+      def parse_array(array_string)
+        return [] if array_string.nil? || array_string.empty?
+
+        # PostgreSQL returns arrays as {val1,val2,val3}
+        array_string.gsub(/[{}]/, "").split(",").map(&:strip)
+      end
+
+      # Find ActiveRecord model for a given table name
+      def find_model_for_table(table_name)
+        # Try common naming conventions
+        possible_names = [
+          table_name.classify,                    # users -> User
+          table_name.singularize.classify,        # users -> User
+          table_name.camelize,                    # user_profiles -> UserProfile
+          table_name.singularize.camelize         # user_profiles -> UserProfile
+        ].uniq
+
+        possible_names.each do |model_name|
+          begin
+            model = model_name.constantize
+            return model if model.is_a?(Class) && model < ActiveRecord::Base
+          rescue NameError
+            # Model doesn't exist, try next one
+          end
+        end
+
+        nil
+      end
+
+      # Check if model has uniqueness validation for given columns
+      def check_uniqueness_validation(model, column_names)
+        return false unless model.respond_to?(:validators)
+
+        # Get all uniqueness validators
+        uniqueness_validators = model.validators.select do |v|
+          v.is_a?(ActiveRecord::Validations::UniquenessValidator)
+        end
+
+        return false if uniqueness_validators.empty?
+
+        # Check if any validator covers our columns
+        column_names.each do |column|
+          column_sym = column.to_sym
+
+          # Check if this column has a uniqueness validator
+          has_validator = uniqueness_validators.any? do |validator|
+            validator.attributes.include?(column_sym)
+          end
+
+          return false unless has_validator
+        end
+
+        # If we have multiple columns, check for composite uniqueness
+        if column_names.size > 1
+          # For composite indexes, we need to check if there's a validation with scope
+          primary_column = column_names.first.to_sym
+          scope_columns = column_names[1..-1].map(&:to_sym)
+
+          has_composite = uniqueness_validators.any? do |validator|
+            validator.attributes.include?(primary_column) &&
+              validator.options[:scope] &&
+              Array(validator.options[:scope]).sort == scope_columns.sort
+          end
+
+          return has_composite
+        end
+
+        true
+      end
+
+      # Build validation suggestion based on columns
+      def build_validation_suggestion(column_names)
+        if column_names.size == 1
+          "validates :#{column_names.first}, uniqueness: true"
+        else
+          primary = column_names.first
+          scopes = column_names[1..-1]
+          "validates :#{primary}, uniqueness: { scope: #{scopes.inspect} }"
+        end
+      end
+    end
+  end
+end

data/lib/pg_reports/modules/system.rb

@@ -3,68 +3,16 @@
 module PgReports
   module Modules
     # System-level database statistics
+    # Most report methods are generated from YAML definitions in lib/pg_reports/definitions/system/
     module System
       extend self
 
-      # Database sizes
-      # @return [Report] Report with database sizes
-      def database_sizes
-        data = executor.execute_from_file(:system, :database_sizes)
-
-        Report.new(
-          title: "Database Sizes",
-          data: data,
-          columns: %w[database size_mb size_pretty]
-        )
-      end
-
-      # PostgreSQL settings
-      # @return [Report] Report with important PostgreSQL settings
-      def settings
-        data = executor.execute_from_file(:system, :settings)
-
-        Report.new(
-          title: "PostgreSQL Settings",
-          data: data,
-          columns: %w[name setting unit category description]
-        )
-      end
-
-      # Extension information
-      # @return [Report] Report with installed extensions
-      def extensions
-        data = executor.execute_from_file(:system, :extensions)
-
-        Report.new(
-          title: "Installed Extensions",
-          data: data,
-          columns: %w[name version schema description]
-        )
-      end
-
-      # Database activity overview
-      # @return [Report] Report with current activity
-      def activity_overview
-        data = executor.execute_from_file(:system, :activity_overview)
-
-        Report.new(
-          title: "Database Activity Overview",
-          data: data,
-          columns: %w[metric value]
-        )
-      end
-
-      # Cache hit ratio for the entire database
-      # @return [Report] Report with cache statistics
-      def cache_stats
-        data = executor.execute_from_file(:system, :cache_stats)
-
-        Report.new(
-          title: "Database Cache Statistics",
-          data: data,
-          columns: %w[database heap_hit_ratio index_hit_ratio]
-        )
-      end
+      # The following methods are auto-generated from YAML:
+      # - database_sizes
+      # - settings
+      # - extensions
+      # - activity_overview
+      # - cache_stats
 
       # pg_stat_statements availability check
       # @return [Boolean] Whether pg_stat_statements is available

data/lib/pg_reports/modules/tables.rb

@@ -3,105 +3,18 @@
 module PgReports
   module Modules
     # Table analysis module
+    # All report methods are generated from YAML definitions in lib/pg_reports/definitions/tables/
     module Tables
       extend self
 
-      # Table sizes including indexes
-      # @return [Report] Report with table sizes
-      def table_sizes(limit: 50)
-        data = executor.execute_from_file(:tables, :table_sizes)
-          .first(limit)
-
-        Report.new(
-          title: "Table Sizes (top #{limit})",
-          data: data,
-          columns: %w[schema table_name table_size_mb index_size_mb total_size_mb row_count]
-        )
-      end
-
-      # Bloated tables - tables with high dead tuple ratio
-      # @return [Report] Report with bloated tables
-      def bloated_tables(limit: 20)
-        data = executor.execute_from_file(:tables, :bloated_tables)
-        threshold = PgReports.config.bloat_threshold_percent
-
-        filtered = data.select { |row| row["bloat_percent"].to_f >= threshold }
-          .first(limit)
-
-        Report.new(
-          title: "Bloated Tables (bloat >= #{threshold}%)",
-          data: filtered,
-          columns: %w[schema table_name live_rows dead_rows bloat_percent table_size_mb]
-        )
-      end
-
-      # Tables needing vacuum - high dead rows count
-      # @return [Report] Report with tables needing vacuum
-      def vacuum_needed(limit: 20)
-        data = executor.execute_from_file(:tables, :vacuum_needed)
-        threshold = PgReports.config.dead_rows_threshold
-
-        filtered = data.select { |row| row["n_dead_tup"].to_i >= threshold }
-          .first(limit)
-
-        Report.new(
-          title: "Tables Needing Vacuum (dead rows >= #{threshold})",
-          data: filtered,
-          columns: %w[schema table_name n_live_tup n_dead_tup last_vacuum last_autovacuum]
-        )
-      end
-
-      # Table row counts
-      # @return [Report] Report with table row counts
-      def row_counts(limit: 50)
-        data = executor.execute_from_file(:tables, :row_counts)
-          .first(limit)
-
-        Report.new(
-          title: "Table Row Counts (top #{limit})",
-          data: data,
-          columns: %w[schema table_name row_count table_size_mb]
-        )
-      end
-
-      # Table cache hit ratios
-      # @return [Report] Report with table cache hit ratios
-      def cache_hit_ratios(limit: 50)
-        data = executor.execute_from_file(:tables, :cache_hit_ratios)
-          .first(limit)
-
-        Report.new(
-          title: "Table Cache Hit Ratios",
-          data: data,
-          columns: %w[schema table_name heap_blks_read heap_blks_hit cache_hit_ratio]
-        )
-      end
-
-      # Sequential scan statistics
-      # @return [Report] Report with sequential scan statistics
-      def seq_scans(limit: 20)
-        data = executor.execute_from_file(:tables, :seq_scans)
-          .first(limit)
-
-        Report.new(
-          title: "Sequential Scans (top #{limit})",
-          data: data,
-          columns: %w[schema table_name seq_scan seq_tup_read idx_scan rows_per_seq_scan]
-        )
-      end
-
-      # Recently modified tables
-      # @return [Report] Report with recently modified tables
-      def recently_modified(limit: 20)
-        data = executor.execute_from_file(:tables, :recently_modified)
-          .first(limit)
-
-        Report.new(
-          title: "Recently Modified Tables",
-          data: data,
-          columns: %w[schema table_name n_tup_ins n_tup_upd n_tup_del last_analyze]
-        )
-      end
+      # The following methods are auto-generated from YAML:
+      # - table_sizes(limit: 50)
+      # - bloated_tables(limit: 20)
+      # - vacuum_needed(limit: 20)
+      # - row_counts(limit: 50)
+      # - cache_hit_ratios(limit: 50)
+      # - seq_scans(limit: 20)
+      # - recently_modified(limit: 20)
 
       private
 

data/lib/pg_reports/report_definition.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module PgReports
+  # Parses YAML report definitions and generates Report objects
+  class ReportDefinition
+    attr_reader :config
+
+    def initialize(yaml_path)
+      @config = YAML.load_file(yaml_path)["report"]
+      @yaml_path = yaml_path
+    end
+
+    def generate_report(**params)
+      # 1. Execute SQL
+      data = execute_sql(**params)
+
+      # 2. Apply filters
+      data = apply_filters(data, params)
+
+      # 3. Apply enrichment
+      data = apply_enrichment(data) if enrichment?
+
+      # 4. Apply limit
+      limit = params[:limit] || default_limit
+      data = data.first(limit) if limit && data.respond_to?(:first)
+
+      # 5. Create Report
+      Report.new(
+        title: interpolate_title(params),
+        data: data,
+        columns: config["columns"]
+      )
+    end
+
+    private
+
+    def execute_sql(**params)
+      sql_config = config["sql"]
+      sql_params = extract_sql_params(params)
+
+      executor = Executor.new
+      executor.execute_from_file(
+        sql_config["category"].to_sym,
+        sql_config["file"].to_sym,
+        **sql_params
+      )
+    end
+
+    def extract_sql_params(params)
+      return {} unless config["sql"]["params"]
+
+      config["sql"]["params"].each_with_object({}) do |(key, value_config), result|
+        result[key.to_sym] = resolve_value(value_config, params)
+      end
+    end
+
+    def apply_filters(data, params)
+      return data unless config["filters"]
+
+      config["filters"].reduce(data) do |filtered, filter_config|
+        Filter.new(filter_config).apply(filtered, params)
+      end
+    end
+
+    def enrichment?
+      config["enrichment"].present?
+    end
+
+    def apply_enrichment(data)
+      enrichment = config["enrichment"]
+      module_name = enrichment["module"]
+      hook_name = enrichment["hook"]
+
+      # Call private method from module
+      module_class = PgReports::Modules.const_get(module_name.capitalize)
+      module_class.send(hook_name, data)
+    end
+
+    def interpolate_title(params)
+      title = config["title"]
+      return title unless config["title_vars"]
+
+      config["title_vars"].each do |var_name, var_config|
+        value = resolve_value(var_config, params)
+        title = title.gsub("${#{var_name}}", value.to_s)
+      end
+
+      title
+    end
+
+    def resolve_value(value_config, params)
+      case value_config["source"]
+      when "config"
+        PgReports.config.public_send(value_config["key"])
+      when "param"
+        key = value_config["key"].to_sym
+        # Try to get from params, fallback to default value
+        params[key] || get_default_param_value(key)
+      else
+        raise ArgumentError, "Unknown value source: #{value_config["source"]}"
+      end
+    end
+
+    def get_default_param_value(param_key)
+      return nil unless config["parameters"]&.dig(param_key.to_s)
+
+      config["parameters"][param_key.to_s]["default"]
+    end
+
+    def default_limit
+      return nil unless config["parameters"]&.dig("limit")
+
+      config["parameters"]["limit"]["default"]
+    end
+
+    public
+
+    # Extract filter parameters for UI
+    def filter_parameters
+      params = {}
+
+      # Parameters from parameters section
+      if config["parameters"]
+        config["parameters"].each do |name, param_config|
+          params[name] = {
+            type: param_config["type"],
+            default: param_config["default"],
+            description: param_config["description"],
+            label: name.to_s.titleize
+          }
+        end
+      end
+
+      # Add threshold parameters from filters (config-based)
+      if config["filters"]
+        config["filters"].each do |filter|
+          if filter["value"]["source"] == "config"
+            config_key = filter["value"]["key"]
+            field_name = filter["field"]
+
+            params["#{field_name}_threshold"] = {
+              type: filter["cast"] || "integer",
+              default: PgReports.config.public_send(config_key),
+              description: "Override threshold for #{field_name}",
+              label: "#{field_name.titleize} Threshold",
+              current_config: PgReports.config.public_send(config_key),
+              is_threshold: true
+            }
+          end
+        end
+      end
+
+      params
+    end
+
+    # Extract problem explanations mapping from YAML
+    def problem_explanations
+      config["problem_explanations"] || {}
+    end
+  end
+end

data/lib/pg_reports/report_loader.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module PgReports
+  # Loads YAML report definitions from the definitions directory
+  class ReportLoader
+    def self.load_all
+      @definitions ||= begin
+        definitions = {}
+
+        Dir.glob(definitions_path.join("**/*.yml")).each do |yaml_file|
+          definition = ReportDefinition.new(yaml_file)
+          module_name = definition.config["module"]
+          report_name = definition.config["name"]
+
+          definitions[module_name] ||= {}
+          definitions[module_name][report_name] = definition
+        end
+
+        definitions
+      end
+    end
+
+    def self.get(module_name, report_name)
+      load_all.dig(module_name.to_s, report_name.to_s)
+    end
+
+    def self.definitions_path
+      Pathname.new(__dir__).join("definitions")
+    end
+
+    def self.reload!
+      @definitions = nil
+      load_all
+    end
+  end
+end