pg_reports 0.3.1 → 0.5.0

data/lib/pg_reports/query_monitor.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "singleton"
+require "securerandom"
+require "json"
+
+module PgReports
+  class QueryMonitor
+    include Singleton
+
+    attr_reader :enabled, :session_id
+
+    def initialize
+      @enabled = false
+      @subscriber = nil
+      @mutex = Mutex.new
+      @session_id = nil
+      @queries = []
+    end
+
+    def start
+      @mutex.synchronize do
+        if @enabled
+          return {success: false, message: "Monitoring already active"}
+        end
+
+        @session_id = SecureRandom.uuid
+        @queries = []
+        @enabled = true
+
+        # Subscribe to sql.active_record events
+        @subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |name, started, finished, unique_id, payload|
+          handle_sql_event(name, started, finished, unique_id, payload)
+        end
+
+        # Write session start marker to file
+        write_session_marker("session_start")
+
+        {success: true, message: "Query monitoring started", session_id: @session_id}
+      end
+    rescue => e
+      @enabled = false
+      {success: false, error: e.message}
+    end
+
+    def stop
+      @mutex.synchronize do
+        unless @enabled
+          return {success: false, message: "Monitoring not active"}
+        end
+
+        # Unsubscribe from notifications
+        if @subscriber
+          ActiveSupport::Notifications.unsubscribe(@subscriber)
+          @subscriber = nil
+        end
+
+        # Write session end marker to file
+        write_session_marker("session_end")
+
+        # Flush queries to file
+        flush_to_file
+
+        @enabled = false
+        @queries = []
+        session_id = @session_id
+        @session_id = nil
+
+        {success: true, message: "Query monitoring stopped", session_id: session_id}
+      end
+    rescue => e
+      {success: false, error: e.message}
+    end
+
+    def status
+      {
+        enabled: @enabled,
+        session_id: @session_id,
+        query_count: @queries.size
+      }
+    end
+
+    def queries(limit: nil, session_id: nil)
+      result = @queries.dup
+
+      # Filter by session_id if provided
+      if session_id
+        result = result.select { |q| q[:session_id] == session_id }
+      end
+
+      # Limit results if requested
+      if limit
+        result = result.last(limit)
+      end
+
+      result
+    end
+
+    def load_from_log(session_id: nil, limit: nil)
+      return [] unless log_file_enabled?
+      return [] unless File.exist?(log_file_path)
+
+      queries = []
+
+      begin
+        File.open(log_file_path, "r") do |f|
+          f.each_line do |line|
+            entry = JSON.parse(line.strip, symbolize_names: true)
+            next unless entry[:type] == "query"
+
+            # Filter by session_id if provided
+            next if session_id && entry[:session_id] != session_id
+
+            queries << entry
+          rescue JSON::ParserError
+            # Skip malformed lines
+            next
+          end
+        end
+
+        # Limit results if requested
+        queries = queries.last(limit) if limit
+
+        queries
+      rescue => e
+        Rails.logger.warn("PgReports: Failed to load queries from log: #{e.message}")
+        []
+      end
+    end
+
+    private
+
+    def handle_sql_event(name, started, finished, unique_id, payload)
+      return unless @enabled
+
+      # Skip if should be filtered
+      return if should_skip?(payload)
+
+      duration_ms = ((finished - started) * 1000).round(2)
+      sql = payload[:sql]
+      query_name = payload[:name]
+
+      # Extract source location
+      source_location = extract_source_location
+
+      # Build query entry
+      query_entry = {
+        type: "query",
+        session_id: @session_id,
+        sql: sql,
+        duration_ms: duration_ms,
+        name: query_name,
+        source_location: source_location,
+        timestamp: Time.current.iso8601
+      }
+
+      add_to_buffer(query_entry)
+    end
+
+    def should_skip?(payload)
+      sql = payload[:sql]
+      name = payload[:name]
+
+      # Skip if query is from pg_reports itself (check by name)
+      return true if name&.start_with?("PgReports")
+
+      # Skip if query is from pg_reports gem (check backtrace)
+      if query_from_pg_reports?
+        return true
+      end
+
+      # Skip SCHEMA queries
+      return true if name&.start_with?("SCHEMA")
+
+      # Skip CACHE queries
+      return true if name == "CACHE"
+
+      # Skip if cached
+      return true if payload[:cached]
+
+      # Skip EXPLAIN queries
+      return true if sql&.match?(/\bEXPLAIN\b/i)
+
+      # Skip DDL statements
+      return true if sql&.match?(/\b(CREATE|ALTER|DROP)\b/i)
+
+      false
+    end
+
+    def query_from_pg_reports?
+      # Check if query originates from pg_reports gem code (not tests)
+      locations = caller_locations(0, 30)
+      return false unless locations
+
+      locations.any? do |location|
+        path = location.path
+        # Match gem paths: /gems/pg_reports-X.Y.Z/lib/ or local /lib/pg_reports/
+        # But exclude test paths: /spec/
+        next if path.include?("/spec/")
+
+        # Match both gem installation and local development lib paths
+        path.match?(%r{/gems/pg_reports[-\d.]+/lib/}) ||
+          path.match?(%r{/lib/pg_reports/})
+      end
+    end
+
+    def extract_source_location
+      filter_proc = PgReports.config.query_monitor_backtrace_filter
+
+      # Get caller locations, skip first few frames (this file, active_support)
+      locations = caller_locations(5, 20)
+
+      return nil unless locations
+
+      # Find first application code location
+      app_location = locations.find do |location|
+        filter_proc.call(location)
+      end
+
+      return nil unless app_location
+
+      {
+        file: app_location.path,
+        line: app_location.lineno,
+        method: app_location.label
+      }
+    rescue
+      # If source extraction fails, return nil
+      nil
+    end
+
+    def add_to_buffer(query_entry)
+      @queries << query_entry
+
+      # Trim to max_queries to prevent memory bloat
+      max_queries = PgReports.config.query_monitor_max_queries
+      if @queries.size > max_queries
+        @queries = @queries.last(max_queries)
+      end
+    end
+
+    def write_session_marker(marker_type)
+      return unless log_file_enabled?
+
+      marker = {
+        type: marker_type,
+        session_id: @session_id,
+        timestamp: Time.current.iso8601
+      }
+
+      File.open(log_file_path, "a") do |f|
+        f.puts marker.to_json
+      end
+    rescue => e
+      # Silently fail - don't break monitoring if file write fails
+      Rails.logger.warn("PgReports: Failed to write session marker: #{e.message}")
+    end
+
+    def flush_to_file
+      return unless log_file_enabled?
+      return if @queries.empty?
+
+      File.open(log_file_path, "a") do |f|
+        @queries.each do |query|
+          f.puts query.to_json
+        end
+      end
+    rescue => e
+      Rails.logger.warn("PgReports: Failed to flush queries to file: #{e.message}")
+    end
+
+    def log_file_enabled?
+      PgReports.config.query_monitor_log_file.present?
+    end
+
+    def log_file_path
+      PgReports.config.query_monitor_log_file
+    end
+  end
+end

data/lib/pg_reports/report_definition.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module PgReports
+  # Parses YAML report definitions and generates Report objects
+  class ReportDefinition
+    attr_reader :config
+
+    def initialize(yaml_path)
+      @config = YAML.load_file(yaml_path)["report"]
+      @yaml_path = yaml_path
+    end
+
+    def generate_report(**params)
+      # 1. Execute SQL
+      data = execute_sql(**params)
+
+      # 2. Apply filters
+      data = apply_filters(data, params)
+
+      # 3. Apply enrichment
+      data = apply_enrichment(data) if enrichment?
+
+      # 4. Apply limit
+      limit = params[:limit] || default_limit
+      data = data.first(limit) if limit && data.respond_to?(:first)
+
+      # 5. Create Report
+      Report.new(
+        title: interpolate_title(params),
+        data: data,
+        columns: config["columns"]
+      )
+    end
+
+    private
+
+    def execute_sql(**params)
+      sql_config = config["sql"]
+      sql_params = extract_sql_params(params)
+
+      executor = Executor.new
+      executor.execute_from_file(
+        sql_config["category"].to_sym,
+        sql_config["file"].to_sym,
+        **sql_params
+      )
+    end
+
+    def extract_sql_params(params)
+      return {} unless config["sql"]["params"]
+
+      config["sql"]["params"].each_with_object({}) do |(key, value_config), result|
+        result[key.to_sym] = resolve_value(value_config, params)
+      end
+    end
+
+    def apply_filters(data, params)
+      return data unless config["filters"]
+
+      config["filters"].reduce(data) do |filtered, filter_config|
+        Filter.new(filter_config).apply(filtered, params)
+      end
+    end
+
+    def enrichment?
+      config["enrichment"].present?
+    end
+
+    def apply_enrichment(data)
+      enrichment = config["enrichment"]
+      module_name = enrichment["module"]
+      hook_name = enrichment["hook"]
+
+      # Call private method from module
+      module_class = PgReports::Modules.const_get(module_name.capitalize)
+      module_class.send(hook_name, data)
+    end
+
+    def interpolate_title(params)
+      title = config["title"]
+      return title unless config["title_vars"]
+
+      config["title_vars"].each do |var_name, var_config|
+        value = resolve_value(var_config, params)
+        title = title.gsub("${#{var_name}}", value.to_s)
+      end
+
+      title
+    end
+
+    def resolve_value(value_config, params)
+      case value_config["source"]
+      when "config"
+        PgReports.config.public_send(value_config["key"])
+      when "param"
+        key = value_config["key"].to_sym
+        # Try to get from params, fallback to default value
+        params[key] || get_default_param_value(key)
+      else
+        raise ArgumentError, "Unknown value source: #{value_config["source"]}"
+      end
+    end
+
+    def get_default_param_value(param_key)
+      return nil unless config["parameters"]&.dig(param_key.to_s)
+
+      config["parameters"][param_key.to_s]["default"]
+    end
+
+    def default_limit
+      return nil unless config["parameters"]&.dig("limit")
+
+      config["parameters"]["limit"]["default"]
+    end
+
+    public
+
+    # Extract filter parameters for UI
+    def filter_parameters
+      params = {}
+
+      # Parameters from parameters section
+      if config["parameters"]
+        config["parameters"].each do |name, param_config|
+          params[name] = {
+            type: param_config["type"],
+            default: param_config["default"],
+            description: param_config["description"],
+            label: name.to_s.titleize
+          }
+        end
+      end
+
+      # Add threshold parameters from filters (config-based)
+      if config["filters"]
+        config["filters"].each do |filter|
+          if filter["value"]["source"] == "config"
+            config_key = filter["value"]["key"]
+            field_name = filter["field"]
+
+            params["#{field_name}_threshold"] = {
+              type: filter["cast"] || "integer",
+              default: PgReports.config.public_send(config_key),
+              description: "Override threshold for #{field_name}",
+              label: "#{field_name.titleize} Threshold",
+              current_config: PgReports.config.public_send(config_key),
+              is_threshold: true
+            }
+          end
+        end
+      end
+
+      params
+    end
+
+    # Extract problem explanations mapping from YAML
+    def problem_explanations
+      config["problem_explanations"] || {}
+    end
+  end
+end

data/lib/pg_reports/report_loader.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module PgReports
+  # Loads YAML report definitions from the definitions directory
+  class ReportLoader
+    def self.load_all
+      @definitions ||= begin
+        definitions = {}
+
+        Dir.glob(definitions_path.join("**/*.yml")).each do |yaml_file|
+          definition = ReportDefinition.new(yaml_file)
+          module_name = definition.config["module"]
+          report_name = definition.config["name"]
+
+          definitions[module_name] ||= {}
+          definitions[module_name][report_name] = definition
+        end
+
+        definitions
+      end
+    end
+
+    def self.get(module_name, report_name)
+      load_all.dig(module_name.to_s, report_name.to_s)
+    end
+
+    def self.definitions_path
+      Pathname.new(__dir__).join("definitions")
+    end
+
+    def self.reload!
+      @definitions = nil
+      load_all
+    end
+  end
+end

data/lib/pg_reports/sql/connections/connection_churn.sql

@@ -0,0 +1,37 @@
+-- Connection Churn Analysis
+-- Identifies applications with excessive connection turnover
+
+WITH connection_ages AS (
+  SELECT
+    datname AS database,
+    COALESCE(application_name, 'unknown') AS application,
+    EXTRACT(EPOCH FROM (NOW() - backend_start)) AS connection_age_seconds
+  FROM pg_stat_activity
+  WHERE pid != pg_backend_pid()
+    AND datname IS NOT NULL
+    AND backend_type = 'client backend'
+),
+connection_stats AS (
+  SELECT
+    database,
+    application,
+    COUNT(*) AS total_connections,
+    ROUND(AVG(connection_age_seconds)::numeric, 2) AS avg_connection_age_seconds,
+    ROUND(MIN(connection_age_seconds)::numeric, 2) AS min_connection_age_seconds,
+    ROUND(MAX(connection_age_seconds)::numeric, 2) AS max_connection_age_seconds,
+    COUNT(*) FILTER (WHERE connection_age_seconds < 10) AS short_lived_connections
+  FROM connection_ages
+  GROUP BY database, application
+)
+SELECT
+  database,
+  application,
+  total_connections,
+  avg_connection_age_seconds,
+  min_connection_age_seconds,
+  max_connection_age_seconds,
+  short_lived_connections,
+  ROUND((short_lived_connections::numeric / total_connections::numeric) * 100, 2) AS churn_rate_pct
+FROM connection_stats
+WHERE total_connections > 1  -- Filter out single connections
+ORDER BY churn_rate_pct DESC, short_lived_connections DESC;

data/lib/pg_reports/sql/connections/pool_saturation.sql

@@ -0,0 +1,90 @@
+-- Connection Pool Saturation Analysis
+-- Provides overall pool health metrics with warnings
+
+WITH settings AS (
+  SELECT
+    (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') AS max_conn,
+    (SELECT setting::int FROM pg_settings WHERE name = 'superuser_reserved_connections') AS reserved_conn
+),
+current_state AS (
+  SELECT
+    COUNT(*) AS total_connections,
+    COUNT(*) FILTER (WHERE state = 'active') AS active_conn,
+    COUNT(*) FILTER (WHERE state = 'idle') AS idle_conn,
+    COUNT(*) FILTER (WHERE state = 'idle in transaction') AS idle_in_txn,
+    COUNT(*) FILTER (WHERE state = 'idle in transaction (aborted)') AS idle_in_txn_aborted,
+    COUNT(*) FILTER (WHERE wait_event IS NOT NULL) AS waiting_conn
+  FROM pg_stat_activity
+  WHERE pid != pg_backend_pid()
+),
+metrics AS (
+  SELECT
+    'Total Connections' AS metric,
+    cs.total_connections AS current_value,
+    s.max_conn - s.reserved_conn AS max_value,
+    ROUND((cs.total_connections::numeric / (s.max_conn - s.reserved_conn)::numeric) * 100, 2) AS utilization_pct
+  FROM current_state cs, settings s
+
+  UNION ALL
+
+  SELECT
+    'Active Connections',
+    cs.active_conn,
+    s.max_conn - s.reserved_conn,
+    ROUND((cs.active_conn::numeric / (s.max_conn - s.reserved_conn)::numeric) * 100, 2)
+  FROM current_state cs, settings s
+
+  UNION ALL
+
+  SELECT
+    'Idle Connections',
+    cs.idle_conn,
+    s.max_conn - s.reserved_conn,
+    ROUND((cs.idle_conn::numeric / (s.max_conn - s.reserved_conn)::numeric) * 100, 2)
+  FROM current_state cs, settings s
+
+  UNION ALL
+
+  SELECT
+    'Idle in Transaction',
+    cs.idle_in_txn,
+    s.max_conn / 4, -- Should be max 25% of pool
+    ROUND((cs.idle_in_txn::numeric / (s.max_conn / 4)::numeric) * 100, 2)
+  FROM current_state cs, settings s
+
+  UNION ALL
+
+  SELECT
+    'Waiting Connections',
+    cs.waiting_conn,
+    s.max_conn / 10, -- Should be max 10% of pool
+    ROUND((cs.waiting_conn::numeric / GREATEST(s.max_conn / 10, 1)::numeric) * 100, 2)
+  FROM current_state cs, settings s
+)
+SELECT
+  metric,
+  current_value,
+  max_value,
+  utilization_pct,
+  CASE
+    WHEN utilization_pct >= 85 THEN '🔴 Critical'
+    WHEN utilization_pct >= 70 THEN '🟡 Warning'
+    WHEN utilization_pct >= 50 THEN '🟡 Elevated'
+    ELSE '🟢 Normal'
+  END AS status,
+  CASE
+    WHEN metric = 'Total Connections' AND utilization_pct >= 85 THEN
+      'CRITICAL: Pool near exhaustion. Scale up max_connections or implement connection pooling immediately.'
+    WHEN metric = 'Total Connections' AND utilization_pct >= 70 THEN
+      'WARNING: High pool utilization. Consider increasing max_connections or adding connection pooler.'
+    WHEN metric = 'Idle in Transaction' AND utilization_pct >= 70 THEN
+      'WARNING: Too many idle in transaction connections. Review application transaction handling and set idle_in_transaction_session_timeout.'
+    WHEN metric = 'Waiting Connections' AND utilization_pct >= 70 THEN
+      'WARNING: Many connections waiting. Check for lock contention and long-running queries.'
+    WHEN metric = 'Active Connections' AND utilization_pct >= 85 THEN
+      'High active connection count. Monitor query performance and consider read replicas.'
+    ELSE
+      'Pool is healthy.'
+  END AS recommendation
+FROM metrics
+ORDER BY utilization_pct DESC;

data/lib/pg_reports/sql/connections/pool_usage.sql

@@ -0,0 +1,31 @@
+-- Connection Pool Usage Statistics
+-- Shows current pool utilization across databases
+
+WITH connection_counts AS (
+  SELECT
+    COALESCE(datname, 'system') AS database,
+    COUNT(*) AS total_connections,
+    COUNT(*) FILTER (WHERE state = 'active') AS active_connections,
+    COUNT(*) FILTER (WHERE state = 'idle') AS idle_connections,
+    COUNT(*) FILTER (WHERE state = 'idle in transaction') AS idle_in_transaction
+  FROM pg_stat_activity
+  WHERE pid != pg_backend_pid()
+  GROUP BY datname
+),
+limits AS (
+  SELECT setting::int AS max_connections
+  FROM pg_settings
+  WHERE name = 'max_connections'
+)
+SELECT
+  cc.database,
+  cc.total_connections,
+  cc.active_connections,
+  cc.idle_connections,
+  cc.idle_in_transaction,
+  l.max_connections,
+  ROUND((cc.total_connections::numeric / l.max_connections::numeric) * 100, 2) AS utilization_pct,
+  l.max_connections - cc.total_connections AS available_connections
+FROM connection_counts cc
+CROSS JOIN limits l
+ORDER BY cc.total_connections DESC;

data/lib/pg_reports/sql/connections/pool_wait_times.sql

@@ -0,0 +1,19 @@
+-- Connection Pool Wait Time Analysis
+-- Shows queries currently waiting for resources
+
+SELECT
+  pid,
+  datname AS database,
+  usename AS username,
+  wait_event_type,
+  wait_event,
+  state,
+  ROUND(EXTRACT(EPOCH FROM (NOW() - state_change))::numeric, 2) AS wait_duration_seconds,
+  state_change AS query_start,
+  LEFT(query, 500) AS query
+FROM pg_stat_activity
+WHERE wait_event IS NOT NULL
+  AND pid != pg_backend_pid()
+  AND datname IS NOT NULL
+  AND state != 'idle'
+ORDER BY wait_duration_seconds DESC;

data/lib/pg_reports/sql/queries/all_queries.sql

@@ -2,19 +2,21 @@
 -- Requires pg_stat_statements extension
 
 SELECT
-  query,
-  calls,
-  ROUND((total_exec_time)::numeric, 2) AS total_time_ms,
-  ROUND((mean_exec_time)::numeric, 2) AS mean_time_ms,
-  ROUND((min_exec_time)::numeric, 2) AS min_time_ms,
-  ROUND((max_exec_time)::numeric, 2) AS max_time_ms,
-  ROUND((stddev_exec_time)::numeric, 2) AS stddev_time_ms,
-  rows,
-  shared_blks_hit,
-  shared_blks_read,
-  ROUND((shared_blks_hit * 100.0 / NULLIF(shared_blks_hit + shared_blks_read, 0))::numeric, 2) AS cache_hit_ratio
-FROM pg_stat_statements
-WHERE calls > 0
-  AND query NOT LIKE '%pg_stat_statements%'
-ORDER BY total_exec_time DESC
+  s.query,
+  s.calls,
+  ROUND((s.total_exec_time)::numeric, 2) AS total_time_ms,
+  ROUND((s.mean_exec_time)::numeric, 2) AS mean_time_ms,
+  ROUND((s.min_exec_time)::numeric, 2) AS min_time_ms,
+  ROUND((s.max_exec_time)::numeric, 2) AS max_time_ms,
+  ROUND((s.stddev_exec_time)::numeric, 2) AS stddev_time_ms,
+  s.rows,
+  s.shared_blks_hit,
+  s.shared_blks_read,
+  ROUND((s.shared_blks_hit * 100.0 / NULLIF(s.shared_blks_hit + s.shared_blks_read, 0))::numeric, 2) AS cache_hit_ratio
+FROM pg_stat_statements s
+JOIN pg_database d ON s.dbid = d.oid
+WHERE s.calls > 0
+  AND s.query NOT LIKE '%pg_stat_statements%'
+  AND d.datname = current_database()
+ORDER BY s.total_exec_time DESC
 LIMIT 200;