pg_reports 0.9.1 → 1.0.0

data/lib/pg_reports/connection/target.rb

@@ -13,6 +13,29 @@ module PgReports
     class Target
       class ConnectionFailed < PgReports::Error; end
 
+      DATABASE_SIZES_TTL = 60 # seconds
+
+      DATABASE_NAMES_SQL = <<~SQL
+        #{Executor::QUERY_TAG}
+        SELECT datname AS name
+        FROM pg_database
+        WHERE datistemplate = false AND datallowconn = true
+        ORDER BY datname
+      SQL
+
+      # pg_database_size raises for a database the role cannot connect to, which
+      # would take the whole list down over one unreadable size.
+      DATABASE_SIZES_SQL = <<~SQL
+        #{Executor::QUERY_TAG}
+        SELECT datname AS name,
+               CASE WHEN has_database_privilege(datname, 'CONNECT')
+                    THEN pg_size_pretty(pg_database_size(datname))
+               END AS size
+        FROM pg_database
+        WHERE datistemplate = false AND datallowconn = true
+        ORDER BY datname
+      SQL
+
       attr_reader :name
 
       def initialize(name, spec)
@@ -48,17 +71,15 @@ module PgReports
 
       # List all databases visible on this target's cluster (using pg_database).
       # Result rows: { "name" => String, "size" => String, "current" => Boolean }
-      def list_databases(current: nil)
-        rows = connection_for.exec_query(<<~SQL, "PgReports").to_a
-          SELECT datname AS name,
-                 pg_size_pretty(pg_database_size(datname)) AS size
-          FROM pg_database
-          WHERE datistemplate = false AND datallowconn = true
-          ORDER BY datname
-        SQL
+      #
+      # pg_database_size walks every file of every database, which on a cluster with
+      # many tables takes far longer than anything else the dashboard asks for. Only
+      # a page showing the selector wants the sizes, and they change slowly, so ask
+      # for `sizes: false` to validate a name and take cached sizes otherwise.
+      def list_databases(current: nil, sizes: true)
+        rows = sizes ? databases_with_sizes : connection_for.exec_query(DATABASE_NAMES_SQL, "PgReports").to_a
         current_db = (current || default_database).to_s
-        rows.each { |r| r["current"] = (r["name"].to_s == current_db) }
-        rows
+        rows.map { |r| r.merge("current" => r["name"].to_s == current_db) }
       end
 
       # Close all derived pools we own. The :primary AR::Base pool is never touched.
@@ -70,10 +91,22 @@ module PgReports
           # Best-effort cleanup
         end
         @pools.clear
+        @sized_databases = nil
       end
 
       private
 
+      def databases_with_sizes
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        cached = @sized_databases
+
+        return cached[:rows] if cached && now - cached[:at] < DATABASE_SIZES_TTL
+
+        rows = connection_for.exec_query(DATABASE_SIZES_SQL, "PgReports").to_a
+        @sized_databases = {at: now, rows: rows}
+        rows
+      end
+
       def normalize_spec(spec)
         hash = case spec
         when Hash

data/lib/pg_reports/csv_cell.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module PgReports
+  # Keeps database text from running as a formula when a CSV opens in a spreadsheet.
+  #
+  # A table named =HYPERLINK(...) is ordinary data here and a live formula in Excel,
+  # so anyone able to name a table or send a query could reach whoever opens the
+  # export. A leading apostrophe makes the spreadsheet read the cell as text. The
+  # exported value changes only for cells that would otherwise be evaluated.
+  module CsvCell
+    FORMULA_START = /\A\s*[=+\-@]/
+    NUMBER = /\A\s*[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?\s*\z/i
+
+    def self.safe(value)
+      return value unless value.is_a?(String) && value.match?(FORMULA_START) && !value.match?(NUMBER)
+
+      "'#{value}"
+    end
+  end
+end

data/lib/pg_reports/dashboard/reports_registry.rb

@@ -16,16 +16,20 @@ module PgReports
           thresholds: {calls: {warning: 10000, critical: 100000}},
           problem_fields: ["calls"]
         },
+        frequent_queries: {
+          thresholds: {calls_per_second: {warning: 100, critical: 1000}},
+          problem_fields: ["calls_per_second"]
+        },
         expensive_queries: {
           thresholds: {total_time_ms: {warning: 60000, critical: 300000}},
           problem_fields: ["total_time_ms"]
         },
         missing_index_queries: {
-          thresholds: {seq_tup_read: {warning: 100000, critical: 1000000}},
-          problem_fields: ["seq_tup_read", "seq_scan"]
+          thresholds: {rows_per_call: {warning: 1000, critical: 10000}},
+          problem_fields: ["rows_per_call"]
         },
         low_cache_hit_queries: {
-          thresholds: {cache_hit_ratio: {warning: 0.95, critical: 0.80, inverted: true}},
+          thresholds: {cache_hit_ratio: {warning: 95, critical: 80, inverted: true}},
           problem_fields: ["cache_hit_ratio"]
         },
         all_queries: {
@@ -51,8 +55,8 @@ module PgReports
           problem_fields: []
         },
         missing_indexes: {
-          thresholds: {seq_scan_ratio: {warning: 0.5, critical: 0.9}},
-          problem_fields: ["seq_scan", "seq_tup_read"]
+          thresholds: {avg_seq_tup_read: {warning: 10000, critical: 100000}},
+          problem_fields: ["avg_seq_tup_read"]
         },
         inefficient_indexes: {
           thresholds: {read_to_fetch_ratio: {warning: 10, critical: 100}},
@@ -62,31 +66,33 @@ module PgReports
           thresholds: {},
           problem_fields: []
         },
+        # No limits until the estimate improves: it assumes 8 bytes per entry, so a
+        # freshly built index reads as 65-75% bloated and every row would be critical
         bloated_indexes: {
-          thresholds: {bloat_ratio: {warning: 0.3, critical: 0.5}},
-          problem_fields: ["bloat_ratio", "bloat_size"]
+          thresholds: {},
+          problem_fields: []
         },
         index_sizes: {
-          thresholds: {size_bytes: {warning: 1073741824, critical: 10737418240}},
-          problem_fields: ["size_bytes"]
+          thresholds: {index_size_mb: {warning: 1024, critical: 10240}},
+          problem_fields: ["index_size_mb"]
         },
         fk_without_indexes: {
           thresholds: {},
           problem_fields: ["child_table_size_mb"]
         },
         index_correlation: {
-          thresholds: {correlation: {warning: 0.5, critical: 0.2, inverted: true}},
-          problem_fields: ["correlation"]
+          thresholds: {abs_correlation: {warning: 0.5, critical: 0.2, inverted: true}},
+          problem_fields: ["abs_correlation"]
         },
 
         # === TABLES ===
         table_sizes: {
-          thresholds: {total_size_bytes: {warning: 10737418240, critical: 107374182400}},
-          problem_fields: ["total_size_bytes"]
+          thresholds: {total_size_mb: {warning: 10240, critical: 102400}},
+          problem_fields: ["total_size_mb"]
         },
         bloated_tables: {
-          thresholds: {dead_tuple_ratio: {warning: 0.1, critical: 0.2}},
-          problem_fields: ["dead_tuple_ratio", "n_dead_tup"]
+          thresholds: {bloat_percent: {warning: 10, critical: 20}},
+          problem_fields: ["bloat_percent"]
         },
         vacuum_needed: {
           thresholds: {n_dead_tup: {warning: 10000, critical: 100000}},
@@ -97,7 +103,7 @@ module PgReports
           problem_fields: []
         },
         cache_hit_ratios: {
-          thresholds: {cache_hit_ratio: {warning: 0.95, critical: 0.80, inverted: true}},
+          thresholds: {cache_hit_ratio: {warning: 95, critical: 80, inverted: true}},
           problem_fields: ["cache_hit_ratio"]
         },
         seq_scans: {
@@ -115,8 +121,8 @@ module PgReports
 
         # === CONNECTIONS ===
         active_connections: {
-          thresholds: {connection_count: {warning: 50, critical: 100}},
-          problem_fields: ["connection_count"]
+          thresholds: {},
+          problem_fields: []
         },
         connection_stats: {
           thresholds: {idle_in_transaction: {warning: 5, critical: 20}},
@@ -124,19 +130,19 @@ module PgReports
         },
         long_running_queries: {
           thresholds: {duration_seconds: {warning: 60, critical: 300}},
-          problem_fields: ["duration_seconds", "duration"]
+          problem_fields: ["duration_seconds"]
         },
         blocking_queries: {
-          thresholds: {blocked_count: {warning: 1, critical: 5}},
-          problem_fields: ["blocked_count"]
+          thresholds: {blocked_duration: {warning: 10, critical: 60}},
+          problem_fields: ["blocked_duration"]
         },
         locks: {
-          thresholds: {waiting_locks: {warning: 5, critical: 20}},
-          problem_fields: ["waiting_locks"]
+          thresholds: {waiting_seconds: {warning: 10, critical: 60}},
+          problem_fields: ["waiting_seconds"]
         },
         idle_connections: {
-          thresholds: {idle_count: {warning: 30, critical: 80}},
-          problem_fields: ["idle_count"]
+          thresholds: {idle_in_transaction_seconds: {warning: 60, critical: 300}},
+          problem_fields: ["idle_in_transaction_seconds"]
         },
         pool_usage: {
           thresholds: {utilization_pct: {warning: 70, critical: 85}},
@@ -160,8 +166,8 @@ module PgReports
 
         # === SYSTEM ===
         database_sizes: {
-          thresholds: {size_bytes: {warning: 10737418240, critical: 107374182400}},
-          problem_fields: ["size_bytes"]
+          thresholds: {size_mb: {warning: 10240, critical: 102400}},
+          problem_fields: ["size_mb"]
         },
         settings: {
           thresholds: {},
@@ -176,8 +182,8 @@ module PgReports
           problem_fields: []
         },
         cache_stats: {
-          thresholds: {cache_hit_ratio: {warning: 0.95, critical: 0.90, inverted: true}},
-          problem_fields: ["cache_hit_ratio"]
+          thresholds: {heap_hit_ratio: {warning: 95, critical: 90, inverted: true}},
+          problem_fields: ["heap_hit_ratio"]
         },
         wraparound_risk: {
           thresholds: {pct_towards_wraparound: {warning: 50, critical: 75}},
@@ -228,7 +234,7 @@ module PgReports
         },
         unused_tables: {
           thresholds: {total_size_mb: {warning: 10, critical: 100}},
-          problem_fields: ["table_name"]
+          problem_fields: ["total_size_mb"]
         }
       }.freeze
 
@@ -244,6 +250,7 @@ module PgReports
           reports: {
             slow_queries: {name: "Slow Queries", description: "Queries with high mean execution time"},
             heavy_queries: {name: "Heavy Queries", description: "Most frequently called queries"},
+            frequent_queries: {name: "Most Frequent Queries", description: "Queries executed at the highest rate (calls per second)", new: true},
             expensive_queries: {name: "Expensive Queries", description: "Queries consuming most total time"},
             missing_index_queries: {name: "Missing Index Queries", description: "Queries potentially missing indexes"},
             low_cache_hit_queries: {name: "Low Cache Hit", description: "Queries with poor cache utilization"},

data/lib/pg_reports/definitions/connections/connection_stats.yml

@@ -16,3 +16,7 @@ report:
     - database
     - state
     - count
+    - idle_in_transaction
+
+  problem_explanations:
+    idle_in_transaction: idle_in_transaction

data/lib/pg_reports/definitions/connections/idle_connections.yml

@@ -18,4 +18,8 @@ report:
     - username
     - application
     - idle_duration
+    - idle_in_transaction_seconds
     - state_change
+
+  problem_explanations:
+    idle_in_transaction_seconds: idle_in_transaction

data/lib/pg_reports/definitions/connections/locks.yml

@@ -20,3 +20,4 @@ report:
     - mode
     - granted
     - waiting
+    - waiting_seconds

data/lib/pg_reports/definitions/connections/long_running_queries.yml

@@ -40,4 +40,3 @@ report:
 
   problem_explanations:
     duration_seconds: long_running
-    duration: long_running

data/lib/pg_reports/definitions/indexes/bloated_indexes.yml

@@ -39,5 +39,4 @@ report:
       cast: float
 
   problem_explanations:
-    bloat_ratio: high_bloat
-    bloat_size: high_bloat
+    bloat_percent: high_bloat

data/lib/pg_reports/definitions/indexes/index_correlation.yml

@@ -18,6 +18,7 @@ report:
     - column_name
     - index_name
     - correlation
+    - abs_correlation
     - table_size_mb
     - idx_scan
 
@@ -28,4 +29,4 @@ report:
       description: "Maximum number of results"
 
   problem_explanations:
-    correlation: low_correlation
+    abs_correlation: low_correlation

data/lib/pg_reports/definitions/indexes/missing_indexes.yml

@@ -17,6 +17,7 @@ report:
     - table_name
     - seq_scan
     - seq_tup_read
+    - avg_seq_tup_read
     - idx_scan
     - table_size_mb
 

data/lib/pg_reports/definitions/queries/frequent_queries.yml

@@ -0,0 +1,50 @@
+# Frequent queries - queries executed at the highest rate (calls per second)
+# Unlike heavy_queries, which ranks by the cumulative call count, this ranks by
+# rate, so a query's position does not depend on how long it has been tracked
+
+report:
+  name: frequent_queries
+  module: queries
+  description: "Queries executed at the highest rate"
+
+  sql:
+    category: queries
+    file: frequent_queries
+
+  title: "Most Frequent Queries (calls/sec >= ${threshold})"
+  title_vars:
+    threshold:
+      source: config
+      key: frequent_query_threshold_per_second
+
+  columns:
+    - query
+    - source
+    - calls_per_second
+    - calls
+    - mean_time_ms
+    - total_time_ms
+    - tracked_since
+
+  parameters:
+    limit:
+      type: integer
+      default: 20
+      description: "Maximum number of results"
+
+  filters:
+    - field: calls_per_second
+      operator: gte
+      value:
+        source: config
+        key: frequent_query_threshold_per_second
+      cast: float
+
+  enrichment:
+    hook: enrich_with_annotations
+    module: queries
+
+  problem_explanations:
+    calls_per_second: high_calls
+    total_time_ms: high_total_time
+    mean_time_ms: high_mean_time

data/lib/pg_reports/definitions/system/cache_stats.yml

@@ -15,4 +15,5 @@ report:
   columns:
     - database
     - heap_hit_ratio
-    - index_hit_ratio
+    - blks_hit
+    - blks_read

data/lib/pg_reports/definitions/tables/bloated_tables.yml

@@ -39,5 +39,4 @@ report:
       cast: float
 
   problem_explanations:
-    dead_tuple_ratio: many_dead_tuples
-    n_dead_tup: many_dead_tuples
+    bloat_percent: many_dead_tuples

data/lib/pg_reports/definitions/tables/recently_modified.yml

@@ -15,9 +15,9 @@ report:
   columns:
     - schema
     - table_name
-    - n_tup_ins
-    - n_tup_upd
-    - n_tup_del
+    - inserts
+    - updates
+    - deletes
     - last_analyze
 
   parameters:

data/lib/pg_reports/definitions/tables/unused_tables.yml

@@ -27,4 +27,4 @@ report:
       description: "Maximum number of results"
 
   problem_explanations:
-    table_name: unused_table
+    total_size_mb: unused_table

data/lib/pg_reports/drop_index_migration.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PgReports
+  # Writes the Rails migration that drops one index.
+  #
+  # Identifiers come from the database and anyone with DDL rights chooses them, so
+  # they only ever reach the generated Ruby through #inspect, which escapes quotes,
+  # newlines and `#{`. A name is never spliced in as a symbol or left bare: this
+  # file is executed by `rails db:migrate`.
+  #
+  # It drops with SQL rather than remove_index, which parses the name and, for one
+  # holding a quote or a dot, looks up something else. With if_exists that is a
+  # migration that reports success and leaves the index in place.
+  class DropIndexMigration
+    # @param index [Hash] a pg_indexes row: schemaname, indexname, indexdef
+    # @param version [#to_s] the Migration[] version, from the host's Rails
+    def initialize(index, version:)
+      @schema = index.fetch("schemaname")
+      @index = index.fetch("indexname")
+      @definition = index.fetch("indexdef")
+      @version = version
+    end
+
+    # Which migrations this would clash with: Rails refuses two of the same name.
+    def name
+      slug = @index.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/\A_+|_+\z/, "")
+      "remove_#{slug.empty? ? "index" : slug}"
+    end
+
+    def file_name(timestamp)
+      "#{timestamp}_#{name}.rb"
+    end
+
+    def code
+      <<~RUBY
+        # frozen_string_literal: true
+
+        class #{name.camelize} < ActiveRecord::Migration[#{@version}]
+          def up
+            execute #{drop_statement.inspect}
+          end
+
+          def down
+            execute #{@definition.inspect}
+          end
+        end
+      RUBY
+    end
+
+    private
+
+    def drop_statement
+      "DROP INDEX IF EXISTS #{PG::Connection.quote_ident([@schema, @index])}"
+    end
+  end
+end

data/lib/pg_reports/engine.rb

@@ -13,9 +13,10 @@ module PgReports
       config.i18n.load_path += Dir[root.join("config", "locales", "*.yml")]
     end
 
-    initializer "pg_reports.compatibility_check" do
-      ActiveSupport.on_load(:active_record) do
-        PgReports::Compatibility.check_postgresql!
+    # after_initialize so that the host's initializers have set the configuration
+    config.after_initialize do
+      if PgReports.config.query_monitor_enabled && PgReports.config.query_monitor_available_sources.include?(:notifications)
+        PgReports::QueryMonitor.instance.subscribe
       end
     end
 

data/lib/pg_reports/executor.rb

@@ -7,25 +7,39 @@ module PgReports
   # context set by PgReports.with_target / with_database is honored even when an
   # Executor instance has been memoized at the module level.
   class Executor
+    # Marks the SQL as ours where a statement name cannot reach: inside a host
+    # application these queries share its connections and application_name, so
+    # this is all that tells them apart in pg_stat_activity.
+    QUERY_TAG = "/* pg_reports */"
+
+    # A report that needs a lock someone else holds has nothing to wait for: during
+    # an incident the answer is more use now than after the full statement timeout.
+    REPORT_LOCK_TIMEOUT_MS = 3_000
+
+    # Not the second colon of a :: cast, nor the middle of a word
+    PLACEHOLDER = /(?<![:\w]):([a-zA-Z_]\w*)/
+
     def initialize(connection: nil)
       @connection_override = connection
     end
 
-    # Execute SQL from a file and return results as array of hashes
+    # Execute SQL from a file and return results as array of hashes.
+    #
+    # These files only read, so they run under the report time limits.
     def execute_from_file(category, name, **params)
       sql = SqlLoader.load(category, name)
-      execute(sql, **params)
+      within_report_limits { execute(sql, **params) }
     end
 
     # Execute raw SQL and return results as array of hashes.
     #
     # Every query is tagged with the "PgReports" AR statement name so the Query
-    # Monitor can skip our own queries by name (see QueryMonitor#should_skip?),
+    # Monitor can skip our own queries by name (see QueryMonitor#skip_by_payload?),
     # reliably and independent of backtrace depth — the internal live_metrics /
     # status polling would otherwise leak into the monitor's history.
     def execute(sql, **params)
       processed_sql = interpolate_params(sql, params)
-      result = connection.exec_query(processed_sql, "PgReports")
+      result = connection.exec_query("#{QUERY_TAG} #{processed_sql}", "PgReports")
       result.to_a
     end
 
@@ -36,14 +50,37 @@ module PgReports
 
     private
 
-    # Simple parameter interpolation for SQL
-    # Replaces :param_name with quoted values
+    # The limits are local to a transaction, and a savepoint inside the host
+    # application's transaction would hand them on to it when released. Rolling back
+    # discards them in both cases, and there is nothing here to keep.
+    def within_report_limits
+      timeout_ms = PgReports.config.report_statement_timeout_ms.to_i
+      return yield unless timeout_ms.positive?
+
+      rows = nil
+      connection.transaction(requires_new: true) do
+        connection.exec_query(report_limits_sql(timeout_ms), "PgReports")
+        rows = yield
+        raise ActiveRecord::Rollback
+      end
+      rows
+    end
+
+    def report_limits_sql(timeout_ms)
+      lock_timeout_ms = [timeout_ms, REPORT_LOCK_TIMEOUT_MS].min
+
+      "#{QUERY_TAG} SELECT set_config('statement_timeout', '#{timeout_ms}', true), " \
+        "set_config('lock_timeout', '#{lock_timeout_ms}', true)"
+    end
+
+    # Replaces :param_name with quoted values in a single pass, so a value is never
+    # read back as SQL: one holding ":other" stays text, and :limit leaves :limit_x alone.
     def interpolate_params(sql, params)
       return sql if params.empty?
 
-      params.reduce(sql) do |query, (key, value)|
-        quoted_value = quote_value(value)
-        query.gsub(":#{key}", quoted_value)
+      sql.gsub(PLACEHOLDER) do |placeholder|
+        key = Regexp.last_match(1).to_sym
+        params.key?(key) ? quote_value(params[key]) : placeholder
       end
     end
 

data/lib/pg_reports/modules/queries.rb

@@ -10,6 +10,7 @@ module PgReports
       # The following methods are auto-generated from YAML:
       # - slow_queries(limit: 20)
       # - heavy_queries(limit: 20)
+      # - frequent_queries(limit: 20)
       # - expensive_queries(limit: 20)
       # - missing_index_queries(limit: 20)
       # - low_cache_hit_queries(limit: 20, min_calls: 100)