pghero_fork 2.7.3

data/config/routes.rb

@@ -0,0 +1,32 @@
+PgHero::Engine.routes.draw do
+  scope "(:database)", constraints: proc { |req| (PgHero.config["databases"].keys + [nil]).include?(req.params[:database]) } do
+    get "space", to: "home#space"
+    get "space/:relation", to: "home#relation_space", as: :relation_space
+    get "index_bloat", to: "home#index_bloat"
+    get "live_queries", to: "home#live_queries"
+    get "queries", to: "home#queries"
+    get "queries/:query_hash", to: "home#show_query", as: :show_query
+    get "system", to: "home#system"
+    get "cpu_usage", to: "home#cpu_usage"
+    get "connection_stats", to: "home#connection_stats"
+    get "replication_lag_stats", to: "home#replication_lag_stats"
+    get "load_stats", to: "home#load_stats"
+    get "free_space_stats", to: "home#free_space_stats"
+    get "explain", to: "home#explain"
+    get "tune", to: "home#tune"
+    get "connections", to: "home#connections"
+    get "maintenance", to: "home#maintenance"
+    post "kill", to: "home#kill"
+    post "kill_long_running_queries", to: "home#kill_long_running_queries"
+    post "kill_all", to: "home#kill_all"
+    post "enable_query_stats", to: "home#enable_query_stats"
+    post "explain", to: "home#explain"
+    post "reset_query_stats", to: "home#reset_query_stats"
+
+    # legacy routes
+    get "system_stats" => redirect("system")
+    get "query_stats" => redirect("queries")
+
+    root to: "home#index"
+  end
+end

data/lib/generators/pghero/config_generator.rb

@@ -0,0 +1,13 @@
+require "rails/generators"
+
+module Pghero
+  module Generators
+    class ConfigGenerator < Rails::Generators::Base
+      source_root File.join(__dir__, "templates")
+
+      def create_initializer
+        template "config.yml", "config/pghero.yml"
+      end
+    end
+  end
+end

data/lib/generators/pghero/query_stats_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Pghero
+  module Generators
+    class QueryStatsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "query_stats.rb", "db/migrate/create_pghero_query_stats.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/pghero/space_stats_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators/active_record"
+
+module Pghero
+  module Generators
+    class SpaceStatsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      source_root File.join(__dir__, "templates")
+
+      def copy_migration
+        migration_template "space_stats.rb", "db/migrate/create_pghero_space_stats.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/generators/pghero/templates/config.yml.tt

@@ -0,0 +1,46 @@
+databases:
+  primary:
+    # Database URL (defaults to app database)
+    # url: <%%= ENV["DATABASE_URL"] %>
+
+    # System stats
+    # aws_db_instance_identifier: my-instance
+    # gcp_database_id: my-project:my-instance
+    # azure_resource_id: my-resource-id
+
+  # Add more databases
+  # other:
+  #   url: <%%= ENV["OTHER_DATABASE_URL"] %>
+
+# Minimum time for long running queries
+# long_running_query_sec: 60
+
+# Minimum average time for slow queries
+# slow_query_ms: 20
+
+# Minimum calls for slow queries
+# slow_query_calls: 100
+
+# Minimum connections for high connections warning
+# total_connections_threshold: 500
+
+# Statement timeout for explain
+# explain_timeout_sec: 10
+
+# Time zone (defaults to app time zone)
+# time_zone: "Pacific Time (US & Canada)"
+
+# Basic authentication
+# username: admin
+# password: <%%= ENV["PGHERO_PASSWORD"] %>
+
+# Stats database URL (defaults to app database)
+# stats_database_url: <%%= ENV["PGHERO_STATS_DATABASE_URL"] %>
+
+# AWS configuration (defaults to app AWS config)
+# aws_access_key_id: <%%= ENV["AWS_ACCESS_KEY_ID"] %>
+# aws_secret_access_key: <%%= ENV["AWS_SECRET_ACCESS_KEY"] %>
+# aws_region: us-east-1
+
+# Filter data from queries (experimental)
+# filter_data: true

data/lib/generators/pghero/templates/query_stats.rb.tt

@@ -0,0 +1,15 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pghero_query_stats do |t|
+      t.text :database
+      t.text :user
+      t.text :query
+      t.integer :query_hash, limit: 8
+      t.float :total_time
+      t.integer :calls, limit: 8
+      t.timestamp :captured_at
+    end
+
+    add_index :pghero_query_stats, [:database, :captured_at]
+  end
+end

data/lib/generators/pghero/templates/space_stats.rb.tt

@@ -0,0 +1,13 @@
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pghero_space_stats do |t|
+      t.text :database
+      t.text :schema
+      t.text :relation
+      t.integer :size, limit: 8
+      t.timestamp :captured_at
+    end
+
+    add_index :pghero_space_stats, [:database, :captured_at]
+  end
+end

data/lib/pghero.rb

@@ -0,0 +1,246 @@
+# dependencies
+require "active_support"
+require "forwardable"
+
+# methods
+require "pghero/methods/basic"
+require "pghero/methods/connections"
+require "pghero/methods/constraints"
+require "pghero/methods/explain"
+require "pghero/methods/indexes"
+require "pghero/methods/kill"
+require "pghero/methods/maintenance"
+require "pghero/methods/queries"
+require "pghero/methods/query_stats"
+require "pghero/methods/replication"
+require "pghero/methods/sequences"
+require "pghero/methods/settings"
+require "pghero/methods/space"
+require "pghero/methods/suggested_indexes"
+require "pghero/methods/system"
+require "pghero/methods/tables"
+require "pghero/methods/users"
+
+require "pghero/database"
+require "pghero/engine" if defined?(Rails)
+require "pghero/version"
+
+module PgHero
+  autoload :Connection, "pghero/connection"
+  autoload :Stats, "pghero/stats"
+  autoload :QueryStats, "pghero/query_stats"
+  autoload :SpaceStats, "pghero/space_stats"
+
+  class Error < StandardError; end
+  class NotEnabled < Error; end
+
+  MUTEX = Mutex.new
+
+  # settings
+  class << self
+    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :explain_timeout_sec, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations, :config_path, :filter_data
+  end
+  self.long_running_query_sec = (ENV["PGHERO_LONG_RUNNING_QUERY_SEC"] || 60).to_i
+  self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
+  self.slow_query_calls = (ENV["PGHERO_SLOW_QUERY_CALLS"] || 100).to_i
+  self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_f
+  self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 500).to_i
+  self.cache_hit_rate_threshold = 99
+  self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+  self.show_migrations = true
+  self.config_path = ENV["PGHERO_CONFIG_PATH"] || "config/pghero.yml"
+  self.filter_data = ENV["PGHERO_FILTER_DATA"].to_s.size > 0
+
+  class << self
+    extend Forwardable
+    def_delegators :primary_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autovacuum_danger,
+      :best_index, :blocked_queries, :connections, :connection_sources, :connection_states, :connection_stats,
+      :cpu_usage, :create_user, :database_size, :db_instance_identifier, :disable_query_stats, :drop_user,
+      :duplicate_indexes, :enable_query_stats, :explain, :historical_query_stats_enabled?, :index_caching,
+      :index_hit_rate, :index_usage, :indexes, :invalid_constraints, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,
+      :last_stats_reset_time, :long_running_queries, :maintenance_info, :missing_indexes, :query_stats,
+      :query_stats_available?, :query_stats_enabled?, :query_stats_extension_enabled?, :query_stats_readable?,
+      :rds_stats, :read_iops_stats, :region, :relation_sizes, :replica?, :replication_lag, :replication_lag_stats,
+      :reset_query_stats, :reset_stats, :running_queries, :secret_access_key, :sequence_danger, :sequences, :settings,
+      :slow_queries, :space_growth, :ssl_used?, :stats_connection, :suggested_indexes, :suggested_indexes_by_query,
+      :suggested_indexes_enabled?, :system_stats_enabled?, :table_caching, :table_hit_rate, :table_stats,
+      :total_connections, :transaction_id_danger, :unused_indexes, :unused_tables, :write_iops_stats
+
+    def time_zone=(time_zone)
+      @time_zone = time_zone.is_a?(ActiveSupport::TimeZone) ? time_zone : ActiveSupport::TimeZone[time_zone.to_s]
+    end
+
+    def time_zone
+      @time_zone || Time.zone
+    end
+
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after PgHero is loaded
+    def username
+      @username ||= config["username"] || ENV["PGHERO_USERNAME"]
+    end
+
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after PgHero is loaded
+    def password
+      @password ||= config["password"] || ENV["PGHERO_PASSWORD"]
+    end
+
+    def stats_database_url
+      @stats_database_url ||= config["stats_database_url"] || ENV["PGHERO_STATS_DATABASE_URL"]
+    end
+
+    def config
+      @config ||= begin
+        require "erb"
+        require "yaml"
+
+        path = config_path
+
+        config_file_exists = File.exist?(path)
+
+        config = YAML.load(ERB.new(File.read(path)).result) if config_file_exists
+        config ||= {}
+
+        if config[env]
+          config[env]
+        elsif config["databases"] # preferred format
+          config
+        elsif config_file_exists
+          raise "Invalid config file"
+        else
+          databases = {}
+
+          if !ENV["PGHERO_DATABASE_URL"] && spec_supported?
+            ActiveRecord::Base.configurations.configs_for(env_name: env, include_replicas: true).each do |db|
+              databases[db.send(spec_name_key)] = {"spec" => db.send(spec_name_key)}
+            end
+          end
+
+          if databases.empty?
+            databases["primary"] = {
+              "url" => ENV["PGHERO_DATABASE_URL"] || connection_config(ActiveRecord::Base)
+            }
+          end
+
+          if databases.size == 1
+            databases.values.first.merge!(
+              "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"],
+              "gcp_database_id" => ENV["PGHERO_GCP_DATABASE_ID"],
+              "azure_resource_id" => ENV["PGHERO_AZURE_RESOURCE_ID"]
+            )
+          end
+
+          {
+            "databases" => databases
+          }
+        end
+      end
+    end
+
+    # ensure we only have one copy of databases
+    # so there's only one connection pool per database
+    def databases
+      unless defined?(@databases)
+        # only use mutex on initialization
+        MUTEX.synchronize do
+          # return if another process initialized while we were waiting
+          return @databases if defined?(@databases)
+
+          @databases = config["databases"].map { |id, c| [id.to_sym, Database.new(id, c)] }.to_h
+        end
+      end
+
+      @databases
+    end
+
+    def primary_database
+      databases.values.first
+    end
+
+    def capture_query_stats(verbose: false)
+      each_database do |database|
+        next unless database.capture_query_stats?
+        puts "Capturing query stats for #{database.id}..." if verbose
+        database.capture_query_stats(raise_errors: true)
+      end
+    end
+
+    def capture_space_stats(verbose: false)
+      each_database do |database|
+        puts "Capturing space stats for #{database.id}..." if verbose
+        database.capture_space_stats
+      end
+    end
+
+    def analyze_all(**options)
+      each_database do |database|
+        next if database.replica?
+        database.analyze_tables(**options)
+      end
+    end
+
+    def autoindex_all(create: false, verbose: true)
+      each_database do |database|
+        puts "Autoindexing #{database.id}..." if verbose
+        database.autoindex(create: create)
+      end
+    end
+
+    def pretty_size(value)
+      ActiveSupport::NumberHelper.number_to_human_size(value, precision: 3)
+    end
+
+    # delete previous stats
+    # go database by database to use an index
+    # stats for old databases are not cleaned up since we can't use an index
+    def clean_query_stats
+      each_database do |database|
+        database.clean_query_stats
+      end
+    end
+
+    def clean_space_stats
+      each_database do |database|
+        database.clean_space_stats
+      end
+    end
+
+    # private
+    def spec_supported?
+      ActiveRecord::VERSION::MAJOR >= 6
+    end
+
+    # private
+    def connection_config(model)
+      ActiveRecord::VERSION::STRING.to_f >= 6.1 ? model.connection_db_config.configuration_hash : model.connection_config
+    end
+
+    # private
+    # Rails 6.1 deprecate `spec_name` and use `name` for configurations
+    # https://github.com/rails/rails/pull/38536
+    def spec_name_key
+      ActiveRecord::VERSION::STRING.to_f >= 6.1 ? :name : :spec_name
+    end
+
+    private
+
+    def each_database
+      first_error = nil
+
+      databases.each do |_, database|
+        begin
+          yield database
+        rescue => e
+          puts "#{e.class.name}: #{e.message}"
+          puts
+          first_error ||= e
+        end
+      end
+
+      raise first_error if first_error
+
+      true
+    end
+  end
+end

data/lib/pghero/connection.rb

@@ -0,0 +1,5 @@
+module PgHero
+  class Connection < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/lib/pghero/database.rb

@@ -0,0 +1,175 @@
+module PgHero
+  class Database
+    include Methods::Basic
+    include Methods::Connections
+    include Methods::Constraints
+    include Methods::Explain
+    include Methods::Indexes
+    include Methods::Kill
+    include Methods::Maintenance
+    include Methods::Queries
+    include Methods::QueryStats
+    include Methods::Replication
+    include Methods::Sequences
+    include Methods::Settings
+    include Methods::Space
+    include Methods::SuggestedIndexes
+    include Methods::System
+    include Methods::Tables
+    include Methods::Users
+
+    attr_reader :id, :config
+
+    def initialize(id, config)
+      @id = id
+      @config = config || {}
+
+      # preload model to ensure only one connection pool
+      # this doesn't actually start any connections
+      @adapter_checked = false
+      @connection_model = build_connection_model
+    end
+
+    def name
+      @name ||= @config["name"] || id.titleize
+    end
+
+    def capture_query_stats?
+      config["capture_query_stats"] != false
+    end
+
+    def cache_hit_rate_threshold
+      (config["cache_hit_rate_threshold"] || PgHero.config["cache_hit_rate_threshold"] || PgHero.cache_hit_rate_threshold).to_i
+    end
+
+    def total_connections_threshold
+      (config["total_connections_threshold"] || PgHero.config["total_connections_threshold"] || PgHero.total_connections_threshold).to_i
+    end
+
+    def slow_query_ms
+      (config["slow_query_ms"] || PgHero.config["slow_query_ms"] || PgHero.slow_query_ms).to_i
+    end
+
+    def slow_query_calls
+      (config["slow_query_calls"] || PgHero.config["slow_query_calls"] || PgHero.slow_query_calls).to_i
+    end
+
+    def explain_timeout_sec
+      (config["explain_timeout_sec"] || PgHero.config["explain_timeout_sec"] || PgHero.explain_timeout_sec).to_f
+    end
+
+    def long_running_query_sec
+      (config["long_running_query_sec"] || PgHero.config["long_running_query_sec"] || PgHero.long_running_query_sec).to_i
+    end
+
+    # defaults to 100 megabytes
+    def index_bloat_bytes
+      (config["index_bloat_bytes"] || PgHero.config["index_bloat_bytes"] || 104857600).to_i
+    end
+
+    def aws_access_key_id
+      config["aws_access_key_id"] || PgHero.config["aws_access_key_id"] || ENV["PGHERO_ACCESS_KEY_ID"] || ENV["AWS_ACCESS_KEY_ID"]
+    end
+
+    def aws_secret_access_key
+      config["aws_secret_access_key"] || PgHero.config["aws_secret_access_key"] || ENV["PGHERO_SECRET_ACCESS_KEY"] || ENV["AWS_SECRET_ACCESS_KEY"]
+    end
+
+    def aws_region
+      config["aws_region"] || PgHero.config["aws_region"] || ENV["PGHERO_REGION"] || ENV["AWS_REGION"] || (defined?(Aws) && Aws.config[:region]) || "us-east-1"
+    end
+
+    # environment variable is only used if no config file
+    def aws_db_instance_identifier
+      @aws_db_instance_identifier ||= config["aws_db_instance_identifier"] || config["db_instance_identifier"]
+    end
+
+    # environment variable is only used if no config file
+    def gcp_database_id
+      @gcp_database_id ||= config["gcp_database_id"]
+    end
+
+    # environment variable is only used if no config file
+    def azure_resource_id
+      @azure_resource_id ||= config["azure_resource_id"]
+    end
+
+    # must check keys for booleans
+    def filter_data
+      unless defined?(@filter_data)
+        @filter_data =
+          if config.key?("filter_data")
+            config["filter_data"]
+          elsif PgHero.config.key?("filter_data")
+            PgHero.config.key?("filter_data")
+          else
+            PgHero.filter_data
+          end
+
+        if @filter_data
+          begin
+            require "pg_query"
+          rescue LoadError
+            raise Error, "pg_query required for filter_data"
+          end
+        end
+      end
+
+      @filter_data
+    end
+
+    # TODO remove in next major version
+    alias_method :access_key_id, :aws_access_key_id
+    alias_method :secret_access_key, :aws_secret_access_key
+    alias_method :region, :aws_region
+    alias_method :db_instance_identifier, :aws_db_instance_identifier
+
+    private
+
+    # check adapter lazily
+    def connection_model
+      unless @adapter_checked
+        # rough check for Postgres adapter
+        # keep this message generic so it's useful
+        # when empty url set in Docker image pghero.yml
+        unless @connection_model.connection.adapter_name =~ /postg/i
+          raise Error, "Invalid connection URL"
+        end
+        @adapter_checked = true
+      end
+
+      @connection_model
+    end
+
+    # just return the model
+    # do not start a connection
+    def build_connection_model
+      url = config["url"]
+
+      # resolve spec
+      if !url && config["spec"]
+        raise Error, "Spec requires Rails 6+" unless PgHero.spec_supported?
+        config_options = {env_name: PgHero.env, PgHero.spec_name_key => config["spec"], include_replicas: true}
+        resolved = ActiveRecord::Base.configurations.configs_for(**config_options)
+        raise Error, "Spec not found: #{config["spec"]}" unless resolved
+        url = ActiveRecord::VERSION::STRING.to_f >= 6.1 ? resolved.configuration_hash : resolved.config
+      end
+
+      url = url.dup
+
+      Class.new(PgHero::Connection) do
+        def self.name
+          "PgHero::Connection::Database#{object_id}"
+        end
+
+        case url
+        when String
+          url = "#{url}#{url.include?("?") ? "&" : "?"}connect_timeout=5" unless url.include?("connect_timeout=")
+        when Hash
+          url[:connect_timeout] ||= 5
+        end
+        establish_connection url if url
+      end
+    end
+  end
+end