pghero 2.3.0 → 2.5.1

data/app/helpers/pg_hero/home_helper.rb

@@ -15,5 +15,16 @@ module PgHero
     def pghero_js_var(name, value)
       "var #{name} = #{json_escape(value.to_json(root: false))};".html_safe
     end
+
+    def pghero_remove_index(query)
+      if query[:columns]
+        columns = query[:columns].map(&:to_sym)
+        columns = columns.first if columns.size == 1
+      end
+      ret = String.new("remove_index #{query[:table].to_sym.inspect}")
+      ret << ", name: #{(query[:name] || query[:index]).to_s.inspect}"
+      ret << ", column: #{columns.inspect}" if columns
+      ret
+    end
   end
 end

data/app/views/pg_hero/home/_live_queries_table.html.erb

@@ -11,7 +11,17 @@
     <% queries.reverse.each do |query| %>
       <tr>
         <td><%= query[:pid] %></td>
-        <td><%= number_with_delimiter(query[:duration_ms].round) %> ms</td>
+        <td>
+          <% sec = query[:duration_ms] / 1000.0 %>
+          <% if sec < 1.minute %>
+            <%= sec.round(1) %> s
+          <% elsif sec < 1.day %>
+            <%= Time.at(sec).utc.strftime("%H:%M:%S") %>
+          <% else %>
+            <% days = (sec / 1.day).floor %>
+            <%= days %>d <%= Time.at(sec - days.days).utc.strftime("%H:%M:%S") %>
+          <% end %>
+        </td>
         <td>
           <%= query[:state] %>
           <% if vacuum_progress[query[:pid]] %>
@@ -20,7 +30,9 @@
           <% end %>
         </td>
         <td class="text-right">
-          <%= button_to "Explain", explain_path, params: {query: query[:query]}, form: {target: "_blank"}, class: "btn btn-info" %>
+          <% unless @database.filter_data %>
+            <%= button_to "Explain", explain_path, params: {query: query[:query]}, form: {target: "_blank"}, class: "btn btn-info" %>
+          <% end %>
           <%= button_to "Kill", kill_path(pid: query[:pid]), class: "btn btn-danger" %>
         </td>
       </tr>

data/app/views/pg_hero/home/connections.html.erb

@@ -18,6 +18,15 @@
       new Chartkick.PieChart("chart-2", <%= json_escape(@connections_by_user.to_json).html_safe %>);
     </script>
 
+    <% if @connections_by_ssl_status %>
+      <h3>By Security</h3>
+
+      <div id="chart-3" class="chart" style="height: 260px; line-height: 260px; margin-bottom: 20px;">Loading...</div>
+      <script>
+        new Chartkick.PieChart("chart-3", <%= json_escape(@connections_by_ssl_status.to_json).html_safe %>);
+      </script>
+    <% end %>
+
     <%= render partial: "connections_table", locals: {connection_sources: @connection_sources} %>
   <% end %>
 </div>

data/app/views/pg_hero/home/index.html.erb

@@ -62,11 +62,17 @@
       (<%= link_to pluralize(@unreadable_sequences.size, "unreadable sequence", "unreadable sequences"), {unreadable: "t"} %>)
     <% end %>
   </div>
-  <div class="alert alert-<%= @invalid_indexes.empty? ? "success" : "warning" %>">
-    <% if @invalid_indexes.any? %>
-      <%= pluralize(@invalid_indexes.size, "invalid index", "invalid indexes") %>
+  <div class="alert alert-<%= @invalid_indexes.empty? && @invalid_constraints.empty? ? "success" : "warning" %>">
+    <% if @invalid_indexes.empty? && @invalid_constraints.empty?  %>
+      No invalid indexes or constraints
     <% else %>
-      No invalid indexes
+      <% if @invalid_indexes.any? %>
+        <%= pluralize(@invalid_indexes.size, "invalid index", "invalid indexes") %>
+      <% end %>
+      <% if @invalid_constraints.any? %>
+        <% if @invalid_indexes.any? %>and<% end %>
+        <%= pluralize(@invalid_constraints.size, "invalid constraint", "invalid constraints") %>
+      <% end %>
     <% end %>
   </div>
   <% if @duplicate_indexes %>
@@ -332,6 +338,39 @@
   </div>
 <% end %>
 
+<% if @invalid_constraints.any? %>
+  <div class="content">
+    <h1>Invalid Constraints</h1>
+
+    <p>These constraints are marked as <code>NOT VALID</code>. You should validate them.</p>
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th>Name</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @invalid_constraints.each do |constraint| %>
+          <tr>
+            <td>
+              <%= constraint[:name] %>
+              <% if constraint[:schema] != "public" %>
+                <span class="text-muted"><%= constraint[:schema] %></span>
+              <% end %>
+            </td>
+          </tr>
+          <tr>
+            <td style="border-top: none; padding: 0;">
+              <pre><code>ALTER TABLE <%= pghero_pretty_ident(constraint[:table], schema: constraint[:schema]) %> VALIDATE CONSTRAINT <%= pghero_pretty_ident(constraint[:name]) %>;</code></pre>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% end %>
+
 <% if @duplicate_indexes && @duplicate_indexes.any? %>
   <div class="content">
     <h1>Duplicate Indexes</h1>
@@ -345,10 +384,10 @@
     </p>
 
     <div id="migration2" style="display: none;">
-      <pre>rails g migration remove_unneeded_indexes</pre>
+      <pre>rails generate migration remove_unneeded_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @duplicate_indexes.each do |query| %>
-remove_index <%= query[:unneeded_index][:table].to_sym.inspect %>, name: <%= query[:unneeded_index][:name].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query[:unneeded_index]) %><% end %></pre>
     </div>
 
     <table class="table duplicate-indexes">
@@ -387,12 +426,12 @@ remove_index <%= query[:unneeded_index][:table].to_sym.inspect %>, name: <%= que
     </p>
 
     <div id="migration3" style="display: none;">
-      <pre>rails g migration add_suggested_indexes</pre>
+      <pre>rails generate migration add_suggested_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;">commit_db_transaction
 <% @suggested_indexes.each do |index| %>
 <% if index[:using] && index[:using] != "btree" %>
-connection.execute("CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)")
+add_index <%= index[:table].to_sym.inspect %>, <%= index[:columns].first.inspect %>, using: <%= index[:using].inspect %>, algorithm: :concurrently
 <% else %>
 add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(", ") %>], algorithm: :concurrently<% end %>
 <% end %></pre>
@@ -449,10 +488,10 @@ pg_stat_statements.track = all
     </p>
 
     <div id="migration" style="display: none;">
-      <pre>rails g migration remove_unused_indexes</pre>
+      <pre>rails generate migration remove_unused_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.each do |query| %>
-remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query)%><% end %></pre>
     </div>
 
     <table class="table">

data/app/views/pg_hero/home/live_queries.html.erb

@@ -7,5 +7,5 @@
 
   <p><%= button_to "Kill all connections", kill_all_path, class: "btn btn-danger" %></p>
 
-  <p class="text-muted">You may need to restart your app server afterwards.</p>
+  <p class="text-muted">You may need to restart your app servers afterwards.</p>
 </div>

data/app/views/pg_hero/home/maintenance.html.erb

@@ -7,6 +7,9 @@
         <th>Table</th>
         <th style="width: 20%;">Last Vacuum</th>
         <th style="width: 20%;">Last Analyze</th>
+        <% if @show_dead_rows %>
+          <th style="width: 20%;">Dead Rows</th>
+        <% end %>
       </tr>
     </thead>
     <tbody>
@@ -21,7 +24,7 @@
           <td>
             <% time = [table[:last_autovacuum], table[:last_vacuum]].compact.max %>
             <% if time %>
-              <%= time.in_time_zone(@time_zone).strftime("%-m/%-e %l:%M %P") %>
+              <%= l time.in_time_zone(@time_zone), format: :short %>
             <% else %>
               <span class="text-muted">Unknown</span>
             <% end %>
@@ -29,11 +32,22 @@
           <td>
             <% time = [table[:last_autoanalyze], table[:last_analyze]].compact.max %>
             <% if time %>
-              <%= time.in_time_zone(@time_zone).strftime("%-m/%-e %l:%M %P") %>
+              <%= l time.in_time_zone(@time_zone), format: :short %>
             <% else %>
               <span class="text-muted">Unknown</span>
             <% end %>
           </td>
+          <% if @show_dead_rows %>
+            <td>
+              <% if table[:live_rows] != 0 %>
+                <%# use live rows only for denominator to make it easier to compare with autovacuum_vacuum_scale_factor %>
+                <%# it's not a true percentage, since it can go above 100% %>
+                <%= (100.0 * table[:dead_rows] / table[:live_rows]).round %>%
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+          <% end %>
         </tr>
       <% end %>
     </tbody>

data/app/views/pg_hero/home/relation_space.html.erb

@@ -6,9 +6,9 @@
     <% end %>
   </h1>
 
-  <h1>Size <small>MB</small></h1>
+  <h1>Size</h1>
   <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
   <script>
-    new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, min: null})
+    new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, min: null, bytes: true, library: {tooltips: {intersect: false, mode: "index"}}})
   </script>
 </div>

data/app/views/pg_hero/home/show_query.html.erb

@@ -49,19 +49,19 @@
     <h1>Total Time <small>ms</small></h1>
     <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
     <script>
-      new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+      new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, library: {tooltips: {intersect: false, mode: "index"}}})
     </script>
 
     <h1>Average Time <small>ms</small></h1>
     <div id="chart-2" class="chart" style="margin-bottom: 20px;">Loading...</div>
     <script>
-      new Chartkick.LineChart("chart-2", <%= json_escape(@chart2_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+      new Chartkick.LineChart("chart-2", <%= json_escape(@chart2_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, library: {tooltips: {intersect: false, mode: "index"}}})
     </script>
 
     <h1>Calls</h1>
     <div id="chart-3" class="chart" style="margin-bottom: 20px;">Loading...</div>
     <script>
-      new Chartkick.LineChart("chart-3", <%= json_escape(@chart3_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+      new Chartkick.LineChart("chart-3", <%= json_escape(@chart3_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, library: {tooltips: {intersect: false, mode: "index"}}})
     </script>
   <% else %>
     <p>

data/app/views/pg_hero/home/space.html.erb

@@ -6,7 +6,7 @@
   <% if @system_stats_enabled %>
     <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
     <script>
-      new Chartkick.LineChart("chart-1", <%= json_escape(free_space_stats_path.to_json).html_safe %>, {colors: ["#5bc0de"]})
+      new Chartkick.LineChart("chart-1", <%= json_escape(free_space_stats_path.to_json).html_safe %>, {colors: ["#5bc0de"], bytes: true, library: {tooltips: {intersect: false, mode: "index"}}})
     </script>
   <% end %>
 
@@ -30,10 +30,10 @@
     </p>
 
     <div id="migration" style="display: none;">
-      <pre>rails g migration remove_unused_indexes</pre>
+      <pre>rails generate migration remove_unused_indexes</pre>
       <p>And paste</p>
       <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.sort_by { |q| [-q[:size_bytes], q[:index]] }.each do |query| %>
-remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+<%= pghero_remove_index(query) %><% end %></pre>
     </div>
   <% end %>
 

data/app/views/pg_hero/home/system.html.erb

@@ -9,26 +9,26 @@
   <h1>CPU</h1>
   <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
   <script>
-    new Chartkick.LineChart("chart-1", <%= json_escape(cpu_usage_path(path_options).to_json).html_safe %>, {max: 100, colors: ["#5bc0de"]})
+    new Chartkick.LineChart("chart-1", <%= json_escape(cpu_usage_path(path_options).to_json).html_safe %>, {max: 100, colors: ["#5bc0de"], suffix: "%", library: {tooltips: {intersect: false, mode: "index"}}})
   </script>
 
   <h1>Load</h1>
   <div id="chart-2" class="chart" style="margin-bottom: 20px;">Loading...</div>
   <script>
-    new Chartkick.LineChart("chart-2", <%= json_escape(load_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de", "#d9534f"]})
+    new Chartkick.LineChart("chart-2", <%= json_escape(load_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de", "#d9534f"], library: {tooltips: {intersect: false, mode: "nearest"}}})
   </script>
 
   <h1>Connections</h1>
   <div id="chart-3" class="chart" style="margin-bottom: 20px;">Loading...</div>
   <script>
-    new Chartkick.LineChart("chart-3", <%= json_escape(connection_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"]})
+    new Chartkick.LineChart("chart-3", <%= json_escape(connection_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"], library: {tooltips: {intersect: false, mode: "index"}}})
   </script>
 
   <% if @database.replica? %>
     <h1>Replication Lag</h1>
     <div id="chart-4" class="chart" style="margin-bottom: 20px;">Loading...</div>
     <script>
-      new Chartkick.LineChart("chart-4", <%= json_escape(replication_lag_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"]})
+      new Chartkick.LineChart("chart-4", <%= json_escape(replication_lag_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"], library: {tooltips: {intersect: false, mode: "index"}}})
     </script>
   <% end %>
 </div>

data/app/views/pg_hero/home/tune.html.erb

@@ -18,7 +18,8 @@
     </tbody>
   </table>
 
-  <p>Check out <%= link_to "PgTune", "https://pgtune.leopard.in.ua/", target: "_blank" %> for recommendations. DB version is <%= @database.server_version.split(" ").first.split(".").first(2).join(".") %>.</p>
+  <% version_parts = @database.server_version.split(" ").first.split(".") %>
+  <p>Check out <%= link_to "PgTune", "https://pgtune.leopard.in.ua/", target: "_blank" %> for recommendations. DB version is <%= version_parts[0].to_i >= 10 ? version_parts[0] : version_parts.first(2).join(".") %>.</p>
 </div>
 
 <% if @autovacuum_settings %>

data/lib/generators/pghero/templates/config.yml.tt

@@ -1,8 +1,13 @@
 databases:
-  main:
+  primary:
     # Database URL (defaults to app database)
     # url: <%%= ENV["DATABASE_URL"] %>
 
+    # System stats
+    # aws_db_instance_identifier: my-instance
+    # gcp_database_id: my-project:my-instance
+    # azure_resource_id: my-resource-id
+
   # Add more databases
   # other:
   #   url: <%%= ENV["OTHER_DATABASE_URL"] %>
@@ -24,3 +29,18 @@ databases:
 
 # Time zone (defaults to app time zone)
 # time_zone: "Pacific Time (US & Canada)"
+
+# Basic authentication
+# username: admin
+# password: <%%= ENV["PGHERO_PASSWORD"] %>
+
+# Stats database URL (defaults to app database)
+# stats_database_url: <%%= ENV["PGHERO_STATS_DATABASE_URL"] %>
+
+# AWS configuration (defaults to app AWS config)
+# aws_access_key_id: <%%= ENV["AWS_ACCESS_KEY_ID"] %>
+# aws_secret_access_key: <%%= ENV["AWS_SECRET_ACCESS_KEY"] %>
+# aws_region: us-east-1
+
+# Filter data from queries (experimental)
+# filter_data: true

data/lib/pghero.rb

@@ -5,6 +5,7 @@ require "forwardable"
 # methods
 require "pghero/methods/basic"
 require "pghero/methods/connections"
+require "pghero/methods/constraints"
 require "pghero/methods/explain"
 require "pghero/methods/indexes"
 require "pghero/methods/kill"
@@ -33,9 +34,11 @@ module PgHero
   class Error < StandardError; end
   class NotEnabled < Error; end
 
+  MUTEX = Mutex.new
+
   # settings
   class << self
-    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :explain_timeout_sec, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations, :config_path
+    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :explain_timeout_sec, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations, :config_path, :filter_data
   end
   self.long_running_query_sec = (ENV["PGHERO_LONG_RUNNING_QUERY_SEC"] || 60).to_i
   self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
@@ -46,14 +49,15 @@ module PgHero
   self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
   self.show_migrations = true
   self.config_path = ENV["PGHERO_CONFIG_PATH"] || "config/pghero.yml"
+  self.filter_data = ENV["PGHERO_FILTER_DATA"].to_s.size > 0
 
   class << self
     extend Forwardable
     def_delegators :primary_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autovacuum_danger,
-      :best_index, :blocked_queries, :connection_sources, :connection_states, :connection_stats,
+      :best_index, :blocked_queries, :connections, :connection_sources, :connection_states, :connection_stats,
       :cpu_usage, :create_user, :database_size, :db_instance_identifier, :disable_query_stats, :drop_user,
       :duplicate_indexes, :enable_query_stats, :explain, :historical_query_stats_enabled?, :index_caching,
-      :index_hit_rate, :index_usage, :indexes, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,
+      :index_hit_rate, :index_usage, :indexes, :invalid_constraints, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,
       :last_stats_reset_time, :long_running_queries, :maintenance_info, :missing_indexes, :query_stats,
       :query_stats_available?, :query_stats_enabled?, :query_stats_extension_enabled?, :query_stats_readable?,
       :rds_stats, :read_iops_stats, :region, :relation_sizes, :replica?, :replication_lag, :replication_lag_stats,
@@ -70,6 +74,22 @@ module PgHero
       @time_zone || Time.zone
     end
 
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after PgHero is loaded
+    def username
+      @username ||= config["username"] || ENV["PGHERO_USERNAME"]
+    end
+
+    # use method instead of attr_accessor to ensure
+    # this works if variable set after PgHero is loaded
+    def password
+      @password ||= config["password"] || ENV["PGHERO_PASSWORD"]
+    end
+
+    def stats_database_url
+      @stats_database_url ||= config["stats_database_url"] || ENV["PGHERO_STATS_DATABASE_URL"]
+    end
+
     def config
       @config ||= begin
         require "erb"
@@ -89,26 +109,49 @@ module PgHero
         elsif config_file_exists
           raise "Invalid config file"
         else
-          {
-            "databases" => {
-              "primary" => {
-                "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config,
-                "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"]
-              }
+          databases = {}
+
+          if !ENV["PGHERO_DATABASE_URL"] && spec_supported?
+            ActiveRecord::Base.configurations.configs_for(env_name: env, include_replicas: true).each do |db|
+              databases[db.spec_name] = {"spec" => db.spec_name}
+            end
+          end
+
+          if databases.empty?
+            databases["primary"] = {
+              "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config
             }
+          end
+
+          if databases.size == 1
+            databases.values.first.merge!(
+              "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"],
+              "gcp_database_id" => ENV["PGHERO_GCP_DATABASE_ID"],
+              "azure_resource_id" => ENV["PGHERO_AZURE_RESOURCE_ID"]
+            )
+          end
+
+          {
+            "databases" => databases
           }
         end
       end
     end
 
+    # ensure we only have one copy of databases
+    # so there's only one connection pool per database
     def databases
-      @databases ||= begin
-        Hash[
-          config["databases"].map do |id, c|
-            [id.to_sym, PgHero::Database.new(id, c)]
-          end
-        ]
+      unless defined?(@databases)
+        # only use mutex on initialization
+        MUTEX.synchronize do
+          # return if another process initialized while we were waiting
+          return @databases if defined?(@databases)
+
+          @databases = config["databases"].map { |id, c| [id.to_sym, Database.new(id, c)] }.to_h
+        end
       end
+
+      @databases
     end
 
     def primary_database
@@ -163,6 +206,11 @@ module PgHero
       end
     end
 
+    # private
+    def spec_supported?
+      ActiveRecord::VERSION::MAJOR >= 6
+    end
+
     private
 
     def each_database