pghero 1.1.4 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb4fc6839e13e1fcb1d5e1ca1dee05c34b215dc2
-  data.tar.gz: c362762b0b918bbb786adc12c185b20f4bbf9562
+  metadata.gz: c032f7f0fd5b3d6484cae193d125d3899ee87594
+  data.tar.gz: 02db1720b5201380dff89ac2dceba86c42d8dc49
 SHA512:
-  metadata.gz: 4604d456402f86697ba18e0ffa261ce4747cb3eb9e9284c8b35ef8e1e9ce6856745a6b6f43fbbd3b510ea6c138fbc31ca475a9b0e2ac3ab5f92d60796244c84e
-  data.tar.gz: 422a78d125a1ad063e50823cead73194acf7aeb82ebab57b01d3680ef1eb7f5d6c2d9263909b7bd58c512d937a42f30cb32bb99e9dfcfcf419a9283ae5ab3ca3
+  metadata.gz: f07fe4ece79f34731bb6bc28dc19cd09d365a9f9968caeb0918fb347d9be9d050ce5c1286b09feccc0312d2e379afbf3278fd65ca5c802f4c817e6b1ee7f83c0
+  data.tar.gz: dd334b3251a68372218f765b2c5d986e4bd71172e3a428511f05f4dfb7580123b38421a08eebacc204360cc51b60a9b24bc74dea31889eda9f1d6013bd4bc204

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 1.2.0
+
+- Added suggested indexes
+- Added duplicate indexes
+- Added maintenance tab
+- Added load stats for RDS
+- Added `table_caching` and `index_caching` methods
+- Added configurable cache hit rate threshold
+- Show all connections in connections tab
+
 ## 1.1.4
 
 - Added check for transaction ID wraparound failure

data/Gemfile

@@ -2,3 +2,5 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in pghero.gemspec
 gemspec
+
+gem "pg_query"

data/app/controllers/pg_hero/home_controller.rb

@@ -11,24 +11,27 @@ module PgHero
 
     def index
       @title = "Overview"
-      @slow_queries = PgHero.slow_queries(historical: true, start_at: 3.hours.ago)
+      @query_stats = PgHero.query_stats(historical: true, start_at: 3.hours.ago)
+      @slow_queries = PgHero.slow_queries(query_stats: @query_stats)
       @long_running_queries = PgHero.long_running_queries
       @index_hit_rate = PgHero.index_hit_rate
       @table_hit_rate = PgHero.table_hit_rate
       @missing_indexes = PgHero.missing_indexes
       @unused_indexes = PgHero.unused_indexes.select { |q| q["index_scans"].to_i == 0 }
       @invalid_indexes = PgHero.invalid_indexes
-      @good_cache_rate = @table_hit_rate >= 0.99 && @index_hit_rate >= 0.99
+      @duplicate_indexes = PgHero.duplicate_indexes
+      @good_cache_rate = @table_hit_rate >= PgHero.cache_hit_rate_threshold.to_f / 100 && @index_hit_rate >= PgHero.cache_hit_rate_threshold.to_f / 100
       @query_stats_available = PgHero.query_stats_available?
       @total_connections = PgHero.total_connections
       @good_total_connections = @total_connections < PgHero.total_connections_threshold
-      @replica = PgHero.replica?
       if @replica
         @replication_lag = PgHero.replication_lag
         @good_replication_lag = @replication_lag < 5
       end
       @transaction_id_danger = PgHero.transaction_id_danger
       @autovacuum_danger = PgHero.autovacuum_danger
+      set_suggested_indexes
+      @show_migrations = PgHero.show_migrations
     end
 
     def index_usage
@@ -51,6 +54,9 @@ module PgHero
       @title = "Queries"
       @historical_query_stats_enabled = PgHero.historical_query_stats_enabled?
       @sort = %w[average_time calls].include?(params[:sort]) ? params[:sort] : nil
+      @min_average_time = params[:min_average_time] ? params[:min_average_time].to_i : nil
+      @min_calls = params[:min_calls] ? params[:min_calls].to_i : nil
+      @debug = params[:debug] == "true"
 
       @query_stats =
         begin
@@ -62,13 +68,22 @@ module PgHero
           if @historical_query_stats_enabled && !request.xhr?
             []
           else
-            PgHero.query_stats(historical: true, start_at: @start_at, end_at: @end_at, sort: @sort)
+            PgHero.query_stats(
+              historical: true,
+              start_at: @start_at,
+              end_at: @end_at,
+              sort: @sort,
+              min_average_time: @min_average_time,
+              min_calls: @min_calls
+            )
           end
         rescue
           @error = true
           []
         end
 
+      set_suggested_indexes
+
       if request.xhr?
         render layout: false, partial: "queries_table", locals: {queries: @query_stats, xhr: true}
       end
@@ -90,6 +105,13 @@ module PgHero
       render json: PgHero.replication_lag_stats
     end
 
+    def load_stats
+      render json: [
+        {name: "Read IOPS", data: PgHero.read_iops_stats.map { |k, v| [k, v.round] }},
+        {name: "Write IOPS", data: PgHero.write_iops_stats.map { |k, v| [k, v.round] }}
+      ]
+    end
+
     def explain
       @title = "Explain"
       @query = params[:query]
@@ -98,6 +120,7 @@ module PgHero
       if request.post? && @query
         begin
           @explanation = PgHero.explain("#{params[:commit] == "Analyze" ? "ANALYZE " : ""}#{@query}")
+          @suggested_index = PgHero.suggested_indexes(queries: [@query]).first
         rescue ActiveRecord::StatementInvalid => e
           @error = e.message
         end
@@ -114,6 +137,11 @@ module PgHero
       @total_connections = PgHero.total_connections
     end
 
+    def maintenance
+      @title = "Maintenance"
+      @maintenance_info = PgHero.maintenance_info
+    end
+
     def kill
       if PgHero.kill(params[:pid])
         redirect_to root_path, notice: "Query killed"
@@ -168,6 +196,12 @@ module PgHero
     def set_query_stats_enabled
       @query_stats_enabled = PgHero.query_stats_enabled?
       @system_stats_enabled = PgHero.system_stats_enabled?
+      @replica = PgHero.replica?
+    end
+
+    def set_suggested_indexes
+      @suggested_indexes_by_query = PgHero.suggested_indexes_by_query(query_stats: @query_stats)
+      @suggested_indexes = PgHero.suggested_indexes(suggested_indexes_by_query: @suggested_indexes_by_query)
     end
   end
 end

data/app/views/layouts/pg_hero/application.html.erb

@@ -78,6 +78,12 @@
         padding: 10px;
       }
 
+      hr {
+        border: none;
+        height: 0;
+        border-top: solid 1px #ddd;
+      }
+
       .container {
         max-width: 1000px;
         margin-left: auto;
@@ -414,6 +420,9 @@
             <li class="<%= controller.action_name == "index_usage" ? "active" : "" %>"><%= link_to "Index Usage", index_usage_path %></li>
             <li class="<%= controller.action_name == "space" ? "active" : "" %>"><%= link_to "Space", space_path %></li>
             <li class="<%= controller.action_name == "connections" ? "active" : "" %>"><%= link_to "Connections", connections_path %></li>
+            <% unless @replica %>
+              <li class="<%= controller.action_name == "maintenance" ? "active" : "" %>"><%= link_to "Maintenance", maintenance_path %></li>
+            <% end %>
             <li class="<%= controller.action_name == "live_queries" ? "active" : "" %>"><%= link_to "Live Queries", live_queries_path %></li>
             <li class="<%= controller.action_name == "explain" ? "active" : "" %>"><%= link_to "Explain", explain_path %></li>
             <li class="<%= controller.action_name == "tune" ? "active" : "" %>"><%= link_to "Tune", tune_path %></li>

data/app/views/pg_hero/home/_connections_table.html.erb

@@ -13,9 +13,9 @@
     </tr>
   </thead>
   <tbody>
-    <% PgHero.connection_sources.first(10).each do |source| %>
+    <% connection_sources.each do |source| %>
       <tr>
-        <td><%= source["source"] %> <span class="text-muted"><%= source["ip"] %></span></td>
+        <td><%= source["source"] %> <div class="text-muted"><%= [source["database"], source["ip"]].compact.join(" ") %></div></td>
         <td><%= number_with_delimiter(source["total_connections"]) %></td>
       </tr>
     <% end %>

data/app/views/pg_hero/home/_queries_table.html.erb

@@ -46,6 +46,16 @@
           <% if query["query"] == "<insufficient privilege>" %>
             <p class="text-muted">For security reasons, only superusers can see queries executed by other users.</p>
           <% end %>
+          <% if (i2 = @suggested_indexes_by_query[query["query"]]) %>
+            <% if (index = i2[:index]) && !i2[:covering_index] %>
+              <%= render partial: "suggested_index", locals: {index: index} %>
+            <% end %>
+            <% if @debug %>
+              <code><pre style="color: #f0ad4e; background-color: #333;"><% if i2[:explanation] %><%= i2[:explanation] %><% else %>Rows: <%= i2[:rows] %>
+Row estimates: <%= i2[:row_estimates].to_a.map { |k, v| "#{k}=#{v}" }.join(", ") %>
+Row progresssion: <%= i2[:row_progression].to_a.join(", ") %><% end %></pre></code>
+            <% end %>
+          <% end %>
         </td>
       </tr>
     <% end %>

data/app/views/pg_hero/home/_query_stats_slider.html.erb

@@ -43,6 +43,9 @@
   }
 
   var sort = <%= @sort.to_json.html_safe %>;
+  var minAverageTime = <%= @min_average_time.to_json %>;
+  var minCalls = <%= @min_calls.to_json %>;
+  var debug = <%= @debug.to_json %>;
 
   var months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
 
@@ -100,7 +103,7 @@
 
   function queriesPath(params) {
     var path = "queries";
-    if (params.start_at || params.end_at || params.sort) {
+    if (params.start_at || params.end_at || params.sort || params.min_average_time || params.min_calls || params.debug) {
       path += "?" + $.param(params);
     }
     return path;
@@ -121,14 +124,32 @@
     if (sort) {
       params.sort = sort;
     }
+    if (minAverageTime) {
+      params.min_average_time = minAverageTime;
+    }
+    if (minCalls) {
+      params.min_calls = minCalls;
+    }
+    if (debug) {
+      params.debug = debug;
+    }
 
     var path = queriesPath(params);
 
     $(".queries-table th a").each( function () {
-      var p = $.extend({}, params, {sort: $(this).data("sort")});
+      var p = $.extend({}, params, {sort: $(this).data("sort"), min_average_time: minAverageTime, min_calls: minCalls, debug: debug});
       if (!p.sort) {
         delete p.sort;
       }
+      if (!p.min_average_time) {
+        delete p.min_average_time;
+      }
+      if (!p.min_calls) {
+        delete p.min_calls;
+      }
+      if (!p.debug) {
+        delete p.debug;
+      }
       $(this).attr("href", queriesPath(p));
     });
 

data/app/views/pg_hero/home/_suggested_index.html.erb

@@ -0,0 +1 @@
+<code><pre style="color: #eee; background-color: #333;">CREATE INDEX CONCURRENTLY ON <%= index[:table] %> (<%= index[:columns].join(", ") %>)</pre></code>

data/app/views/pg_hero/home/connections.html.erb

@@ -1,5 +1,5 @@
 <div class="content">
   <h1>Connections</h1>
 
-  <%= render partial: "connections_table", locals: {total_connections: @total_connections, show_message: false} %>
+  <%= render partial: "connections_table", locals: {total_connections: @total_connections, connection_sources: PgHero.connection_sources(by_database: true), show_message: false} %>
 </div>

data/app/views/pg_hero/home/explain.html.erb

@@ -9,6 +9,9 @@
   <% if @explanation %>
     <pre><%= @explanation %></pre>
     <p><%= link_to "See how to interpret this", "http://www.postgresql.org/docs/current/static/using-explain.html", target: "_blank" %></p>
+    <% if (index = @suggested_index) %>
+      <%= render partial: "suggested_index", locals: {index: index} %>
+    <% end %>
   <% elsif @error %>
     <div class="alert alert-danger"><%= @error %></div>
   <% end %>

data/app/views/pg_hero/home/index.html.erb

@@ -43,9 +43,18 @@
       No long running queries
     <% end %>
   </div>
+  <% if PgHero.suggested_indexes_enabled? %>
+    <div class="alert alert-<%= @suggested_indexes.empty? ? "success" : "warning" %>">
+      <% if @suggested_indexes.any? %>
+        <%= pluralize(@suggested_indexes.size, "suggested index", "suggested indexes") %>
+      <% else %>
+        No suggested indexes
+      <% end %>
+    </div>
+  <% end %>
   <div class="alert alert-<%= @good_cache_rate ? "success" : "warning" %>">
     <% if @good_cache_rate %>
-      Cache hit rate above 99%
+      Cache hit rate above <%= PgHero.cache_hit_rate_threshold %>%
     <% else %>
       Low cache hit rate
     <% end %>
@@ -81,6 +90,13 @@
       No invalid indexes
     <% end %>
   </div>
+  <div class="alert alert-<%= @duplicate_indexes.empty? ? "success" : "warning" %>">
+    <% if @duplicate_indexes.any? %>
+      <%= pluralize(@duplicate_indexes.size, "duplicate index", "duplicate indexes") %>
+    <% else %>
+      No duplicate indexes
+    <% end %>
+  </div>
   <div class="alert alert-<%= @missing_indexes.empty? ? "success" : "warning" %>">
     <% if @missing_indexes.any? %>
       <%= pluralize(@missing_indexes.size, "table appears", "tables appear") %> to be missing indexes
@@ -114,6 +130,35 @@
   </div>
 <% end %>
 
+<% if @suggested_indexes.any? %>
+  <div class="content">
+    <h1>Suggested Indexes</h1>
+    <p>
+      Add indexes to speed up queries.
+      <% if @show_migrations %>
+      Here’s a
+      <a href="javascript: void(0);" onclick="document.getElementById('migration3').style.display = 'block';">migration</a> to help.
+      <% end %>
+    </p>
+
+    <div id="migration3" style="display: none;">
+      <pre>rails g migration add_suggested_indexes</pre>
+      <p>And paste</p>
+      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @suggested_indexes.each do |index| %>
+add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(" ,") %>], algorithm: :concurrently<% end %></pre>
+    </div>
+
+    <% @suggested_indexes.each_with_index do |index, i| %>
+      <hr />
+      <%= render partial: "suggested_index", locals: {index: index} %>
+      <p>to speed up</p>
+      <% index[:queries].each do |query| %>
+        <code><pre><%= query %></pre></code>
+      <% end %>
+    <% end %>
+  </div>
+<% end %>
+
 <% if !@good_cache_rate %>
   <div class="content">
     <h1>Low Cache Hit Rate</h1>
@@ -134,7 +179,7 @@
 <% if !@good_total_connections %>
   <div class="content">
     <h1>High Number of Connections</h1>
-    <%= render partial: "connections_table", locals: {total_connections: @total_connections, show_message: true} %>
+    <%= render partial: "connections_table", locals: {total_connections: @total_connections, connection_sources: PgHero.connection_sources(by_database: true).first(10), show_message: true} %>
   </div>
 <% end %>
 
@@ -215,6 +260,49 @@ pg_stat_statements.track = all</pre>
   </div>
 <% end %>
 
+<% if @duplicate_indexes.any? %>
+  <div class="content">
+    <h1>Duplicate Indexes</h1>
+
+    <p>
+      These indexes exist, but aren’t needed. Remove them
+      <% if @show_migrations %>
+      <a href="javascript: void(0);" onclick="document.getElementById('migration2').style.display = 'block';">with a migration</a>
+      <% end %>
+      for faster writes.
+    </p>
+
+    <div id="migration2" style="display: none;">
+      <pre>rails g migration remove_unneeded_indexes</pre>
+      <p>And paste</p>
+      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @duplicate_indexes.each do |query| %>
+remove_index <%= query["unneeded_index"]["table"].to_sym.inspect %>, name: <%= query["unneeded_index"]["name"].to_s.inspect %><% end %></pre>
+    </div>
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th>Details</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @duplicate_indexes.each do |index| %>
+          <% unneeded_index = index["unneeded_index"] %>
+          <% covering_index = index["covering_index"] %>
+          <tr>
+            <td style="padding-top: 15px; padding-bottom: 5px;">
+              On <%= unneeded_index["table"] %>
+              <pre><%= unneeded_index["name"] %> (<%= unneeded_index["columns"].join(", ") %>)</pre>
+              is covered by
+              <pre><%= covering_index["name"] %> (<%= covering_index["columns"].join(", ") %>)</pre>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% end %>
+
 <% if @missing_indexes.any? %>
   <div class="content">
     <h1>Missing Indexes</h1>
@@ -248,7 +336,9 @@ pg_stat_statements.track = all</pre>
 
     <p>
       Unused indexes cause unnecessary overhead. Remove them
+      <% if @show_migrations %>
       <a href="javascript: void(0);" onclick="document.getElementById('migration').style.display = 'block';">with a migration</a>
+      <% end %>
       for faster writes.
     </p>
 

data/app/views/pg_hero/home/maintenance.html.erb

@@ -0,0 +1,32 @@
+<div class="content">
+  <h1>Maintenance</h1>
+
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Table</th>
+        <th style="width: 20%;">Last Vacuum</th>
+        <th style="width: 20%;">Last Analyze</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @maintenance_info.each do |table| %>
+        <tr>
+          <td><%= table["table"] %></td>
+          <td>
+            <% time = [table["last_autovacuum"], table["last_vacuum"]].compact.max %>
+            <% if time %>
+              <%= PgHero.time_zone.parse(time).strftime("%m/%-e %l:%M %P") %>
+            <% end %>
+          </td>
+          <td>
+            <% time = [table["last_autoanalyze"], table["last_analyze"]].compact.max %>
+            <% if time %>
+              <%= PgHero.time_zone.parse(time).strftime("%m/%-e %l:%M %P") %>
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/app/views/pg_hero/home/queries.html.erb

@@ -9,6 +9,10 @@
     <%= render partial: "query_stats_slider" %>
   <% end %>
 
+  <% if PgHero.suggested_indexes_enabled? %>
+    <p style="text-align: center; margin-top: 7px;">PgHero now suggests indexes. <%= link_to "See how it thinks", {debug: true} %>.</p>
+  <% end %>
+
   <% if @query_stats_enabled %>
     <% if @error %>
       <div class="alert alert-danger">Cannot understand start or end time.</div>