pghero 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c032f7f0fd5b3d6484cae193d125d3899ee87594
-  data.tar.gz: 02db1720b5201380dff89ac2dceba86c42d8dc49
+  metadata.gz: 3975a93a4da7c4199e2e3a5f98c4c98e35c991e9
+  data.tar.gz: 7a2a1ba863c86c42d09316c624cbd9fa411bda41
 SHA512:
-  metadata.gz: f07fe4ece79f34731bb6bc28dc19cd09d365a9f9968caeb0918fb347d9be9d050ce5c1286b09feccc0312d2e379afbf3278fd65ca5c802f4c817e6b1ee7f83c0
-  data.tar.gz: dd334b3251a68372218f765b2c5d986e4bd71172e3a428511f05f4dfb7580123b38421a08eebacc204360cc51b60a9b24bc74dea31889eda9f1d6013bd4bc204
+  metadata.gz: 745ac33fea1c485e3263075b4a17c04b79894f240776a65331818a160b8ef3a0f308f362aeb590c433be96e1f684dd36d764ae3087bf16ef4826d284c168f8c7
+  data.tar.gz: c671471aa2a742f428a73b8d05c4af7624baa324e80654ae3d20c3b4fbf1c615ee942860b383c6c03c76ae798efe27476f4adb2152a4744f007de924bdbcc0c5

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.2.1
+
+- Better suggested indexes
+- Removed unused indexes noise
+- Removed autovacuum danger noise
+- Removed maintenance tab
+- Fixed suggested indexes for replicas
+- Fixed issue w/ suggested indexes where same table name exists in multiple schemas
+
 ## 1.2.0
 
 - Added suggested indexes

data/README.md

@@ -1,8 +1,8 @@
 # PgHero
 
-Postgres insights made easy
+The missing dashboard for Postgres
 
-[View the demo](https://pghero.herokuapp.com/)
+[See it in action](https://pghero.herokuapp.com/)
 
 [![Screenshot](https://pghero.herokuapp.com/assets/screenshot-34a33ee68c77d64c1f89f143f6297a47.png)](https://pghero.herokuapp.com/)
 

data/app/controllers/pg_hero/home_controller.rb

@@ -16,7 +16,12 @@ module PgHero
       @long_running_queries = PgHero.long_running_queries
       @index_hit_rate = PgHero.index_hit_rate
       @table_hit_rate = PgHero.table_hit_rate
-      @missing_indexes = PgHero.missing_indexes
+      @missing_indexes =
+        if PgHero.suggested_indexes_enabled?
+          []
+        else
+          PgHero.missing_indexes
+        end
       @unused_indexes = PgHero.unused_indexes.select { |q| q["index_scans"].to_i == 0 }
       @invalid_indexes = PgHero.invalid_indexes
       @duplicate_indexes = PgHero.duplicate_indexes
@@ -28,9 +33,8 @@ module PgHero
         @replication_lag = PgHero.replication_lag
         @good_replication_lag = @replication_lag < 5
       end
-      @transaction_id_danger = PgHero.transaction_id_danger
-      @autovacuum_danger = PgHero.autovacuum_danger
-      set_suggested_indexes
+      @transaction_id_danger = PgHero.transaction_id_danger(threshold: 1500000000)
+      set_suggested_indexes((params[:min_average_time] || 20).to_f, (params[:min_calls] || 50).to_i)
       @show_migrations = PgHero.show_migrations
     end
 
@@ -56,7 +60,6 @@ module PgHero
       @sort = %w[average_time calls].include?(params[:sort]) ? params[:sort] : nil
       @min_average_time = params[:min_average_time] ? params[:min_average_time].to_i : nil
       @min_calls = params[:min_calls] ? params[:min_calls].to_i : nil
-      @debug = params[:debug] == "true"
 
       @query_stats =
         begin
@@ -199,9 +202,11 @@ module PgHero
       @replica = PgHero.replica?
     end
 
-    def set_suggested_indexes
-      @suggested_indexes_by_query = PgHero.suggested_indexes_by_query(query_stats: @query_stats)
+    def set_suggested_indexes(min_average_time = 0, min_calls = 0)
+      @suggested_indexes_by_query = PgHero.suggested_indexes_by_query(query_stats: @query_stats.select { |qs| qs["average_time"].to_f >= min_average_time && qs["calls"].to_i >= min_calls })
       @suggested_indexes = PgHero.suggested_indexes(suggested_indexes_by_query: @suggested_indexes_by_query)
+      @query_stats_by_query = @query_stats.index_by { |q| q["query"] }
+      @debug = params[:debug] == "true"
     end
   end
 end

data/app/views/layouts/pg_hero/application.html.erb

@@ -420,9 +420,6 @@
             <li class="<%= controller.action_name == "index_usage" ? "active" : "" %>"><%= link_to "Index Usage", index_usage_path %></li>
             <li class="<%= controller.action_name == "space" ? "active" : "" %>"><%= link_to "Space", space_path %></li>
             <li class="<%= controller.action_name == "connections" ? "active" : "" %>"><%= link_to "Connections", connections_path %></li>
-            <% unless @replica %>
-              <li class="<%= controller.action_name == "maintenance" ? "active" : "" %>"><%= link_to "Maintenance", maintenance_path %></li>
-            <% end %>
             <li class="<%= controller.action_name == "live_queries" ? "active" : "" %>"><%= link_to "Live Queries", live_queries_path %></li>
             <li class="<%= controller.action_name == "explain" ? "active" : "" %>"><%= link_to "Explain", explain_path %></li>
             <li class="<%= controller.action_name == "tune" ? "active" : "" %>"><%= link_to "Tune", tune_path %></li>

data/app/views/pg_hero/home/_queries_table.html.erb

@@ -42,18 +42,19 @@
       </tr>
       <tr>
         <td colspan="3" style="border-top: none; padding: 0;">
-          <pre><%= query["query"] %></pre>
+          <code><pre style="max-height: 230px; overflow: hidden;" onclick="this.style.maxHeight = 'none';"><%= query["query"] %></pre></code>
           <% if query["query"] == "<insufficient privilege>" %>
             <p class="text-muted">For security reasons, only superusers can see queries executed by other users.</p>
           <% end %>
-          <% if (i2 = @suggested_indexes_by_query[query["query"]]) %>
+          <% if local_assigns[:suggested_indexes] != false && (i2 = @suggested_indexes_by_query[query["query"]]) %>
             <% if (index = i2[:index]) && !i2[:covering_index] %>
               <%= render partial: "suggested_index", locals: {index: index} %>
             <% end %>
             <% if @debug %>
-              <code><pre style="color: #f0ad4e; background-color: #333;"><% if i2[:explanation] %><%= i2[:explanation] %><% else %>Rows: <%= i2[:rows] %>
+              <code><pre style="color: #f0ad4e; background-color: #333;"><% if i2[:explanation] %><%= i2[:explanation] %><% end %>
+<% if i2[:row_estimates] %>Rows: <%= i2[:rows] %>
 Row estimates: <%= i2[:row_estimates].to_a.map { |k, v| "#{k}=#{v}" }.join(", ") %>
-Row progresssion: <%= i2[:row_progression].to_a.join(", ") %><% end %></pre></code>
+Row progression: <%= i2[:row_progression].to_a.join(", ") %><% end %></pre></code>
             <% end %>
           <% end %>
         </td>

data/app/views/pg_hero/home/_suggested_index.html.erb

@@ -1 +1 @@
-<code><pre style="color: #eee; background-color: #333;">CREATE INDEX CONCURRENTLY ON <%= index[:table] %> (<%= index[:columns].join(", ") %>)</pre></code>
+<code><pre style="color: #eee; background-color: #333;">CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)</pre></code>

data/app/views/pg_hero/home/index.html.erb

@@ -1,30 +1,3 @@
-<% if @transaction_id_danger.any? %>
-  <div class="alert alert-danger">
-    Database shutdown imminent due to transaction ID wraparound failure
-  </div>
-  <div class="content">
-    <p>For each table, run:</p>
-    <code><pre>VACUUM FREEZE VERBOSE table</pre></code>
-    <p><%= link_to "Read more", "http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND", target: "_blank" %></p>
-    <table class="table">
-      <thead>
-        <tr>
-          <th>Table</th>
-          <th style="width: 20%;">Transactions Before Shutdown</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @transaction_id_danger.each do |query| %>
-          <tr>
-            <td><%= query["table"] %></td>
-            <td><%= number_with_delimiter(query["transactions_before_shutdown"]) %></td>
-          </tr>
-        <% end %>
-      </tbody>
-    </table>
-  </div>
-<% end %>
-
 <div id="status">
   <% if @replica %>
     <div class="alert alert-<%= @good_replication_lag ? "success" : "warning" %>">
@@ -43,15 +16,6 @@
       No long running queries
     <% end %>
   </div>
-  <% if PgHero.suggested_indexes_enabled? %>
-    <div class="alert alert-<%= @suggested_indexes.empty? ? "success" : "warning" %>">
-      <% if @suggested_indexes.any? %>
-        <%= pluralize(@suggested_indexes.size, "suggested index", "suggested indexes") %>
-      <% else %>
-        No suggested indexes
-      <% end %>
-    </div>
-  <% end %>
   <div class="alert alert-<%= @good_cache_rate ? "success" : "warning" %>">
     <% if @good_cache_rate %>
       Cache hit rate above <%= PgHero.cache_hit_rate_threshold %>%
@@ -67,20 +31,11 @@
     <% end %>
     <span class="tiny"><%= @total_connections %></span>
   </div>
-  <div class="alert alert-<%= @autovacuum_danger.empty? ? "success" : "warning" %>">
-    <% if @autovacuum_danger.any? %>
-      <%= pluralize(@autovacuum_danger.size, "table") %> approaching autovacuum freeze max age
+  <div class="alert alert-<%= @transaction_id_danger.empty? ? "success" : "warning" %>">
+    <% if @transaction_id_danger.any? %>
+      <%= pluralize(@transaction_id_danger.size, "table") %> not vacuuming properly
     <% else %>
-      No autovacuum danger
-    <% end %>
-  </div>
-  <div class="alert alert-<%= @query_stats_enabled && @slow_queries.empty? ? "success" : "warning" %>">
-    <% if !@query_stats_enabled %>
-      Query stats must be enabled for slow queries
-    <% elsif @slow_queries.any? %>
-      <%= pluralize(@slow_queries.size, "slow query") %>
-    <% else %>
-      No slow queries
+      Vacuuming healthy
     <% end %>
   </div>
   <div class="alert alert-<%= @invalid_indexes.empty? ? "success" : "warning" %>">
@@ -97,18 +52,30 @@
       No duplicate indexes
     <% end %>
   </div>
-  <div class="alert alert-<%= @missing_indexes.empty? ? "success" : "warning" %>">
-    <% if @missing_indexes.any? %>
-      <%= pluralize(@missing_indexes.size, "table appears", "tables appear") %> to be missing indexes
-    <% else %>
-      No missing indexes
-    <% end %>
-  </div>
-  <div class="alert alert-<%= @unused_indexes.empty? ? "success" : "warning" %>">
-    <% if @unused_indexes.any? %>
-      <%= pluralize(@unused_indexes.size, "unused index", "unused indexes") %>
+  <% if PgHero.suggested_indexes_enabled? %>
+    <div class="alert alert-<%= @suggested_indexes.empty? ? "success" : "warning" %>">
+      <% if @suggested_indexes.any? %>
+        <%= pluralize(@suggested_indexes.size, "suggested index", "suggested indexes") %>
+      <% else %>
+        No suggested indexes
+      <% end %>
+    </div>
+  <% else %>
+    <div class="alert alert-<%= @missing_indexes.empty? ? "success" : "warning" %>">
+      <% if @missing_indexes.any? %>
+        <%= pluralize(@missing_indexes.size, "table appears", "tables appear") %> to be missing indexes
+      <% else %>
+        No missing indexes
+      <% end %>
+    </div>
+  <% end %>
+  <div class="alert alert-<%= @query_stats_enabled && @slow_queries.empty? ? "success" : "warning" %>">
+    <% if !@query_stats_enabled %>
+      Query stats must be enabled for slow queries
+    <% elsif @slow_queries.any? %>
+      <%= pluralize(@slow_queries.size, "slow query") %>
     <% else %>
-      No unused indexes
+      No slow queries
     <% end %>
   </div>
 </div>
@@ -130,35 +97,6 @@
   </div>
 <% end %>
 
-<% if @suggested_indexes.any? %>
-  <div class="content">
-    <h1>Suggested Indexes</h1>
-    <p>
-      Add indexes to speed up queries.
-      <% if @show_migrations %>
-      Here’s a
-      <a href="javascript: void(0);" onclick="document.getElementById('migration3').style.display = 'block';">migration</a> to help.
-      <% end %>
-    </p>
-
-    <div id="migration3" style="display: none;">
-      <pre>rails g migration add_suggested_indexes</pre>
-      <p>And paste</p>
-      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @suggested_indexes.each do |index| %>
-add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(" ,") %>], algorithm: :concurrently<% end %></pre>
-    </div>
-
-    <% @suggested_indexes.each_with_index do |index, i| %>
-      <hr />
-      <%= render partial: "suggested_index", locals: {index: index} %>
-      <p>to speed up</p>
-      <% index[:queries].each do |query| %>
-        <code><pre><%= query %></pre></code>
-      <% end %>
-    <% end %>
-  </div>
-<% end %>
-
 <% if !@good_cache_rate %>
   <div class="content">
     <h1>Low Cache Hit Rate</h1>
@@ -183,24 +121,24 @@ add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym
   </div>
 <% end %>
 
-<% if @autovacuum_danger.any? %>
+<% if @transaction_id_danger.any? %>
   <div class="content">
-    <h1>Autovacuum Danger</h1>
+    <h2>Vacuuming Needed</h2>
+    <p>The database <strong>will shutdown</strong> when there are fewer than 1,000,000 transactions left. <%= link_to "Read more", "http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND", target: "_blank" %>.</p>
     <p>For each table, run:</p>
     <code><pre>VACUUM FREEZE VERBOSE table</pre></code>
-    <p><%= link_to "Read more", "http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND", target: "_blank" %></p>
     <table class="table">
       <thead>
         <tr>
           <th>Table</th>
-          <th style="width: 20%;">Transactions Before Autovacuum</th>
+          <th style="width: 20%;">Transactions Left</th>
         </tr>
       </thead>
       <tbody>
-        <% @autovacuum_danger.each do |query| %>
+        <% @transaction_id_danger.each do |query| %>
           <tr>
             <td><%= query["table"] %></td>
-            <td><%= number_with_delimiter(query["transactions_before_autovacuum"]) %></td>
+            <td><%= number_with_delimiter(query["transactions_before_shutdown"]) %></td>
           </tr>
         <% end %>
       </tbody>
@@ -208,35 +146,6 @@ add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym
   </div>
 <% end %>
 
-<% if !@query_stats_enabled %>
-  <div class="content">
-    <h1>Query Stats</h1>
-
-    <% if @query_stats_available %>
-      <p>
-        Query stats are available but not enabled.
-        <%= button_to "Enable", enable_query_stats_path, class: "btn btn-info" %>
-      </p>
-    <% else %>
-      <p>Make them available by adding the following lines to <code>postgresql.conf</code>:</p>
-      <pre>shared_preload_libraries = 'pg_stat_statements'
-pg_stat_statements.track = all</pre>
-      <p>Restart the server for the changes to take effect.</p>
-    <% end %>
-  </div>
-<% end %>
-
-<% if @query_stats_enabled && @slow_queries.any? %>
-  <div class="content">
-    <h1>Slow Queries</h1>
-
-    <p>Slow queries take <%= PgHero.slow_query_ms %> ms or more on average and have been called at least <%= PgHero.slow_query_calls %> times.</p>
-    <p><%= link_to "Explain queries", explain_path %> to see where to add indexes.</p>
-
-    <%= render partial: "queries_table", locals: {queries: @slow_queries} %>
-  </div>
-<% end %>
-
 <% if @invalid_indexes.any? %>
   <div class="content">
     <h1>Invalid Indexes</h1>
@@ -303,6 +212,36 @@ remove_index <%= query["unneeded_index"]["table"].to_sym.inspect %>, name: <%= q
   </div>
 <% end %>
 
+<% if @suggested_indexes.any? %>
+  <div class="content">
+    <h1>Suggested Indexes</h1>
+    <p>
+      Add indexes to speed up queries.
+      <% if @show_migrations %>
+      Here’s a
+      <a href="javascript: void(0);" onclick="document.getElementById('migration3').style.display = 'block';">migration</a> to help.
+      <% end %>
+    </p>
+
+    <div id="migration3" style="display: none;">
+      <pre>rails g migration add_suggested_indexes</pre>
+      <p>And paste</p>
+      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @suggested_indexes.each do |index| %>
+<% if index[:using] == "gist" %>
+connection.execute("CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)")
+<% else %>
+add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(" ,") %>], algorithm: :concurrently<% end %><% end %></pre>
+    </div>
+
+    <% @suggested_indexes.each_with_index do |index, i| %>
+      <hr />
+      <%= render partial: "suggested_index", locals: {index: index} %>
+      <p>to speed up</p>
+      <%= render partial: "queries_table", locals: {queries: index[:queries].map { |q| @query_stats_by_query[q] }, suggested_indexes: false} %>
+    <% end %>
+  </div>
+<% end %>
+
 <% if @missing_indexes.any? %>
   <div class="content">
     <h1>Missing Indexes</h1>
@@ -330,40 +269,31 @@ remove_index <%= query["unneeded_index"]["table"].to_sym.inspect %>, name: <%= q
   </div>
 <% end %>
 
-<% if @unused_indexes.any? %>
+<% if !@query_stats_enabled %>
   <div class="content">
-    <h1>Unused Indexes</h1>
+    <h1>Query Stats</h1>
 
-    <p>
-      Unused indexes cause unnecessary overhead. Remove them
-      <% if @show_migrations %>
-      <a href="javascript: void(0);" onclick="document.getElementById('migration').style.display = 'block';">with a migration</a>
-      <% end %>
-      for faster writes.
-    </p>
+    <% if @query_stats_available %>
+      <p>
+        Query stats are available but not enabled.
+        <%= button_to "Enable", enable_query_stats_path, class: "btn btn-info" %>
+      </p>
+    <% else %>
+      <p>Make them available by adding the following lines to <code>postgresql.conf</code>:</p>
+      <pre>shared_preload_libraries = 'pg_stat_statements'
+pg_stat_statements.track = all</pre>
+      <p>Restart the server for the changes to take effect.</p>
+    <% end %>
+  </div>
+<% end %>
 
-    <div id="migration" style="display: none;">
-      <pre>rails g migration remove_unused_indexes</pre>
-      <p>And paste</p>
-      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.each do |query| %>
-remove_index <%= query["table"].to_sym.inspect %>, name: <%= query["index"].to_s.inspect %><% end %></pre>
-    </div>
+<% if @query_stats_enabled && @slow_queries.any? %>
+  <div class="content">
+    <h1>Slow Queries</h1>
 
-    <table class="table">
-      <thead>
-        <tr>
-          <th>Name</th>
-          <th style="width: 20%;">Index Size</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @unused_indexes.each do |query| %>
-          <tr>
-            <td><%= query["index"] %><div class="text-muted">on <%= query["table"] %></div></td>
-            <td><%= query["index_size"] %></td>
-          </tr>
-        <% end %>
-      </tbody>
-    </table>
+    <p>Slow queries take <%= PgHero.slow_query_ms %> ms or more on average and have been called at least <%= PgHero.slow_query_calls %> times.</p>
+    <p><%= link_to "Explain queries", explain_path %> to see where to add indexes.</p>
+
+    <%= render partial: "queries_table", locals: {queries: @slow_queries} %>
   </div>
 <% end %>

data/lib/pghero.rb

@@ -375,7 +375,8 @@ module PgHero
     # "the system will shut down and refuse to start any new transactions
     # once there are fewer than 1 million transactions left until wraparound"
     # warn when 10,000,000 transactions left
-    def transaction_id_danger
+    def transaction_id_danger(options = {})
+      threshold = options[:threshold] || 10000000
       select_all <<-SQL
         SELECT
           c.oid::regclass::text AS table,
@@ -386,9 +387,9 @@ module PgHero
           pg_class t ON c.reltoastrelid = t.oid
         WHERE
           c.relkind = 'r'
-          AND (2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < 10000000
+          AND (2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < #{threshold}
         ORDER BY
-          transactions_before_shutdown
+         2, 1
       SQL
     end
 
@@ -805,15 +806,17 @@ module PgHero
         best_indexes = best_index_helper(queries)
 
         if best_indexes.any?
-          existing_columns = Hash.new { |hash, key| hash[key] = [] }
-          self.indexes.each do |i|
-            existing_columns[i["table"]] << i["columns"]
+          existing_columns = Hash.new { |hash, key| hash[key] = Hash.new { |hash2, key2| hash2[key2] = [] } }
+          self.indexes.group_by { |g| g["using"] }.each do |group, inds|
+            inds.each do |i|
+              existing_columns[group][i["table"]] << i["columns"]
+            end
           end
 
           best_indexes.each do |query, best_index|
             if best_index[:found]
               index = best_index[:index]
-              covering_index = existing_columns[index[:table]].find { |e| index_covers?(e, index[:columns]) }
+              covering_index = existing_columns[index[:using] || "btree"][index[:table]].find { |e| index_covers?(e, index[:columns]) }
               if covering_index
                 best_index[:covering_index] = covering_index
                 best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
@@ -859,22 +862,41 @@ module PgHero
     end
 
     def column_stats(options = {})
+      schema = options[:schema]
       tables = options[:table] ? Array(options[:table]) : nil
       select_all <<-SQL
         SELECT
+          schemaname AS schema,
           tablename AS table,
           attname AS column,
           null_frac,
-          n_distinct,
-          n_live_tup
+          n_distinct
         FROM
           pg_stats
+        WHERE
+          #{tables ? "tablename IN (#{tables.map { |t| quote(t) }.join(", ")})" : "1 = 1"}
+          AND schemaname = #{quote(schema)}
+        ORDER BY
+          1, 2, 3
+      SQL
+    end
+
+    def table_stats(options = {})
+      schema = options[:schema]
+      tables = options[:table] ? Array(options[:table]) : nil
+      select_all <<-SQL
+        SELECT
+          nspname AS schema,
+          relname AS table,
+          reltuples::bigint
+        FROM
+          pg_class
         INNER JOIN
-          pg_class ON pg_class.relname = pg_stats.tablename
-        INNER JOIN
-          pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname
+          pg_namespace ON pg_namespace.oid = pg_class.relnamespace
         WHERE
-          #{tables ? "pg_class.relname IN (#{tables.map { |t| quote(t) }.join(", ")})" : "1 = 1"}
+          relkind = 'r'
+          AND nspname = #{quote(schema)}
+          #{tables ? "AND relname IN (#{tables.map { |t| quote(t) }.join(", ")})" : nil}
         ORDER BY
           1, 2
       SQL
@@ -893,8 +915,11 @@ module PgHero
 
       # get stats about columns for relevant tables
       tables = parts.values.map { |t| t[:table] }.uniq
+      # TODO get schema from query structure, then try search path
+      schema = connection_model.connection_config[:schema] || "public"
       if tables.any?
-        column_stats = self.column_stats(table: tables).group_by { |i| i["table"] }
+        row_stats = Hash[self.table_stats(table: tables, schema: schema).map { |i| [i["table"], i["reltuples"]] }]
+        column_stats = self.column_stats(table: tables, schema: schema).group_by { |i| i["table"] }
       end
 
       # find best index based on query structure and column stats
@@ -909,61 +934,68 @@ module PgHero
           index[:structure] = structure
 
           table = structure[:table]
-          where = structure[:where]
+          where = structure[:where].uniq
           sort = structure[:sort]
 
-          ranks = Hash[column_stats[table].to_a.map { |r| [r["column"], r] }]
+          total_rows = row_stats[table].to_i
+          index[:rows] = total_rows
 
+          ranks = Hash[column_stats[table].to_a.map { |r| [r["column"], r] }]
           columns = (where + sort).map { |c| c[:column] }.uniq
 
-          if columns.any? && columns.all? { |c| ranks[c] }
-            first_desc = sort.index { |c| c[:direction] == "desc" }
-            if first_desc
-              sort = sort.first(first_desc + 1)
-            end
-            where = where.sort_by { |c| [row_estimates(ranks[c[:column]], nil, c[:op]), c[:column]] } + sort
-
-            index[:row_estimates] = Hash[where.map { |c| [c[:column], row_estimates(ranks[c[:column]], nil, c[:op]).round] }]
-
-            rows_left = ranks[where.first[:column]]["n_live_tup"].to_i
-            index[:rows] = rows_left
-
-            # no index needed if less than 500 rows
-            if rows_left >= 500
-
-              # if most values are unique, no need to index others
-              final_where = []
-              prev_rows_left = [rows_left]
-              where.each do |c|
-                next if final_where.include?(c[:column])
-                final_where << c[:column]
-                rows_left = row_estimates(ranks[c[:column]], rows_left, c[:op])
-                prev_rows_left << rows_left
-                if rows_left < 50
-                  break
-                end
+          if columns.any?
+            if columns.all? { |c| ranks[c] }
+              first_desc = sort.index { |c| c[:direction] == "desc" }
+              if first_desc
+                sort = sort.first(first_desc + 1)
               end
+              where = where.sort_by { |c| [row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]), c[:column]] } + sort
+
+              index[:row_estimates] = Hash[where.map { |c| [c[:column], row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
+
+              # no index needed if less than 500 rows
+              if total_rows >= 500
+
+                if ["~~", "~~*"].include?(where.first[:op])
+                  index[:found] = true
+                  index[:index] = {table: table, columns: ["#{where.first[:column]} gist_trgm_ops"], using: "gist"}
+                else
+                  # if most values are unique, no need to index others
+                  rows_left = total_rows
+                  final_where = []
+                  prev_rows_left = [rows_left]
+                  where.reject { |c| ["~~", "~~*"].include?(c[:op]) }.each do |c|
+                    next if final_where.include?(c[:column])
+                    final_where << c[:column]
+                    rows_left = row_estimates(ranks[c[:column]], total_rows, rows_left, c[:op])
+                    prev_rows_left << rows_left
+                    if rows_left < 50 || final_where.size >= 3 || [">", ">=", "<", "<=", "~~", "~~*"].include?(c[:op])
+                      break
+                    end
+                  end
 
-              index[:row_progression] = prev_rows_left.map(&:round)
+                  index[:row_progression] = prev_rows_left.map(&:round)
 
-              # if the last indexes don't give us much, don't include
-              if prev_rows_left.last > 50
-                prev_rows_left.reverse!
-                (prev_rows_left.size - 1).times do |i|
-                  if prev_rows_left[i] > prev_rows_left[i + 1] * 0.1
-                    final_where.pop
-                  else
-                    break
+                  # if the last indexes don't give us much, don't include
+                  prev_rows_left.reverse!
+                  (prev_rows_left.size - 1).times do |i|
+                    if prev_rows_left[i] > prev_rows_left[i + 1] * 0.3
+                      final_where.pop
+                    else
+                      break
+                    end
                   end
-                end
-              end
 
-              if final_where.any?
-                index[:found] = true
-                index[:index] = {table: table, columns: final_where}
+                  if final_where.any?
+                    index[:found] = true
+                    index[:index] = {table: table, columns: final_where}
+                  end
+                end
+              else
+                index[:explanation] = "No index needed if less than 500 rows"
               end
             else
-              index[:explanation] = "No index needed if less than 500 rows"
+              index[:explanation] = "Stats not found"
             end
           else
             index[:explanation] = "No columns to index"
@@ -993,18 +1025,19 @@ module PgHero
             "INSERT statement"
           when "SET"
             "SET statement"
-          else
-            "Unknown structure"
+          when "SELECT"
+            if (tree["SELECT"]["fromClause"].first["JOINEXPR"] rescue false)
+              "JOIN not supported yet"
+            end
           end
-        return {error: error}
+        return {error: error || "Unknown structure"}
       end
 
       select = tree["SELECT"] || tree["DELETE FROM"] || tree["UPDATE"]
       where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
       return {error: "Unknown structure"} unless where
 
-      sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue nil
-      return {error: "Unknown structure"} unless sort
+      sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue []
 
       {table: table, where: where, sort: sort}
     end
@@ -1015,8 +1048,7 @@ module PgHero
 
     # TODO better row estimation
     # http://www.postgresql.org/docs/current/static/row-estimation-examples.html
-    def row_estimates(stats, rows_left = nil, op = nil)
-      rows_left ||= stats["n_live_tup"].to_i
+    def row_estimates(stats, total_rows, rows_left, op)
       case op
       when "null"
         rows_left * stats["null_frac"].to_f
@@ -1024,16 +1056,26 @@ module PgHero
         rows_left * (1 - stats["null_frac"].to_f)
       else
         rows_left *= (1 - stats["null_frac"].to_f)
-        if stats["n_distinct"].to_f == 0
-          0
-        elsif stats["n_distinct"].to_f < 0
-          if stats["n_live_tup"].to_i > 0
-            (-1 / stats["n_distinct"].to_f) * (rows_left / stats["n_live_tup"].to_f)
-          else
+        ret =
+          if stats["n_distinct"].to_f == 0
             0
+          elsif stats["n_distinct"].to_f < 0
+            if total_rows > 0
+              (-1 / stats["n_distinct"].to_f) * (rows_left / total_rows.to_f)
+            else
+              0
+            end
+          else
+            rows_left / stats["n_distinct"].to_f
           end
+
+        case op
+        when ">", ">=", "<", "<=", "~~", "~~*"
+          (rows_left + ret) / 10.0 # TODO better approximation
+        when "<>"
+          rows_left - ret
         else
-          rows_left / stats["n_distinct"].to_f
+          ret
         end
       end
     end
@@ -1059,10 +1101,10 @@ module PgHero
         if left && right
           left + right
         end
-      elsif tree["AEXPR"] && ["="].include?(tree["AEXPR"]["name"].first)
+      elsif tree["AEXPR"] && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*"].include?(tree["AEXPR"]["name"].first)
         [{column: tree["AEXPR"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR"]["name"].first}]
-      elsif tree["AEXPR IN"] && tree["AEXPR IN"]["name"].first == "="
-        [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: "in"}]
+      elsif tree["AEXPR IN"] && ["=", "<>"].include?(tree["AEXPR IN"]["name"].first)
+        [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR IN"]["name"].first}]
       elsif tree["NULLTEST"]
         op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
         [{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "1.2.0"
+  VERSION = "1.2.1"
 end

data/pghero.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.version       = PgHero::VERSION
   spec.authors       = ["Andrew Kane"]
   spec.email         = ["andrew@chartkick.com"]
-  spec.summary       = "Database insights made easy"
-  spec.description   = "Database insights made easy"
+  spec.summary       = "The missing dashboard for Postgres"
+  spec.description   = "The missing dashboard for Postgres"
   spec.homepage      = "https://github.com/ankane/pghero"
   spec.license       = "MIT"
 

data/test/best_index_test.rb

@@ -50,15 +50,46 @@ class BestIndexTest < Minitest::Test
     assert_best_index ({table: "users", columns: ["login_attempts", "created_at"]}), "SELECT * FROM users WHERE login_attempts = 1 ORDER BY created_at"
   end
 
+  def test_where_order_unknown
+    assert_best_index ({table: "users", columns: ["login_attempts"]}), "SELECT * FROM users WHERE login_attempts = 1 ORDER BY NOW()"
+  end
+
   def test_where_in
     assert_best_index ({table: "users", columns: ["city_id"]}), "SELECT * FROM users WHERE city_id IN (1, 2)"
   end
 
+  def test_like
+    assert_best_index ({table: "users", columns: ["email gist_trgm_ops"], using: "gist"}), "SELECT * FROM users WHERE email LIKE ?"
+  end
+
+  def test_like_where
+    assert_best_index ({table: "users", columns: ["city_id"]}), "SELECT * FROM users WHERE city_id = ? AND email LIKE ?"
+  end
+
+  def test_like_where2
+    assert_best_index ({table: "users", columns: ["email gist_trgm_ops"], using: "gist"}), "SELECT * FROM users WHERE email LIKE ? AND active = ?"
+  end
+
+  def test_ilike
+    assert_best_index ({table: "users", columns: ["email gist_trgm_ops"], using: "gist"}), "SELECT * FROM users WHERE email ILIKE ?"
+  end
+
+  def test_not_equals
+    assert_best_index ({table: "users", columns: ["login_attempts"]}), "SELECT * FROM users WHERE city_id != ? and login_attempts = 2"
+  end
+
+  def test_not_in
+    assert_best_index ({table: "users", columns: ["login_attempts"]}), "SELECT * FROM users WHERE city_id NOT IN (?) and login_attempts = 2"
+  end
+
   def test_between
-    skip
     assert_best_index ({table: "users", columns: ["city_id"]}), "SELECT * FROM users WHERE city_id BETWEEN 1 AND 2"
   end
 
+  def test_multiple_range
+    assert_best_index ({table: "users", columns: ["city_id"]}), "SELECT * FROM users WHERE city_id > ? and login_attempts > ?"
+  end
+
   def test_where_prepared
     assert_best_index ({table: "users", columns: ["city_id"]}), "SELECT * FROM users WHERE city_id = $1"
   end
@@ -91,8 +122,16 @@ class BestIndexTest < Minitest::Test
     assert_no_index "Parse error", "SELECT *123'"
   end
 
+  def test_stats_not_found
+    assert_no_index "Stats not found", "SELECT * FROM non_existent_table WHERE id = 1"
+  end
+
+  def test_unknown_structure
+    assert_no_index "Unknown structure", "SELECT NOW()"
+  end
+
   def test_multiple_tables
-    assert_no_index "Unknown structure", "SELECT * FROM users INNER JOIN cities ON cities.id = users.city_id"
+    assert_no_index "JOIN not supported yet", "SELECT * FROM users INNER JOIN cities ON cities.id = users.city_id"
   end
 
   def test_no_columns
@@ -119,6 +158,7 @@ class BestIndexTest < Minitest::Test
 
   def assert_best_index(expected, statement)
     index = PgHero.best_index(statement)
+    assert_nil index[:explanation]
     assert index[:found]
     assert_equal expected, index[:index]
   end

data/test/test_helper.rb

@@ -28,6 +28,7 @@ if ENV["SEED"]
     t.integer :login_attempts
     t.string :email
     t.string :zip_code
+    t.boolean :active
     t.timestamp :created_at
   end
 
@@ -39,6 +40,7 @@ if ENV["SEED"]
         email: "person#{i}@example.org",
         login_attempts: rand(30),
         zip_code: i % 40 == 0 ? nil : "12345",
+        active: true,
         created_at: Time.now - rand(50).days
       )
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-31 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,7 +80,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Database insights made easy
+description: The missing dashboard for Postgres
 email:
 - andrew@chartkick.com
 executables: []
@@ -155,7 +155,7 @@ rubyforge_project:
 rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Database insights made easy
+summary: The missing dashboard for Postgres
 test_files:
 - guides/Docker.md
 - guides/Heroku.md
@@ -170,3 +170,4 @@ test_files:
 - test/gemfiles/activerecord40.gemfile
 - test/suggested_indexes_test.rb
 - test/test_helper.rb
+has_rdoc: