pghero 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3975a93a4da7c4199e2e3a5f98c4c98e35c991e9
-  data.tar.gz: 7a2a1ba863c86c42d09316c624cbd9fa411bda41
+  metadata.gz: d444b648639843cbae37f0e554e47ff37462222d
+  data.tar.gz: 2482c623785d74f933b836e89233b0c2c44a6e75
 SHA512:
-  metadata.gz: 745ac33fea1c485e3263075b4a17c04b79894f240776a65331818a160b8ef3a0f308f362aeb590c433be96e1f684dd36d764ae3087bf16ef4826d284c168f8c7
-  data.tar.gz: c671471aa2a742f428a73b8d05c4af7624baa324e80654ae3d20c3b4fbf1c615ee942860b383c6c03c76ae798efe27476f4adb2152a4744f007de924bdbcc0c5
+  metadata.gz: 648695c06b8a8264eb9380f0a6b2700c7846540d8490be0d476ba66d063bc73896baadcf9235057b513a7fdf8dbea8819d60a03526d9c32cb5082ab72264a9df
+  data.tar.gz: d69b31b33eb386ee52e761dfee89884e928012913c963aa1ab4758397007753952a17a2506fe930a667a6e29686eea62a8949008e422a809a9382412a49916a9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.2.2
+
+- Better suggested indexes
+- Removed duplicate indexes noise
+- Fixed partial and expression indexes
+
 ## 1.2.1
 
 - Better suggested indexes

data/Gemfile

@@ -2,5 +2,3 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in pghero.gemspec
 gemspec
-
-gem "pg_query"

data/app/controllers/pg_hero/home_controller.rb

@@ -24,7 +24,6 @@ module PgHero
         end
       @unused_indexes = PgHero.unused_indexes.select { |q| q["index_scans"].to_i == 0 }
       @invalid_indexes = PgHero.invalid_indexes
-      @duplicate_indexes = PgHero.duplicate_indexes
       @good_cache_rate = @table_hit_rate >= PgHero.cache_hit_rate_threshold.to_f / 100 && @index_hit_rate >= PgHero.cache_hit_rate_threshold.to_f / 100
       @query_stats_available = PgHero.query_stats_available?
       @total_connections = PgHero.total_connections

data/app/views/layouts/pg_hero/application.html.erb

@@ -82,6 +82,7 @@
         border: none;
         height: 0;
         border-top: solid 1px #ddd;
+        margin-bottom: 15px;
       }
 
       .container {

data/app/views/pg_hero/home/_queries_table.html.erb

@@ -46,16 +46,8 @@
           <% if query["query"] == "<insufficient privilege>" %>
             <p class="text-muted">For security reasons, only superusers can see queries executed by other users.</p>
           <% end %>
-          <% if local_assigns[:suggested_indexes] != false && (i2 = @suggested_indexes_by_query[query["query"]]) %>
-            <% if (index = i2[:index]) && !i2[:covering_index] %>
-              <%= render partial: "suggested_index", locals: {index: index} %>
-            <% end %>
-            <% if @debug %>
-              <code><pre style="color: #f0ad4e; background-color: #333;"><% if i2[:explanation] %><%= i2[:explanation] %><% end %>
-<% if i2[:row_estimates] %>Rows: <%= i2[:rows] %>
-Row estimates: <%= i2[:row_estimates].to_a.map { |k, v| "#{k}=#{v}" }.join(", ") %>
-Row progression: <%= i2[:row_progression].to_a.join(", ") %><% end %></pre></code>
-            <% end %>
+          <% if local_assigns[:suggested_indexes] != false && (details = @suggested_indexes_by_query[query["query"]]) %>
+            <%= render partial: "suggested_index", locals: {index: details[:index], details: details} %>
           <% end %>
         </td>
       </tr>

data/app/views/pg_hero/home/_suggested_index.html.erb

@@ -1 +1,18 @@
-<code><pre style="color: #eee; background-color: #333;">CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)</pre></code>
+<% if index && !details[:covering_index] %>
+  <% unless @debug %>
+    <div style="float: right; color: #f0ad4e; margin-top: 0px; padding: 10px; cursor: pointer;" onclick="document.getElementById('details-<%= index.object_id %>').style.display = 'block'; this.style.display = 'none';">Details</div>
+  <% end %>
+  <code><pre style="color: #eee; background-color: #333;">CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)</pre></code>
+<% end %>
+<div id="details-<%= index.object_id %>" style="<%= "display: none;" unless @debug %>">
+  <code><pre style="color: #f0ad4e; background-color: #333;"><% if details[:explanation] %><%= details[:explanation] %>
+<% end %><% if details[:row_estimates] %>Rows: <%= details[:rows] %>
+Row progression: <%= details[:row_progression].to_a.join(", ") %>
+
+Row estimates
+<%= details[:row_estimates].to_a.map { |k, v| "- #{k}: #{v}" }.join("\n") %><% end %><% if details[:table_indexes] %>
+
+Existing indexes
+<% details[:table_indexes].sort_by { |i| [i["primary"] == "t" ? 0 : 1, i["columns"]] }.each do |i3| %>- <%= i3["columns"].join(", ") %><% if i3["using"] != "btree" %> <%= i3["using"].to_s.upcase %><% end %><% if i3["primary"] == "t" %> PRIMARY<% elsif i3["unique"] != "f" %> UNIQUE<% end %>
+<% end %><% end %></pre></code>
+</div>

data/app/views/pg_hero/home/index.html.erb

@@ -45,13 +45,6 @@
       No invalid indexes
     <% end %>
   </div>
-  <div class="alert alert-<%= @duplicate_indexes.empty? ? "success" : "warning" %>">
-    <% if @duplicate_indexes.any? %>
-      <%= pluralize(@duplicate_indexes.size, "duplicate index", "duplicate indexes") %>
-    <% else %>
-      No duplicate indexes
-    <% end %>
-  </div>
   <% if PgHero.suggested_indexes_enabled? %>
     <div class="alert alert-<%= @suggested_indexes.empty? ? "success" : "warning" %>">
       <% if @suggested_indexes.any? %>
@@ -169,49 +162,6 @@
   </div>
 <% end %>
 
-<% if @duplicate_indexes.any? %>
-  <div class="content">
-    <h1>Duplicate Indexes</h1>
-
-    <p>
-      These indexes exist, but aren’t needed. Remove them
-      <% if @show_migrations %>
-      <a href="javascript: void(0);" onclick="document.getElementById('migration2').style.display = 'block';">with a migration</a>
-      <% end %>
-      for faster writes.
-    </p>
-
-    <div id="migration2" style="display: none;">
-      <pre>rails g migration remove_unneeded_indexes</pre>
-      <p>And paste</p>
-      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @duplicate_indexes.each do |query| %>
-remove_index <%= query["unneeded_index"]["table"].to_sym.inspect %>, name: <%= query["unneeded_index"]["name"].to_s.inspect %><% end %></pre>
-    </div>
-
-    <table class="table">
-      <thead>
-        <tr>
-          <th>Details</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @duplicate_indexes.each do |index| %>
-          <% unneeded_index = index["unneeded_index"] %>
-          <% covering_index = index["covering_index"] %>
-          <tr>
-            <td style="padding-top: 15px; padding-bottom: 5px;">
-              On <%= unneeded_index["table"] %>
-              <pre><%= unneeded_index["name"] %> (<%= unneeded_index["columns"].join(", ") %>)</pre>
-              is covered by
-              <pre><%= covering_index["name"] %> (<%= covering_index["columns"].join(", ") %>)</pre>
-            </td>
-          </tr>
-        <% end %>
-      </tbody>
-    </table>
-  </div>
-<% end %>
-
 <% if @suggested_indexes.any? %>
   <div class="content">
     <h1>Suggested Indexes</h1>
@@ -230,12 +180,12 @@ remove_index <%= query["unneeded_index"]["table"].to_sym.inspect %>, name: <%= q
 <% if index[:using] == "gist" %>
 connection.execute("CREATE INDEX CONCURRENTLY ON <%= index[:table] %><% if index[:using] %> USING <%= index[:using] %><% end %> (<%= index[:columns].join(", ") %>)")
 <% else %>
-add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(" ,") %>], algorithm: :concurrently<% end %><% end %></pre>
+add_index <%= index[:table].to_sym.inspect %>, [<%= index[:columns].map(&:to_sym).map(&:inspect).join(", ") %>], algorithm: :concurrently<% end %><% end %></pre>
     </div>
 
     <% @suggested_indexes.each_with_index do |index, i| %>
       <hr />
-      <%= render partial: "suggested_index", locals: {index: index} %>
+      <%= render partial: "suggested_index", locals: {index: index, details: index[:details]} %>
       <p>to speed up</p>
       <%= render partial: "queries_table", locals: {queries: index[:queries].map { |q| @query_stats_by_query[q] }, suggested_indexes: false} %>
     <% end %>

data/app/views/pg_hero/home/queries.html.erb

@@ -9,10 +9,6 @@
     <%= render partial: "query_stats_slider" %>
   <% end %>
 
-  <% if PgHero.suggested_indexes_enabled? %>
-    <p style="text-align: center; margin-top: 7px;">PgHero now suggests indexes. <%= link_to "See how it thinks", {debug: true} %>.</p>
-  <% end %>
-
   <% if @query_stats_enabled %>
     <% if @error %>
       <div class="alert alert-danger">Cannot understand start or end time.</div>

data/lib/pghero.rb

@@ -757,13 +757,14 @@ module PgHero
         SELECT
           t.relname AS table,
           ix.relname AS name,
-          regexp_replace(pg_get_indexdef(indexrelid), '.*\\((.*)\\)', '\\1') AS columns,
-          regexp_replace(pg_get_indexdef(indexrelid), '.* USING (.*) \\(.*', '\\1') AS using,
+          regexp_replace(pg_get_indexdef(indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
+          regexp_replace(pg_get_indexdef(indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
           indisunique AS unique,
           indisprimary AS primary,
           indisvalid AS valid,
           indexprs::text,
-          indpred::text
+          indpred::text,
+          pg_get_indexdef(indexrelid) AS definition
         FROM
           pg_index i
         INNER JOIN
@@ -773,7 +774,7 @@ module PgHero
         ORDER BY
           1, 2
       SQL
-      ).map { |v| v["columns"] = v["columns"].split(", "); v }
+      ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", "); v }
     end
 
     def duplicate_indexes
@@ -807,15 +808,18 @@ module PgHero
 
         if best_indexes.any?
           existing_columns = Hash.new { |hash, key| hash[key] = Hash.new { |hash2, key2| hash2[key2] = [] } }
-          self.indexes.group_by { |g| g["using"] }.each do |group, inds|
+          indexes = self.indexes
+          indexes.group_by { |g| g["using"] }.each do |group, inds|
             inds.each do |i|
               existing_columns[group][i["table"]] << i["columns"]
             end
           end
+          indexes_by_table = indexes.group_by { |i| i["table"] }
 
           best_indexes.each do |query, best_index|
             if best_index[:found]
               index = best_index[:index]
+              best_index[:table_indexes] = indexes_by_table[index[:table]].to_a
               covering_index = existing_columns[index[:using] || "btree"][index[:table]].find { |e| index_covers?(e, index[:columns]) }
               if covering_index
                 best_index[:covering_index] = covering_index
@@ -833,7 +837,11 @@ module PgHero
       indexes = []
 
       (options[:suggested_indexes_by_query] || suggested_indexes_by_query(options)).select { |s, i| i[:found] && !i[:covering_index] }.group_by { |s, i| i[:index] }.each do |index, group|
-        indexes << index.merge(queries: group.map(&:first))
+        details = {}
+        group.map(&:second).each do |g|
+          details = details.except(:index).deep_merge(g)
+        end
+        indexes << index.merge(queries: group.map(&:first), details: details)
       end
 
       indexes.sort_by { |i| [i[:table], i[:columns]] }
@@ -951,13 +959,14 @@ module PgHero
               end
               where = where.sort_by { |c| [row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]), c[:column]] } + sort
 
-              index[:row_estimates] = Hash[where.map { |c| [c[:column], row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
+              index[:row_estimates] = Hash[where.map { |c| ["#{c[:column]} (#{c[:op] || "sort"})", row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
 
               # no index needed if less than 500 rows
               if total_rows >= 500
 
                 if ["~~", "~~*"].include?(where.first[:op])
                   index[:found] = true
+                  index[:row_progression] = [total_rows, index[:row_estimates].values.first]
                   index[:index] = {table: table, columns: ["#{where.first[:column]} gist_trgm_ops"], using: "gist"}
                 else
                   # if most values are unique, no need to index others
@@ -969,7 +978,7 @@ module PgHero
                     final_where << c[:column]
                     rows_left = row_estimates(ranks[c[:column]], total_rows, rows_left, c[:op])
                     prev_rows_left << rows_left
-                    if rows_left < 50 || final_where.size >= 3 || [">", ">=", "<", "<=", "~~", "~~*"].include?(c[:op])
+                    if rows_left < 50 || final_where.size >= 2 || [">", ">=", "<", "<=", "~~", "~~*"].include?(c[:op])
                       break
                     end
                   end

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/pghero.gemspec

@@ -23,10 +23,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "activerecord-import"
 
   if RUBY_PLATFORM == "java"
     spec.add_development_dependency "activerecord-jdbcpostgresql-adapter"
   else
     spec.add_development_dependency "pg"
+    spec.add_development_dependency "pg_query"
   end
 end

data/test/best_index_test.rb

@@ -12,7 +12,7 @@ class BestIndexTest < Minitest::Test
       structure: {table: "users", where: [{column: "login_attempts", op: "="}], sort: [{column: "created_at", direction: "asc"}]},
       index: {table: "users", columns: ["login_attempts", "created_at"]},
       rows: 10000,
-      row_estimates: {"login_attempts" => 333, "created_at" => 1},
+      row_estimates: {"login_attempts (=)" => 333, "created_at (sort)" => 1},
       row_progression: [10000, 333, 0]
     }
     assert_equal expected, index

data/test/suggested_indexes_test.rb

@@ -9,6 +9,6 @@ class SuggestedIndexesTest < Minitest::Test
     User.where(email: "person1@example.org").first
     # User.where(email: "person1@example.org", city_id: 1).first
     # User.where(city_id: 1).to_a
-    assert_equal [{table: "users", columns: ["email"]}], PgHero.suggested_indexes.map { |q| q.except(:queries) }
+    assert_equal [{table: "users", columns: ["email"]}], PgHero.suggested_indexes.map { |q| q.except(:queries, :details) }
   end
 end

data/test/test_helper.rb

@@ -3,6 +3,7 @@ Bundler.require(:default)
 require "minitest/autorun"
 require "minitest/pride"
 require "pg_query"
+require "activerecord-import"
 
 # for Minitest < 5
 Minitest::Test = MiniTest::Unit::TestCase unless defined?(Minitest::Test)
@@ -33,17 +34,19 @@ if ENV["SEED"]
   end
 
   User.transaction do
-    10000.times do |i|
-      city_id = i % 100
-      User.create!(
-        city_id: city_id,
-        email: "person#{i}@example.org",
-        login_attempts: rand(30),
-        zip_code: i % 40 == 0 ? nil : "12345",
-        active: true,
-        created_at: Time.now - rand(50).days
-      )
-    end
+    users =
+      10000.times.map do |i|
+        city_id = i % 100
+        User.new(
+          city_id: city_id,
+          email: "person#{i}@example.org",
+          login_attempts: rand(30),
+          zip_code: i % 40 == 0 ? nil : "12345",
+          active: true,
+          created_at: Time.now - rand(50).days
+        )
+      end
+    User.import users, validate: false
   end
   ActiveRecord::Base.connection.execute("VACUUM ANALYZE users")
 
@@ -52,9 +55,11 @@ if ENV["SEED"]
   end
 
   State.transaction do
-    50.times do |i|
-      State.create!(name: "State #{i}")
-    end
+    states =
+      50.times.map do |i|
+        State.new(name: "State #{i}")
+      end
+    State.import states, validate: false
   end
   ActiveRecord::Base.connection.execute("VACUUM ANALYZE states")
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg_query
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: The missing dashboard for Postgres
 email:
 - andrew@chartkick.com