pghero 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b627e79e52bdd3c9eb7f850e4350cd286459ac3771ec94c33b527e4061cad6d6
-  data.tar.gz: 33773361eb689372f74d012e5d603c76171abc153dd3eccff3ec6ef15012de1f
+  metadata.gz: 344a413582024c07e65a19f69e4e5a8aa5bd9024d40ce82357ce234349c8f52f
+  data.tar.gz: a9c4c8f537f9e2067582d9dba2b44f7fd83b43ed8ab6cc5aeaeaa04107087e96
 SHA512:
-  metadata.gz: 857c4d36050a5532240173faae721bf93f6219641ed005a77ff3d45fd4eda6aa78e10d2d9c2bc8cc6e93547eb97c47006802b9bfad5f02f186535d1eb219e779
-  data.tar.gz: efba82dcf9d151aee9c94adac95ca27a032da04a358b435aaa86f7255949d7bf63f1a58f39f18976eeba294a0386223ea16c752c9f1e8285d0aa6fa5c8d29921
+  metadata.gz: 3b7999bab7e77cba83016ed05dde01778f4e79cbe4fa5982750dfedfe46c74438f2700d13ff8e07744c92f1a4c87b2c740bf455921d6004f047a57d2aa982720
+  data.tar.gz: eb91e9b0f35115de7369e521cfa5c0c39854f4d2e8ea650782af2fd28a465e825daa5f7a1f485b6944c00dc7f14855524c034cd7f953d93a4f55718d43110183

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 2.2.0
+
+- Added check for connections idle in transaction
+- Improved duplicate index logic to detect more duplicates
+- Don't report concurrent indexes in-progress as invalid on dashboard
+- Fixed error with large number of sequences
+- Bumped `total_connections_threshold` to 500 by default
+
 ## 2.1.1
 
 - Added `explain_timeout_sec` option

data/CONTRIBUTING.md

@@ -2,17 +2,15 @@
 
 First, thanks for wanting to contribute. You’re awesome! :heart:
 
-## Questions
+## Help
 
-Use [Stack Overflow](https://stackoverflow.com/) with the tag `pghero`.
+We’re not able to provide support through GitHub Issues. If you’re looking for help with your code, try posting on [Stack Overflow](https://stackoverflow.com/).
 
-## Feature Requests
+All features should be documented. If you don’t see a feature in the docs, assume it doesn’t exist.
 
-Create an issue. Start the title with `[Idea]`.
+## Bugs
 
-## Issues
-
-Think you’ve discovered an issue?
+Think you’ve discovered a bug?
 
 1. Search existing issues to see if it’s been reported.
 2. Try the `master` branch to make sure it hasn’t been fixed.
@@ -26,6 +24,10 @@ If the above steps don’t help, create an issue. Include:
 - Detailed steps to reproduce
 - Complete backtraces for exceptions
 
+## New Features
+
+If you’d like to discuss a new feature, create an issue and start the title with `[Idea]`.
+
 ## Pull Requests
 
 Fork the project and create a pull request. A few tips:

data/README.md

@@ -1,14 +1,16 @@
 # PgHero
 
-A performance dashboard for Postgres - health checks, suggested indexes, and more :tada:
+A performance dashboard for Postgres
 
 [See it in action](https://pghero.dokkuapp.com/)
 
-[![Screenshot](https://pghero.dokkuapp.com/assets/screenshot-5a368624ada55b32e7668c96926840f9.png)](https://pghero.dokkuapp.com/)
+---
+
+**NEW** PgHero is also available as a [hosted service](https://pghero.org). Get real-time alerts, single sign-on, and more :tada:
 
-PgHero 2.0 was just released! [Read details](https://medium.com/@ankane/pghero-2-0-has-arrived-266c3874d012).
+---
 
-:speech_balloon: Get [handcrafted updates](http://chartkick.us7.list-manage.com/subscribe?u=952c861f99eb43084e0a49f98&id=6ea6541e8e&group[0][16]=true) for new features
+[![Screenshot](https://pghero.dokkuapp.com/assets/screenshot-5a368624ada55b32e7668c96926840f9.png)](https://pghero.dokkuapp.com/)
 
 ## Installation
 

data/app/controllers/pg_hero/home_controller.rb

@@ -33,8 +33,12 @@ module PgHero
 
       @autovacuum_queries, @long_running_queries = @database.long_running_queries.partition { |q| q[:query].starts_with?("autovacuum:") }
 
-      @total_connections = @database.total_connections
+      connection_states = @database.connection_states
+      @total_connections = connection_states.values.sum
+      @idle_connections = connection_states["idle in transaction"].to_i
+
       @good_total_connections = @total_connections < @database.total_connections_threshold
+      @good_idle_connections = @idle_connections < 100
 
       @transaction_id_danger = @database.transaction_id_danger(threshold: 1500000000)
 
@@ -43,7 +47,7 @@ module PgHero
       @sequence_danger = @database.sequence_danger(threshold: (params[:sequence_threshold] || 0.9).to_f, sequences: @readable_sequences)
 
       @indexes = @database.indexes
-      @invalid_indexes = @indexes.select { |i| !i[:valid] }
+      @invalid_indexes = @database.invalid_indexes(indexes: @indexes)
       @duplicate_indexes = @database.duplicate_indexes(indexes: @indexes)
 
       if @query_stats_enabled
@@ -109,6 +113,10 @@ module PgHero
       @title = "Live Queries"
       @running_queries = @database.running_queries(all: true)
       @vacuum_progress = @database.vacuum_progress.index_by { |q| q[:pid] }
+
+      if params[:state]
+        @running_queries.select! { |q| q[:state] == params[:state] }
+      end
     end
 
     def queries

data/app/views/pg_hero/home/explain.html.erb

@@ -12,7 +12,7 @@
 
   <% if @explanation %>
     <% if @visualize %>
-      <p>Paste the output below into the <%= link_to "Postgres Explain Visualizer", "http://tatiyants.com/pev/#/plans/new", target: "_blank" %></p>
+      <p>Paste the output below into the <%= link_to "Postgres Explain Visualizer", "https://tatiyants.com/pev/#/plans/new", target: "_blank" %></p>
     <% end %>
     <pre><code><%= @explanation %></code></pre>
     <% unless @visualize %>

data/app/views/pg_hero/home/index.html.erb

@@ -36,13 +36,14 @@
       <% end %>
     </div>
   <% end %>
-  <div class="alert alert-<%= @good_total_connections ? "success" : "warning" %>">
-    <% if @good_total_connections %>
-      Number of connections healthy
+  <div class="alert alert-<%= @good_total_connections && @good_idle_connections ? "success" : "warning" %>">
+    <% if !@good_total_connections %>
+      High number of connections <span class="tiny"><%= @total_connections %></span>
+    <% elsif !@good_idle_connections %>
+      High number of connections idle in transaction <span class="tiny"><%= @idle_connections %></span>
     <% else %>
-      High number of connections
+      Connections healthy <span class="tiny"><%= @total_connections %></span>
     <% end %>
-    <span class="tiny"><%= @total_connections %></span>
   </div>
   <div class="alert alert-<%= @transaction_id_danger.empty? ? "success" : "warning" %>">
     <% if @transaction_id_danger.any? %>
@@ -217,10 +218,19 @@
   </div>
 <% end %>
 
+<% if !@good_idle_connections %>
+  <div class="content">
+    <h1>High Number of Connections Idle in Transaction</h1>
+    <p><%= pluralize(@idle_connections, "connection") %> idle in transaction</p>
+    <p>Avoid opening transactions and doing work outside the database.</p>
+    <p><%= link_to "View queries", live_queries_path(state: "idle in transaction") %></p>
+  </div>
+<% end %>
+
 <% if @transaction_id_danger.any? %>
   <div class="content">
     <h2>Vacuuming Needed</h2>
-    <p>The database <strong>will shutdown</strong> when there are fewer than 1,000,000 transactions left. <%= link_to "Read more", "http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND", target: "_blank" %>.</p>
+    <p>The database <strong>will shutdown</strong> when there are fewer than 1,000,000 transactions left. <%= link_to "Read more", "https://www.postgresql.org/docs/current/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND", target: "_blank" %>.</p>
 
     <p>Try <%= link_to "tuning autovacuum", "https://blog.2ndquadrant.com/autovacuum-tuning-basics/", target: "_blank" %> - specifically autovacuum_vacuum_cost_limit.</p>
 

data/lib/generators/pghero/templates/{config.yml → config.yml.tt}

@@ -1,7 +1,7 @@
 databases:
   main:
     # Database URL (defaults to app database)
-    # url: <%= ENV["DATABASE_URL"] %>
+    # url: <%%= ENV["DATABASE_URL"] %>
 
     # Minimum time for long running queries
     # long_running_query_sec: 60
@@ -13,14 +13,14 @@ databases:
     # slow_query_calls: 100
 
     # Minimum connections for high connections warning
-    # total_connections_threshold: 100
+    # total_connections_threshold: 500
 
     # Statement timeout for explain
     # explain_timeout_sec: 10
 
   # Add more databases
   # other:
-  #   url: <%= ENV["OTHER_DATABASE_URL"] %>
+  #   url: <%%= ENV["OTHER_DATABASE_URL"] %>
 
 # Time zone (defaults to app time zone)
 # time_zone: "Pacific Time (US & Canada)"

data/lib/pghero.rb

@@ -38,7 +38,7 @@ module PgHero
   self.slow_query_ms = (ENV["PGHERO_SLOW_QUERY_MS"] || 20).to_i
   self.slow_query_calls = (ENV["PGHERO_SLOW_QUERY_CALLS"] || 100).to_i
   self.explain_timeout_sec = (ENV["PGHERO_EXPLAIN_TIMEOUT_SEC"] || 10).to_i
-  self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 100).to_i
+  self.total_connections_threshold = (ENV["PGHERO_TOTAL_CONNECTIONS_THRESHOLD"] || 500).to_i
   self.cache_hit_rate_threshold = 99
   self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
   self.show_migrations = true
@@ -46,7 +46,7 @@ module PgHero
   class << self
     extend Forwardable
     def_delegators :primary_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autovacuum_danger,
-      :best_index, :blocked_queries, :connection_sources, :connection_stats,
+      :best_index, :blocked_queries, :connection_sources, :connection_states, :connection_stats,
       :cpu_usage, :create_user, :database_size, :db_instance_identifier, :disable_query_stats, :drop_user,
       :duplicate_indexes, :enable_query_stats, :explain, :historical_query_stats_enabled?, :index_caching,
       :index_hit_rate, :index_usage, :indexes, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,

data/lib/pghero/methods/connections.rb

@@ -5,6 +5,22 @@ module PgHero
         select_one("SELECT COUNT(*) FROM pg_stat_activity")
       end
 
+      def connection_states
+        states = select_all <<-SQL
+          SELECT
+            state,
+            COUNT(*) AS connections
+          FROM
+            pg_stat_activity
+          GROUP BY
+            1
+          ORDER BY
+            2 DESC, 1
+        SQL
+
+        Hash[states.map { |s| [s[:state], s[:connections]] }]
+      end
+
       def connection_sources
         select_all <<-SQL
           SELECT

data/lib/pghero/methods/indexes.rb

@@ -114,32 +114,19 @@ module PgHero
         SQL
       end
 
-      def invalid_indexes
-        select_all <<-SQL
-          SELECT
-            n.nspname AS schema,
-            c.relname AS index,
-            pg_get_indexdef(i.indexrelid) AS definition
-          FROM
-            pg_catalog.pg_class c,
-            pg_catalog.pg_namespace n,
-            pg_catalog.pg_index i
-          WHERE
-            i.indisvalid = false
-            AND i.indexrelid = c.oid
-            AND c.relnamespace = n.oid
-            AND n.nspname != 'pg_catalog'
-            AND n.nspname != 'information_schema'
-            AND n.nspname != 'pg_toast'
-          ORDER BY
-            c.relname
-        SQL
+      def invalid_indexes(indexes: nil)
+        indexes = (indexes || self.indexes).select { |i| !i[:valid] && !i[:creating] }
+        indexes.each do |index|
+          # map name -> index for backward compatibility
+          index[:index] = index[:name]
+        end
+        indexes
       end
 
       # TODO parse array properly
-      # http://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
+      # https://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
       def indexes
-        select_all(<<-SQL
+        indexes = select_all(<<-SQL
           SELECT
             schemaname AS schema,
             t.relname AS table,
@@ -166,15 +153,28 @@ module PgHero
             1, 2
         SQL
         ).map { |v| v[:columns] = v[:columns].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
+
+        # determine if any invalid indexes being created
+        # hacky, but works for simple cases
+        # can be a race condition, but that's fine
+        invalid_indexes = indexes.select { |i| !i[:valid] }
+        if invalid_indexes.any?
+          create_index_queries = running_queries.select { |q| /\s*CREATE\s+INDEX\s+CONCURRENTLY\s+/i.match(q[:query]) }
+          invalid_indexes.each do |index|
+            index[:creating] = create_index_queries.any? { |q| q[:query].include?(index[:table]) && index[:columns].all? { |c| q[:query].include?(c) } }
+          end
+        end
+
+        indexes
       end
 
       def duplicate_indexes(indexes: nil)
         dup_indexes = []
 
-        indexes_by_table = (indexes || self.indexes).group_by { |i| i[:table] }
-        indexes_by_table.values.flatten.select { |i| !i[:primary] && !i[:unique] && !i[:indexprs] && !i[:indpred] && i[:valid] }.each do |index|
-          covering_index = indexes_by_table[index[:table]].find { |i| index_covers?(i[:columns], index[:columns]) && i[:using] == index[:using] && i[:name] != index[:name] && i[:schema] == index[:schema] && !i[:indexprs] && !i[:indpred] && i[:valid] }
-          if covering_index && (covering_index[:columns] != index[:columns] || index[:name] > covering_index[:name])
+        indexes_by_table = (indexes || self.indexes).group_by { |i| [i[:schema], i[:table]] }
+        indexes_by_table.values.flatten.select { |i| i[:valid] && !i[:primary] && !i[:unique] }.each do |index|
+          covering_index = indexes_by_table[[index[:schema], index[:table]]].find { |i| i[:valid] && i[:name] != index[:name] && index_covers?(i[:columns], index[:columns]) && i[:using] == index[:using] && i[:indexprs] == index[:indexprs] && i[:indpred] == index[:indpred] }
+          if covering_index && (covering_index[:columns] != index[:columns] || index[:name] > covering_index[:name] || covering_index[:primary] || covering_index[:unique])
             dup_indexes << {unneeded_index: index, covering_index: covering_index}
           end
         end
@@ -321,6 +321,12 @@ module PgHero
             index_name
         SQL
       end
+
+      protected
+
+      def index_covers?(indexed_columns, columns)
+        indexed_columns.first(columns.size) == columns
+      end
     end
   end
 end

data/lib/pghero/methods/maintenance.rb

@@ -1,7 +1,7 @@
 module PgHero
   module Methods
     module Maintenance
-      # http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
+      # https://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
       # "the system will shut down and refuse to start any new transactions
       # once there are fewer than 1 million transactions left until wraparound"
       # warn when 10,000,000 transactions left

data/lib/pghero/methods/query_stats.rb

@@ -64,7 +64,7 @@ module PgHero
         false
       end
 
-      # http://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
+      # https://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
       def historical_query_stats_enabled?
         # TODO use schema from config
         # make sure primary database is PostgreSQL first
@@ -143,7 +143,7 @@ module PgHero
               total_time / 1000 / 60 AS total_minutes,
               (total_time / calls) AS average_time,
               calls,
-              (SELECT regexp_matches(query, '/\\*(.+)\\*/'))[1] AS origin
+              (SELECT regexp_matches(query, '.*/\\*(.+?)\\*/'))[1] AS origin
             FROM
               pghero_query_stats
             WHERE

data/lib/pghero/methods/replication.rb

@@ -8,7 +8,7 @@ module PgHero
         @replica
       end
 
-      # http://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
+      # https://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
       def replication_lag
         with_feature_support do
           lag_condition =

data/lib/pghero/methods/sequences.rb

@@ -36,8 +36,12 @@ module PgHero
 
         add_sequence_attributes(sequences)
 
-        select_all(sequences.select { |s| s[:readable] }.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")).each_with_index do |row, i|
-          sequences[i][:last_value] = row[:last_value]
+        sequences.select { |s| s[:readable] }.each_slice(1024) do |slice|
+          sql = slice.map { |s| "SELECT last_value FROM #{quote_ident(s[:schema])}.#{quote_ident(s[:sequence])}" }.join(" UNION ALL ")
+
+          select_all(sql).zip(slice) do |row, seq|
+            seq[:last_value] = row[:last_value]
+          end
         end
 
         sequences.sort_by { |s| s[:sequence] }

data/lib/pghero/methods/suggested_indexes.rb

@@ -217,12 +217,8 @@ module PgHero
         {table: table, where: where, sort: sort}
       end
 
-      def index_covers?(indexed_columns, columns)
-        indexed_columns.first(columns.size) == columns
-      end
-
       # TODO better row estimation
-      # http://www.postgresql.org/docs/current/static/row-estimation-examples.html
+      # https://www.postgresql.org/docs/current/static/row-estimation-examples.html
       def row_estimates(stats, total_rows, rows_left, op)
         case op
         when "null"

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "2.1.1"
+  VERSION = "2.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -109,22 +109,15 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: 
-email:
-- andrew@chartkick.com
+email: andrew@chartkick.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitattributes"
-- ".github/ISSUE_TEMPLATE.md"
-- ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTING.md
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
 - app/assets/images/pghero/favicon.png
 - app/assets/javascripts/pghero/Chart.bundle.js
 - app/assets/javascripts/pghero/application.js
@@ -156,20 +149,12 @@ files:
 - app/views/pg_hero/home/system.html.erb
 - app/views/pg_hero/home/tune.html.erb
 - config/routes.rb
-- guides/Contributing.md
-- guides/Docker.md
-- guides/Heroku.md
-- guides/Linux.md
-- guides/Permissions.md
-- guides/Query-Stats.md
-- guides/Rails.md
-- guides/Suggested-Indexes.md
 - lib/generators/pghero/config_generator.rb
 - lib/generators/pghero/query_stats_generator.rb
 - lib/generators/pghero/space_stats_generator.rb
-- lib/generators/pghero/templates/config.yml
-- lib/generators/pghero/templates/query_stats.rb
-- lib/generators/pghero/templates/space_stats.rb
+- lib/generators/pghero/templates/config.yml.tt
+- lib/generators/pghero/templates/query_stats.rb.tt
+- lib/generators/pghero/templates/space_stats.rb.tt
 - lib/pghero.rb
 - lib/pghero/connection.rb
 - lib/pghero/database.rb
@@ -193,13 +178,6 @@ files:
 - lib/pghero/query_stats.rb
 - lib/pghero/version.rb
 - lib/tasks/pghero.rake
-- pghero.gemspec
-- test/basic_test.rb
-- test/best_index_test.rb
-- test/gemfiles/activerecord41.gemfile
-- test/gemfiles/activerecord42.gemfile
-- test/suggested_indexes_test.rb
-- test/test_helper.rb
 homepage: https://github.com/ankane/pghero
 licenses:
 - MIT
@@ -212,7 +190,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -220,22 +198,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: A performance dashboard for Postgres
-test_files:
-- guides/Contributing.md
-- guides/Docker.md
-- guides/Heroku.md
-- guides/Linux.md
-- guides/Permissions.md
-- guides/Query-Stats.md
-- guides/Rails.md
-- guides/Suggested-Indexes.md
-- test/basic_test.rb
-- test/best_index_test.rb
-- test/gemfiles/activerecord41.gemfile
-- test/gemfiles/activerecord42.gemfile
-- test/suggested_indexes_test.rb
-- test/test_helper.rb
+test_files: []