pghero 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb091e2ecc7b76f41e0af306855d6df60f9ace12
-  data.tar.gz: c82ae33d920f6b7862b8aa9cc46e7abe0fffa5c2
+  metadata.gz: 5e9922be6c95f3dd56d5b41844d239316ce99f8e
+  data.tar.gz: 53c0c88d93f9794ecc698601659695e65bc05cb8
 SHA512:
-  metadata.gz: c24fd383eb9069a9445a2e826e65dbb5eaeb3e9a7ab03c202dc1acaf42ef866e52f6895a8f202043f16b82eb41cc8ad297773d8e57811e5b0e928052e3cf699e
-  data.tar.gz: 0d7193d888ae03896d64080dfa14dd811fc6e8b7aea621c913b34f082a4baa7da113339aa3dfcca9828276034cf0c8c163599a73d5bb23ae5ede1dccca21d028
+  metadata.gz: d00b0e9a93e26343ee7418279778a0e7144af7eae862f4a17edcf3be93af28f2ff76c3cb20c2570422e93f98a430394029ab1f47c2b6b861bac75aec121812e5
+  data.tar.gz: f0053ca3dc71a1de4bee814722033dae44ed4484fc07fb4f68aff4abeff929f90587668d5c91bd7fdd4c777e7929d4446d3510899b65f948ecc1c4f3f82f8795

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.8
+
+- Added `total_percent` to `query_stats`
+- Added `total_connections`
+- Added `connection_stats` for Amazon RDS
+
 ## 0.1.7
 
 - Added support for pg_stat_statments on Amazon RDS

data/README.md

@@ -43,6 +43,7 @@ PgHero.database_size
 PgHero.relation_sizes
 PgHero.index_hit_rate
 PgHero.table_hit_rate
+PgHero.total_connections
 ```
 
 Kill queries
@@ -124,6 +125,8 @@ Add the following to your `postgresql.conf`:
 ```conf
 shared_preload_libraries = 'pg_stat_statements'
 pg_stat_statements.track = all
+pg_stat_statements.max = 10000
+track_activity_query_size = 2048
 ```
 
 Then restart PostgreSQL. As a superuser from the `psql` console, run:
@@ -166,6 +169,10 @@ and reset stats with:
 SELECT pg_stat_statements_reset();
 ```
 
+#### Queries show up as `<insufficient privilege>`
+
+For security reasons, only superusers can see queries executed by other users.
+
 ## System Stats
 
 CPU usage is available for Amazon RDS.  Add these lines to your application’s Gemfile:
@@ -203,6 +210,10 @@ Minimum calls for slow queries
 PgHero.slow_query_calls = 100 # default
 ```
 
+## Bonus
+
+See where queries come from with [Marginalia](https://github.com/basecamp/marginalia) - comments appear on the Live Queries tab.
+
 ## TODO
 
 - show exactly which indexes to add

data/app/controllers/pg_hero/home_controller.rb

@@ -18,6 +18,8 @@ module PgHero
       @unused_indexes = PgHero.unused_indexes
       @good_cache_rate = @table_hit_rate >= 0.99 && @index_hit_rate >= 0.99
       @query_stats_available = PgHero.query_stats_available?
+      @total_connections = PgHero.total_connections
+      @good_total_connections = @total_connections < PgHero.total_connections_threshold
     end
 
     def indexes
@@ -44,6 +46,7 @@ module PgHero
     def system_stats
       @title = "System Stats"
       @cpu_usage = PgHero.cpu_usage
+      @connection_stats = PgHero.connection_stats
     end
 
     def explain

data/app/views/layouts/pg_hero/application.html.erb

@@ -88,6 +88,12 @@
         color: #999;
       }
 
+      .percent {
+        color: #999;
+        font-size: 12px;
+        margin-left: 6px;
+      }
+
       #status, .content, .alert {
         margin-bottom: 20px;
       }

data/app/views/pg_hero/home/_query_stats_table.html.erb

@@ -9,9 +9,21 @@
   <tbody>
     <% queries.each do |query| %>
       <tr>
-        <td><%= query["total_minutes"].to_f.round %> min</td>
+        <td>
+          <%= query["total_minutes"].to_f.round %> min
+          <span class="percent">
+            <% percent = query["total_percent"].to_f %>
+            <% if percent > 1 %>
+              <%= percent.round %>%
+            <% elsif percent > 0.1 %>
+              <%= percent.round(1) %>%
+            <% else %>
+              &lt; 0.1%
+            <% end %>
+          </span>
+        </td>
         <td><%= query["average_time"].to_f.round %> ms</td>
-        <td><% calls = query["calls"].to_i %><%= calls >= 1000 ? number_with_delimiter(calls.round) + "k" : calls %></td>
+        <td><%= number_with_delimiter(query["calls"].to_i) %></td>
       </tr>
       <tr>
         <td colspan="3" style="border-top: none; padding: 0;"><pre><%= query["query"] %></pre></td>

data/app/views/pg_hero/home/index.html.erb

@@ -13,6 +13,13 @@
       Low cache hit rate
     <% end %>
   </div>
+  <div class="alert alert-<%= @good_total_connections ? "success" : "warning" %>">
+    <% if @good_total_connections %>
+      Healthy number of connections
+    <% else %>
+      High number of connections
+    <% end %>
+  </div>
   <div class="alert alert-<%= @query_stats_enabled && @slow_queries.empty? ? "success" : "warning" %>">
     <% if !@query_stats_enabled %>
       Query stats must be enabled for slow queries
@@ -63,6 +70,33 @@
   </div>
 <% end %>
 
+<% if !@good_total_connections %>
+  <div class="content">
+    <h1>High Number of Connections</h1>
+    <p><%= pluralize(@total_connections, "connection") %></p>
+    <p>
+      <%= link_to "Use connection pooling", "http://www.craigkerstiens.com/2014/05/22/on-connection-pooling/", target: "_blank" %> for better performance. <%= link_to "PgBouncer", "https://wiki.postgresql.org/wiki/PgBouncer", target: "_blank" %> is a solid option.
+    </p>
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th>Top Sources</th>
+          <th>Connections</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% PgHero.connection_sources.first(10).each do |source| %>
+          <tr>
+            <td><%= source["source"] %></td>
+            <td style="width: 20%;"><%= number_with_delimiter(source["total_connections"]) %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% end %>
+
 <% if !@query_stats_enabled %>
   <div class="content">
     <h1>Query Stats</h1>

data/app/views/pg_hero/home/system_stats.html.erb

@@ -1,6 +1,9 @@
+<%= javascript_include_tag "//www.google.com/jsapi", "chartkick" %>
+
 <div class="content">
   <h1>CPU Usage</h1>
-
-  <%= javascript_include_tag "//www.google.com/jsapi", "chartkick" %>
   <div style="margin-bottom: 20px;"><%= line_chart @cpu_usage, max: 101 %></div>
+
+  <h1>Connections</h1>
+  <div style="margin-bottom: 20px;"><%= line_chart @connection_stats %></div>
 </div>

data/lib/pghero.rb

@@ -9,16 +9,17 @@ module PgHero
   end
 
   class << self
-    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls
+    attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold
   end
   self.long_running_query_sec = 60
   self.slow_query_ms = 20
   self.slow_query_calls = 100
+  self.total_connections_threshold = 100
 
   class << self
 
     def running_queries
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           pid,
           state,
@@ -35,11 +36,11 @@ module PgHero
           AND pid <> pg_backend_pid()
         ORDER BY
           query_start DESC
-      }
+      SQL
     end
 
     def long_running_queries
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           pid,
           state,
@@ -57,11 +58,11 @@ module PgHero
           AND now() - query_start > interval '#{long_running_query_sec.to_i} seconds'
         ORDER BY
           query_start DESC
-      }
+      SQL
     end
 
     def locks
-      select_all %Q{
+      select_all <<-SQL
         SELECT DISTINCT ON (pid)
           pg_stat_activity.pid,
           pg_stat_activity.query,
@@ -77,29 +78,31 @@ module PgHero
         ORDER BY
           pid,
           query_start
-      }
+      SQL
     end
 
     def index_hit_rate
-      select_all(%Q{
+      select_all(<<-SQL
         SELECT
-          (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read),0) AS rate
+          (sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) AS rate
         FROM
           pg_statio_user_indexes
-      }).first["rate"].to_f
+      SQL
+      ).first["rate"].to_f
     end
 
     def table_hit_rate
-      select_all(%Q{
+      select_all(<<-SQL
         SELECT
-          sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read),0) AS rate
+          sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS rate
         FROM
           pg_statio_user_tables
-      }).first["rate"].to_f
+      SQL
+      ).first["rate"].to_f
     end
 
     def index_usage
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           relname AS table,
           CASE idx_scan
@@ -112,11 +115,11 @@ module PgHero
         ORDER BY
           n_live_tup DESC,
           relname ASC
-       }
+       SQL
     end
 
     def missing_indexes
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           relname AS table,
           CASE idx_scan
@@ -133,11 +136,11 @@ module PgHero
         ORDER BY
           n_live_tup DESC,
           relname ASC
-       }
+       SQL
     end
 
     def unused_tables
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           relname AS table,
           n_live_tup rows_in_table
@@ -148,11 +151,11 @@ module PgHero
         ORDER BY
           n_live_tup DESC,
           relname ASC
-       }
+       SQL
     end
 
     def unused_indexes
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           relname AS table,
           indexrelname AS index,
@@ -169,11 +172,11 @@ module PgHero
         ORDER BY
           pg_relation_size(i.indexrelid) DESC,
           relname ASC
-      }
+      SQL
     end
 
     def relation_sizes
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           c.relname AS name,
           CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
@@ -189,19 +192,40 @@ module PgHero
         ORDER BY
           pg_table_size(c.oid) DESC,
           name ASC
-      }
+      SQL
     end
 
     def database_size
       select_all("SELECT pg_size_pretty(pg_database_size(current_database()))").first["pg_size_pretty"]
     end
 
+    def total_connections
+      select_all("SELECT COUNT(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid()").first["count"].to_i
+    end
+
+    def connection_sources
+      select_all <<-SQL
+        SELECT
+          application_name AS source,
+          COUNT(*) AS total_connections
+        FROM
+          pg_stat_activity
+        WHERE
+          pid <> pg_backend_pid()
+        GROUP BY
+          application_name
+        ORDER BY
+          COUNT(*) DESC,
+          application_name ASC
+      SQL
+    end
+
     def kill(pid)
       execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"] == "t"
     end
 
     def kill_all
-      select_all %Q{
+      select_all <<-SQL
         SELECT
           pg_terminate_backend(pid)
         FROM
@@ -209,29 +233,39 @@ module PgHero
         WHERE
           pid <> pg_backend_pid()
           AND query <> '<insufficient privilege>'
-      }
+      SQL
       true
     end
 
     # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
     def query_stats
       if query_stats_enabled?
-        select_all %Q{
+        select_all <<-SQL
+          WITH query_stats AS (
+            SELECT
+              query,
+              (total_time / 1000 / 60) as total_minutes,
+              (total_time / calls) as average_time,
+              calls
+            FROM
+              pg_stat_statements
+            INNER JOIN
+              pg_database ON pg_database.oid = pg_stat_statements.dbid
+            WHERE
+              pg_database.datname = current_database()
+          )
           SELECT
             query,
-            (total_time / 1000 / 60) as total_minutes,
-            (total_time / calls) as average_time,
-            calls
+            total_minutes,
+            average_time,
+            calls,
+            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent
           FROM
-            pg_stat_statements
-          INNER JOIN
-            pg_database ON pg_database.oid = pg_stat_statements.dbid
-          WHERE
-            pg_database.datname = current_database()
+            query_stats
           ORDER BY
             total_minutes DESC
           LIMIT 100
-        }
+        SQL
       else
         []
       end
@@ -239,24 +273,35 @@ module PgHero
 
     def slow_queries
       if query_stats_enabled?
-        select_all %Q{
+        select_all <<-SQL
+          WITH query_stats AS (
+            SELECT
+              query,
+              (total_time / 1000 / 60) as total_minutes,
+              (total_time / calls) as average_time,
+              calls
+            FROM
+              pg_stat_statements
+            INNER JOIN
+              pg_database ON pg_database.oid = pg_stat_statements.dbid
+            WHERE
+              pg_database.datname = current_database()
+          )
           SELECT
             query,
-            (total_time / 1000 / 60) as total_minutes,
-            (total_time / calls) as average_time,
-            calls
+            total_minutes,
+            average_time,
+            calls,
+            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent
           FROM
-            pg_stat_statements
-          INNER JOIN
-            pg_database ON pg_database.oid = pg_stat_statements.dbid
+            query_stats
           WHERE
-            pg_database.datname = current_database()
-            AND calls >= #{slow_query_calls.to_i}
-            AND (total_time / calls) >= #{slow_query_ms.to_i}
+            calls >= #{slow_query_calls.to_i}
+            AND average_time >= #{slow_query_ms.to_i}
           ORDER BY
             total_minutes DESC
           LIMIT 100
-        }
+        SQL
       else
         []
       end
@@ -298,12 +343,20 @@ module PgHero
     end
 
     def cpu_usage
+      rds_stats("CPUUtilization")
+    end
+
+    def connection_stats
+      rds_stats("DatabaseConnections")
+    end
+
+    def rds_stats(metric_name)
       if system_stats_enabled?
         cw = AWS::CloudWatch.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
         now = Time.now
         resp = cw.client.get_metric_statistics(
           namespace: "AWS/RDS",
-          metric_name: "CPUUtilization",
+          metric_name: metric_name,
           dimensions: [{name: "DBInstanceIdentifier", value: db_instance_identifier}],
           start_time: (now - 1 * 3600).iso8601,
           end_time: now.iso8601,

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-13 00:00:00.000000000 Z
+date: 2015-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -134,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Database insights made easy