pghero 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29a21b5bab969343ade74d9df1643ac808cf81c5
-  data.tar.gz: cbfa3d06c92ec558cb28733fcfdf3a74ad56d295
+  metadata.gz: 1ed028006b69e4e3595170baffc6fb328de3cd3f
+  data.tar.gz: 05f3f70425c1506f3ba7e6106e1be4922b256fc6
 SHA512:
-  metadata.gz: 9b42030927945b2e8a7e6fc802ab74e65a4afaf101e3ba21e3d8f35346bbae1184a4fba4e07d774d591351ca86925a3089d92966ea68f5263a6694dcbbcea7f5
-  data.tar.gz: b44897c9ee84a0622c059f74254f0f15380ce6dba24bbe007a20cf037e884ace4255546c4b3c978edc769e6e96a134aaaa0425950c0963fae7903f632dd0c8f2
+  metadata.gz: a7d7323653811a623f0a7843fd5aaec408570f96909a7ccec71f2aaae343619f67eb8ff81878f1f86cc035f9a1cc2c20539470025136ac004765aa0d67c6e75b
+  data.tar.gz: 951cf7cc4359047ec384905e6dcbd0e8f6de9c205a32a09c399a38191f9edcf10492b07b05f8af6a80948d1366c34ca9edc7c7f26b20812fc39f7f8b90f43e9a

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.1.0
+
+- Added historical query stats
+
 ## 1.0.1
 
 - Fixed connection bad errors

data/app/controllers/pg_hero/home_controller.rb

@@ -11,7 +11,7 @@ module PgHero
 
     def index
       @title = "Overview"
-      @slow_queries = PgHero.slow_queries
+      @slow_queries = PgHero.slow_queries(historical: true, start_at: 3.hours.ago)
       @long_running_queries = PgHero.long_running_queries
       @index_hit_rate = PgHero.index_hit_rate
       @table_hit_rate = PgHero.table_hit_rate
@@ -46,7 +46,28 @@ module PgHero
 
     def queries
       @title = "Queries"
-      @query_stats = PgHero.query_stats
+      @historical_query_stats_enabled = PgHero.historical_query_stats_enabled?
+
+      @query_stats =
+        begin
+          if @historical_query_stats_enabled
+            @start_at = params[:start_at] ? Time.zone.parse(params[:start_at]) : 24.hours.ago
+            @end_at = Time.zone.parse(params[:end_at]) if params[:end_at]
+          end
+
+          if @historical_query_stats_enabled && !request.xhr?
+            []
+          else
+            PgHero.query_stats(historical: true, start_at: @start_at, end_at: @end_at)
+          end
+        rescue
+          @error = true
+          []
+        end
+
+      if request.xhr?
+        render layout: false, partial: "queries_table", locals: {queries: @query_stats, xhr: true}
+      end
     end
 
     def system

data/app/views/layouts/pg_hero/application.html.erb

@@ -120,6 +120,27 @@
         margin-bottom: 0;
       }
 
+      #slider-container {
+        padding: 6px 140px 20px 140px;
+      }
+
+      #slider {
+        margin-bottom: 20px;
+      }
+
+      #range-start {
+        min-height: 20px;
+      }
+
+      #range-end {
+        float: right;
+      }
+
+      .queries-info {
+        text-align: center;
+        margin-top: 40px;
+      }
+
       /* nav */
 
       .nav a {
@@ -137,6 +158,10 @@
         background-color: #ddd;
       }
 
+      .nav li.active-database a {
+        color: #999;
+      }
+
       .nav-header {
         font-weight: bold;
         color: #333;
@@ -391,11 +416,11 @@
           </ul>
 
           <% if @databases.size > 1 %>
-            <p class="nav-header">Other Databases</p>
+            <p class="nav-header">Databases</p>
             <ul class="nav">
-              <% (@databases - [PgHero.current_database]).each do |database| %>
-                <li>
-                  <%= link_to database.titleize, root_path(database: database) %>
+              <% @databases.each do |database| %>
+                <li class="<%= ("active-database" if PgHero.current_database == database) %>">
+                  <%= link_to database.titleize, database: database %>
                 </li>
               <% end %>
             </ul>

data/app/views/pg_hero/home/_queries_table.html.erb

@@ -1,12 +1,27 @@
 <table class="table">
-  <thead>
-    <tr>
-      <th style="width: 33.33%;">Total Time</th>
-      <th style="width: 33.33%;">Average Time</th>
-      <th style="width: 33.33%;">Calls</th>
-    </tr>
-  </thead>
-  <tbody>
+  <% unless local_assigns[:xhr] %>
+    <thead>
+      <tr>
+        <th style="width: 33.33%;">Total Time</th>
+        <th style="width: 33.33%;">Average Time</th>
+        <th style="width: 33.33%;">Calls</th>
+      </tr>
+    </thead>
+  <% end %>
+  <tbody id="queries">
+    <% if queries.empty? %>
+      <tr>
+        <td colspan="3">
+          <p class="queries-info text-muted">
+            <% if local_assigns[:xhr] %>
+              No data available for this time.
+            <% else %>
+              ...
+            <% end %>
+          </p>
+        </td>
+      </tr>
+    <% end %>
     <% queries.each do |query| %>
       <tr>
         <td>

data/app/views/pg_hero/home/_query_stats_slider.html.erb

@@ -0,0 +1,129 @@
+<div id="slider-container">
+  <div id="slider"></div>
+  <div id="range-end"></div>
+  <div id="range-start"></div>
+</div>
+
+<%= stylesheet_link_tag "https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/6.2.0/jquery.nouislider.min.css" %>
+<%= javascript_include_tag "//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js", "https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/6.2.0/jquery.nouislider.min.js" %>
+
+<style>
+  .noUi-connect {
+    box-shadow: none;
+    background: #5bc0de;
+  }
+
+  .noUi-handle {
+    box-shadow: none;
+  }
+
+  .noUi-target {
+    box-shadow: none;
+    border-color: #eee;
+  }
+
+  .noUi-background {
+    box-shadow: none;
+    background-color: #eee;
+  }
+
+  .noUi-origin {
+    border-radius: 0;
+  }
+</style>
+
+<script>
+  function roundTime(time) {
+    var period = 1000 * 60 * 5;
+    return new Date(Math.ceil(time.getTime() / period) * period);
+  }
+
+  function pad(num) {
+    return (num < 10) ? "0" + num : num;
+  }
+
+  var months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+
+  var days = 1;
+  var now = new Date();
+  var sliderStartAt = roundTime(now) - days * 24 * 60 * 60 * 1000;
+  var sliderMax = 24 * 12 * days;
+
+  var startAt = <%= @start_at.to_i %> * 1000;
+  var min = (startAt > 0) ? (startAt - sliderStartAt) / (1000 * 60 * 5) : 0;
+
+  var endAt = <%= @end_at.to_i %> * 1000;
+  var max = (endAt > 0) ? (endAt - sliderStartAt) / (1000 * 60 * 5) : sliderMax;
+
+  var $slider = $("#slider");
+
+  $slider.noUiSlider({
+    range: {
+      min: 0,
+      max: sliderMax
+    },
+    step: 1,
+    connect: true,
+    start: [min, max]
+  });
+
+  function updateText() {
+    var values = $slider.val();
+    setText("#range-start", values[0]);
+    setText("#range-end", values[1]);
+  }
+
+  function setText(selector, offset) {
+    var time = timeAt(offset);
+
+    var html = "";
+    if (time == now) {
+      if (selector == "#range-end") {
+        html = "Now";
+      }
+    } else {
+      html = months[time.getMonth()] + " " + time.getDate() + ", " + pad(time.getHours()) + ":" + pad(time.getMinutes());
+    }
+    $(selector).html(html);
+  }
+
+  function timeAt(offset) {
+    var time = new Date(sliderStartAt + (offset * 5) * 60 * 1000);
+    return (time > now) ? now : time;
+  }
+
+  function timeParam(time) {
+    return time.toISOString();
+  }
+
+  function refreshStats(push) {
+    var values = $slider.val();
+    var startAt = timeAt(values[0]);
+    var endAt = timeAt(values[1]);
+
+    var params = {}
+    if (startAt > sliderStartAt) {
+      params.start_at = timeParam(startAt);
+    }
+    if (endAt < now) {
+      params.end_at = timeParam(endAt);
+    }
+
+    var path = "queries";
+    if (params.start_at || params.end_at) {
+      path += "?" + $.param(params);
+    }
+
+    $("#queries").html('<tr><td colspan="3"><p class="queries-info text-muted">...</p></td></tr>').load(path);
+
+    if (push && history.pushState) {
+      history.pushState(null, null, path);
+    }
+  }
+
+  $slider.on("slide", updateText).on("change", function () {
+    refreshStats(true);
+  });
+  updateText();
+  $(refreshStats);
+</script>

data/app/views/pg_hero/home/queries.html.erb

@@ -3,10 +3,16 @@
     <%= button_to "Reset", reset_query_stats_path, class: "btn btn-danger", style: "float: right;" %>
   <% end %>
 
-  <h1>Queries</h1>
+  <h1 style="float: left;">Queries</h1>
+
+  <% if @historical_query_stats_enabled %>
+    <%= render partial: "query_stats_slider" %>
+  <% end %>
 
   <% if @query_stats_enabled %>
-    <% if @query_stats.any? %>
+    <% if @error %>
+      <div class="alert alert-danger">Cannot understand start or end time.</div>
+    <% elsif @query_stats.any? || @historical_query_stats_enabled %>
       <%= render partial: "queries_table", locals: {queries: @query_stats} %>
     <% else %>
       <p>Stats are not available yet. Come back soon!</p>

data/guides/Rails.md

@@ -118,6 +118,35 @@ end
 
 Query stats can be enabled from the dashboard. If you run into issues, [view the guide](Query-Stats.md).
 
+## Historical Query Stats
+
+To track query stats over time, run:
+
+```sh
+rails generate pghero:query_stats
+rake db:migrate
+```
+
+And schedule the task below to run every 5 minutes.
+
+```sh
+rake pghero:capture_query_stats
+```
+
+Or with a scheduler like Clockwork, use:
+
+```ruby
+PgHero.capture_query_stats
+```
+
+After this, a time range slider will appear on the Queries tab.
+
+By default, historical query stats are stored in your primary database. Change this with:
+
+```ruby
+ENV["PGHERO_STATS_DATABASE_URL"]
+```
+
 ## System Stats
 
 CPU usage is available for Amazon RDS.  Add these lines to your application’s Gemfile:
@@ -154,6 +183,12 @@ production:
   <<: *default
 ```
 
+Specify a database with:
+
+```ruby
+PgHero.with(:replica) { PgHero.running_queries }
+```
+
 ## Customize
 
 Minimum time for long running queries

data/lib/generators/pghero/query_stats_generator.rb

@@ -0,0 +1,29 @@
+# taken from https://github.com/collectiveidea/audited/blob/master/lib/generators/audited/install_generator.rb
+require "rails/generators"
+require "rails/generators/migration"
+require "active_record"
+require "rails/generators/active_record"
+
+module Pghero
+  module Generators
+    class QueryStatsGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+
+      source_root File.expand_path("../templates", __FILE__)
+
+      # Implement the required interface for Rails::Generators::Migration.
+      def self.next_migration_number(dirname) #:nodoc:
+        next_migration_number = current_migration_number(dirname) + 1
+        if ::ActiveRecord::Base.timestamped_migrations
+          [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
+        else
+          "%.3d" % next_migration_number
+        end
+      end
+
+      def copy_migration
+        migration_template "query_stats.rb", "db/migrate/create_pghero_query_stats.rb"
+      end
+    end
+  end
+end

data/lib/generators/pghero/templates/query_stats.rb

@@ -0,0 +1,13 @@
+class <%= migration_class_name %> < ActiveRecord::Migration
+  def change
+    create_table :pghero_query_stats do |t|
+      t.text :database
+      t.text :query
+      t.float :total_time
+      t.integer :calls, limit: 8
+      t.timestamp :captured_at
+    end
+
+    add_index :pghero_query_stats, [:database, :captured_at]
+  end
+end

data/lib/pghero.rb

@@ -1,6 +1,7 @@
 require "pghero/version"
 require "active_record"
 require "pghero/engine" if defined?(Rails)
+require "pghero/tasks"
 
 module PgHero
   # hack for connection
@@ -8,6 +9,12 @@ module PgHero
     self.abstract_class = true
   end
 
+  class QueryStats < ActiveRecord::Base
+    self.abstract_class = true
+    self.table_name = "pghero_query_stats"
+    establish_connection ENV["PGHERO_STATS_DATABASE_URL"] if ENV["PGHERO_STATS_DATABASE_URL"]
+  end
+
   class << self
     attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold, :env
   end
@@ -33,7 +40,7 @@ module PgHero
           {
             "databases" => {
               "primary" => {
-                "url" => ENV["PGHERO_DATABASE_URL"], # nil reverts to default config
+                "url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config,
                 "db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"]
               }
             }
@@ -301,74 +308,28 @@ module PgHero
       true
     end
 
-    # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
-    def query_stats
-      if query_stats_enabled?
-        select_all <<-SQL
-          WITH query_stats AS (
-            SELECT
-              query,
-              (total_time / 1000 / 60) as total_minutes,
-              (total_time / calls) as average_time,
-              calls
-            FROM
-              pg_stat_statements
-            INNER JOIN
-              pg_database ON pg_database.oid = pg_stat_statements.dbid
-            WHERE
-              pg_database.datname = current_database()
-          )
-          SELECT
-            query,
-            total_minutes,
-            average_time,
-            calls,
-            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent
-          FROM
-            query_stats
-          ORDER BY
-            total_minutes DESC
-          LIMIT 100
-        SQL
-      else
-        []
+    def query_stats(options = {})
+      current_query_stats = (options[:historical] && options[:end_at] && options[:end_at] < Time.now ? [] : current_query_stats(options)).index_by { |q| q["query"] }
+      historical_query_stats = (options[:historical] ? historical_query_stats(options) : []).index_by { |q| q["query"] }
+      current_query_stats.default = {}
+      historical_query_stats.default = {}
+
+      query_stats = []
+      (current_query_stats.keys + historical_query_stats.keys).uniq.each do |query|
+        value = {
+          "query" => query,
+          "total_minutes" => current_query_stats[query]["total_minutes"].to_f + historical_query_stats[query]["total_minutes"].to_f,
+          "calls" => current_query_stats[query]["calls"].to_i + historical_query_stats[query]["calls"].to_i
+        }
+        value["average_time"] = value["total_minutes"] * 1000 * 60 / value["calls"]
+        value["total_percent"] = value["total_minutes"] * 100.0 / (current_query_stats[query]["all_queries_total_minutes"].to_f + historical_query_stats[query]["all_queries_total_minutes"].to_f)
+        query_stats << value
       end
+      query_stats.sort_by { |q| -q["total_minutes"] }.first(100)
     end
 
-    def slow_queries
-      if query_stats_enabled?
-        select_all <<-SQL
-          WITH query_stats AS (
-            SELECT
-              query,
-              (total_time / 1000 / 60) as total_minutes,
-              (total_time / calls) as average_time,
-              calls
-            FROM
-              pg_stat_statements
-            INNER JOIN
-              pg_database ON pg_database.oid = pg_stat_statements.dbid
-            WHERE
-              pg_database.datname = current_database()
-          )
-          SELECT
-            query,
-            total_minutes,
-            average_time,
-            calls,
-            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent
-          FROM
-            query_stats
-          WHERE
-            calls >= #{slow_query_calls.to_i}
-            AND average_time >= #{slow_query_ms.to_i}
-          ORDER BY
-            total_minutes DESC
-          LIMIT 100
-        SQL
-      else
-        []
-      end
+    def slow_queries(options = {})
+      query_stats(options).select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
     end
 
     def query_stats_available?
@@ -403,6 +364,53 @@ module PgHero
       end
     end
 
+    def capture_query_stats
+      config["databases"].keys.each do |database|
+        with(database) do
+          now = Time.now
+          query_stats = self.query_stats(limit: 1000000)
+          if query_stats.any? && reset_query_stats
+            values =
+              query_stats.map do |qs|
+                [
+                  database,
+                  qs["query"],
+                  qs["total_minutes"].to_f * 60 * 1000,
+                  qs["calls"],
+                  now
+                ].map { |v| quote(v) }.join(",")
+              end.map { |v| "(#{v})" }.join(",")
+
+            stats_connection.execute("INSERT INTO pghero_query_stats (database, query, total_time, calls, captured_at) VALUES #{values}")
+          end
+        end
+      end
+    end
+
+    # http://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
+    def historical_query_stats_enabled?
+      # TODO use schema from config
+      stats_connection.select_all( squish <<-SQL
+        SELECT EXISTS (
+          SELECT
+            1
+          FROM
+            pg_catalog.pg_class c
+          INNER JOIN
+            pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+          WHERE
+            n.nspname = 'public'
+            AND c.relname = 'pghero_query_stats'
+            AND c.relkind = 'r'
+        )
+      SQL
+      ).to_a.first["exists"] == "t"
+    end
+
+    def stats_connection
+      QueryStats.connection
+    end
+
     def ssl_used?
       ssl_used = nil
       Connection.transaction do
@@ -464,7 +472,7 @@ module PgHero
 
       commands =
         [
-          "CREATE ROLE #{user} LOGIN PASSWORD #{connection.quote(password)}",
+          "CREATE ROLE #{user} LOGIN PASSWORD #{quote(password)}",
           "GRANT CONNECT ON DATABASE #{database} TO #{user}",
           "GRANT USAGE ON SCHEMA #{schema} TO #{user}"
         ]
@@ -571,6 +579,80 @@ module PgHero
       select_all("SELECT EXTRACT(EPOCH FROM NOW() - pg_last_xact_replay_timestamp()) AS replication_lag").first["replication_lag"].to_f
     end
 
+    private
+
+    # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
+    def current_query_stats(options = {})
+      if query_stats_enabled?
+        limit = options[:limit] || 100
+        select_all <<-SQL
+          WITH query_stats AS (
+            SELECT
+              query,
+              (total_time / 1000 / 60) as total_minutes,
+              (total_time / calls) as average_time,
+              calls
+            FROM
+              pg_stat_statements
+            INNER JOIN
+              pg_database ON pg_database.oid = pg_stat_statements.dbid
+            WHERE
+              pg_database.datname = current_database()
+          )
+          SELECT
+            query,
+            total_minutes,
+            average_time,
+            calls,
+            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
+            (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
+          FROM
+            query_stats
+          ORDER BY
+            total_minutes DESC
+          LIMIT #{limit.to_i}
+        SQL
+      else
+        []
+      end
+    end
+
+    def historical_query_stats(options = {})
+      if historical_query_stats_enabled?
+        stats_connection.select_all squish <<-SQL
+          WITH query_stats AS (
+            SELECT
+              query,
+              (SUM(total_time) / 1000 / 60) as total_minutes,
+              (SUM(total_time) / SUM(calls)) as average_time,
+              SUM(calls) as calls
+            FROM
+              pghero_query_stats
+            WHERE
+              database = #{quote(current_database)}
+              #{options[:start_at] ? "AND captured_at >= #{quote(options[:start_at])}" : ""}
+              #{options[:end_at] ? "AND captured_at <= #{quote(options[:end_at])}" : ""}
+            GROUP BY
+              query
+          )
+          SELECT
+            query,
+            total_minutes,
+            average_time,
+            calls,
+            total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
+            (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
+          FROM
+            query_stats
+          ORDER BY
+            total_minutes DESC
+          LIMIT 100
+        SQL
+      else
+        []
+      end
+    end
+
     def friendly_value(setting, unit)
       if %w(kB 8kB).include?(unit)
         value = setting.to_i
@@ -609,5 +691,9 @@ module PgHero
     def squish(str)
       str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
     end
+
+    def quote(value)
+      connection.quote(value)
+    end
   end
 end

data/lib/pghero/tasks.rb

@@ -0,0 +1,8 @@
+require "rake"
+
+namespace :pghero do
+  desc "capture query stats"
+  task capture_query_stats: :environment do
+    PgHero.capture_query_stats
+  end
+end

data/lib/pghero/version.rb

@@ -1,3 +1,3 @@
 module PgHero
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pghero
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-17 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -98,6 +98,7 @@ files:
 - app/views/pg_hero/home/_connections_table.html.erb
 - app/views/pg_hero/home/_live_queries_table.html.erb
 - app/views/pg_hero/home/_queries_table.html.erb
+- app/views/pg_hero/home/_query_stats_slider.html.erb
 - app/views/pg_hero/home/connections.html.erb
 - app/views/pg_hero/home/explain.html.erb
 - app/views/pg_hero/home/index.html.erb
@@ -116,8 +117,11 @@ files:
 - guides/Linux.md
 - guides/Query-Stats.md
 - guides/Rails.md
+- lib/generators/pghero/query_stats_generator.rb
+- lib/generators/pghero/templates/query_stats.rb
 - lib/pghero.rb
 - lib/pghero/engine.rb
+- lib/pghero/tasks.rb
 - lib/pghero/version.rb
 - pghero.gemspec
 - test/pghero_test.rb