pghero 1.5.3 → 2.0.0

data/app/views/pg_hero/home/show_query.html.erb

@@ -0,0 +1,107 @@
+<div class="content">
+  <pre><code style="max-height: 230px; overflow: hidden;" onclick="this.style.maxHeight = 'none';"><%= @query %></code></pre>
+  <script>
+    highlightQueries()
+  </script>
+
+  <% if @explainable_query %>
+    <p>
+      <% button_path, button_options = Rails.version >= "4.1" ? [explain_path, {params: {query: @explainable_query}}] : [explain_path(query: @explainable_query), {}] %>
+      <%= button_to "Explain", button_path, button_options.merge(form: {target: "_blank"}, class: "btn btn-info") %>
+    </p>
+  <% end %>
+
+  <% if @origins && @origins.keys.select { |k| k.length > 0 }.any? %>
+    <table style="table-layout: auto;">
+      <thead>
+        <tr>
+          <th colspan="2">
+            <div style="float: right;">Approx. Time</div>
+            Origin
+          </th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @origins.sort_by { |o, c| [-c, o.to_s] }.each do |origin, count| %>
+          <tr>
+            <td class="origin" style="width: 90%;">
+              <% if origin.length > 0 %>
+                <%= origin %>
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+            <td style="text-align: right; width: 10%;">
+              <% pct = (100.0 * count / @total_count).round %>
+              <% if pct == 0 %>
+                &lt; 1%
+              <% else %>
+                <%= pct %>%
+              <% end %>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+
+  <!-- chart -->
+  <% if @chart_data %>
+    <h1>Total Time <small>ms</small></h1>
+    <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+
+    <h1>Average Time <small>ms</small></h1>
+    <div id="chart-2" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-2", <%= json_escape(@chart2_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+
+    <h1>Calls</h1>
+    <div id="chart-3" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-3", <%= json_escape(@chart3_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+  <% else %>
+    <p>
+      Enable
+      <%= link_to "historical query stats", "https://github.com/ankane/pghero", target: "_blank" %>
+      to see more details
+    </p>
+  <% end %>
+
+  <!-- table info -->
+  <% if @tables.any? %>
+    <h1>Tables</h1>
+    <table>
+      <thead>
+        <tr>
+          <th style="width: 25%;">Name</th>
+          <th style="width: 25%;">Rows</th>
+          <th>Indexes</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @tables.each do |table| %>
+          <tr>
+            <td><%= table %></td>
+            <td><%= @row_counts[table] %></td>
+            <td>
+              <ul class="list-normal">
+                <% @indexes_by_table[table].to_a.sort_by { |i| [i[:primary] ? 0 : 1, i[:columns]] }.each do |i3| %>
+                  <li>
+                    <%= i3[:columns].join(", ") %><% if i3[:using] != "btree" %>
+                    <%= i3[:using].to_s.upcase %><% end %>
+                    <% if i3[:primary] %> PRIMARY<% elsif i3[:unique] %> UNIQUE<% end %>
+                  </li>
+                <% end %>
+              </ul>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+</div>

data/app/views/pg_hero/home/space.html.erb

@@ -3,25 +3,79 @@
 
   <p>Database Size: <%= @database_size %></p>
 
-  <table class="table">
+  <% if @system_stats_enabled %>
+    <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-1", <%= json_escape(free_space_stats_path.to_json).html_safe %>, {colors: ["#5bc0de"]})
+    </script>
+  <% end %>
+
+  <!--
+  <% if @index_bloat.any? %>
+    <p>Check out <%= link_to "index bloat", index_bloat_path %> for an easy way to reclaim space.</p>
+  <% end %>
+  -->
+
+  <% if @unused_indexes.any? %>
+    <p>
+      <%= pluralize(@unused_indexes.size, "unused index") %>. Remove them
+      <% if @show_migrations %>
+        <a href="javascript: void(0);" onclick="document.getElementById('migration').style.display = 'block';">with a migration</a>
+      <% end %>
+      for faster writes.
+
+      <% if @database.replicating? %>
+        Check they aren’t used on replicas.
+      <% end %>
+    </p>
+
+    <div id="migration" style="display: none;">
+      <pre>rails g migration remove_unused_indexes</pre>
+      <p>And paste</p>
+      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.sort_by { |q| q[:index] }.each do |query| %>
+remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+    </div>
+  <% end %>
+
+  <table class="table space-table">
     <thead>
       <tr>
-        <th>Relation</th>
-        <th style="width: 15%;"></th>
-        <th style="width: 15%;">Size</th>
+        <th><%= link_to "Relation", {sort: "name"} %></th>
+        <th style="width: 15%;"><%= link_to "Size", {} %></th>
+        <% if @space_stats_enabled %>
+          <th style="width: 15%;"><%= link_to "#{@days}d Growth", {sort: "growth"} %></th>
+        <% end %>
       </tr>
     </thead>
     <tbody>
       <% @relation_sizes.each do |query| %>
         <tr>
-          <td>
-            <%= query["name"] %>
-            <% if query["schema"] != "public" %>
-              <span class="text-muted"><%= query["schema"] %></span>
+          <td style="<%= query[:type] == "index" ? "font-style: italic;" : "" %>">
+            <span style="word-break: break-all;">
+              <% name = query[:relation] || query[:table] %>
+              <% if @space_stats_enabled %>
+                <%= link_to name, relation_space_path(name), target: "_blank", style: "color: inherit;" %>
+              <% else %>
+                <%= name %>
+              <% end %>
+            </span>
+            <% if query[:schema] != "public" %>
+              <span class="text-muted"><%= query[:schema] %></span>
+            <% end %>
+            <% if @unused_index_names.include?(query[:name]) %>
+              <span class="unused-index">UNUSED</span>
             <% end %>
           </td>
-          <td> <span class="text-muted"><%= query["type"] == "index" ? "index" : "" %></span></td>
-          <td><%= query["size"] %></td>
+          <td><%= query[:size] %></td>
+          <% if @space_stats_enabled %>
+            <td>
+              <% if @growth_bytes_by_relation[query[:relation]] %>
+                <% if @growth_bytes_by_relation[query[:relation]] < 0 %>-<% end %><%= PgHero.pretty_size(@growth_bytes_by_relation[query[:relation]].abs) %>
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+          <% end %>
         </tr>
       <% end %>
     </tbody>

data/app/views/pg_hero/home/system.html.erb

@@ -7,16 +7,28 @@
   <% path_options = {duration: params[:duration], period: params[:period]} %>
 
   <h1>CPU</h1>
-  <div style="margin-bottom: 20px;"><%= line_chart cpu_usage_path(path_options), max: 100, colors: ["#5bc0de"] %></div>
+  <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+  <script>
+    new Chartkick.LineChart("chart-1", <%= json_escape(cpu_usage_path(path_options).to_json).html_safe %>, {max: 100, colors: ["#5bc0de"]})
+  </script>
 
   <h1>Load</h1>
-  <div style="margin-bottom: 20px;"><%= line_chart load_stats_path(path_options), colors: ["#5bc0de", "#d9534f"] %></div>
+  <div id="chart-2" class="chart" style="margin-bottom: 20px;">Loading...</div>
+  <script>
+    new Chartkick.LineChart("chart-2", <%= json_escape(load_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de", "#d9534f"]})
+  </script>
 
   <h1>Connections</h1>
-  <div style="margin-bottom: 20px;"><%= line_chart connection_stats_path(path_options), colors: ["#5bc0de"] %></div>
+  <div id="chart-3" class="chart" style="margin-bottom: 20px;">Loading...</div>
+  <script>
+    new Chartkick.LineChart("chart-3", <%= json_escape(connection_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"]})
+  </script>
 
   <% if @database.replica? %>
     <h1>Replication Lag</h1>
-    <div style="margin-bottom: 20px;"><%= line_chart replication_lag_stats_path(path_options), colors: ["#5bc0de"] %></div>
+    <div id="chart-4" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-4", <%= json_escape(replication_lag_stats_path(path_options).to_json).html_safe %>, {colors: ["#5bc0de"]})
+    </script>
   <% end %>
 </div>

data/config/routes.rb

@@ -1,14 +1,17 @@
 PgHero::Engine.routes.draw do
   scope "(:database)", constraints: proc { |req| (PgHero.config["databases"].keys + [nil]).include?(req.params[:database]) } do
-    get "index_usage", to: "home#index_usage"
     get "space", to: "home#space"
+    get "space/:relation", to: "home#relation_space", as: :relation_space
+    get "index_bloat", to: "home#index_bloat"
     get "live_queries", to: "home#live_queries"
     get "queries", to: "home#queries"
+    get "queries/:query_hash", to: "home#show_query", as: :show_query
     get "system", to: "home#system"
     get "cpu_usage", to: "home#cpu_usage"
     get "connection_stats", to: "home#connection_stats"
     get "replication_lag_stats", to: "home#replication_lag_stats"
     get "load_stats", to: "home#load_stats"
+    get "free_space_stats", to: "home#free_space_stats"
     get "explain", to: "home#explain"
     get "tune", to: "home#tune"
     get "connections", to: "home#connections"
@@ -23,7 +26,6 @@ PgHero::Engine.routes.draw do
     # legacy routes
     get "system_stats" => redirect("system")
     get "query_stats" => redirect("queries")
-    get "indexes" => redirect("index_usage")
 
     root to: "home#index"
   end

data/guides/Linux.md

@@ -186,6 +186,26 @@ sudo pghero config:set PGHERO_SECRET_ACCESS_KEY=secret123
 sudo pghero config:set PGHERO_DB_INSTANCE_IDENTIFIER=epona
 ```
 
+## Multiple Databases
+
+Create a `pghero.yml` with:
+
+```yml
+production:
+  databases:
+    primary:
+      url: postgres://...
+    replica:
+      url: postgres://...
+```
+
+And run:
+
+```sh
+cat pghero.yml | sudo pghero run sh -c "cat > config/pghero.yml"
+sudo service pghero restart
+```
+
 ## Customize
 
 Minimum time for long running queries

data/guides/Rails.md

@@ -19,7 +19,7 @@ Be sure to [secure the dashboard](#security) in production.
 PgHero can suggest indexes to add. To enable, add to your Gemfile:
 
 ```ruby
-gem 'pg_query'
+gem 'pg_query', '>= 0.9.0'
 ```
 
 and make sure [query stats](#query-stats) are enabled. Read about how it works [here](Suggested-Indexes.md).
@@ -38,7 +38,7 @@ ENV["PGHERO_PASSWORD"] = "hyrule"
 #### Devise
 
 ```ruby
-authenticate :user, lambda { |user| user.admin? } do
+authenticate :user, -> (user) { user.admin? } do
   mount PgHero::Engine, at: "pghero"
 end
 ```
@@ -78,11 +78,10 @@ ENV["PGHERO_STATS_DATABASE_URL"]
 
 ## System Stats
 
-CPU usage is available for Amazon RDS.  Add these lines to your application’s Gemfile:
+CPU usage, IOPS, and other stats are available for Amazon RDS. Add these lines to your application’s Gemfile:
 
 ```ruby
 gem 'aws-sdk'
-gem 'chartkick'
 ```
 
 And add these variables to your environment:
@@ -242,6 +241,33 @@ PgHero.drop_user("ganondorf")
 
 ## Upgrading
 
+### 2.0.0
+
+New features
+
+- Query details page
+
+Breaking changes
+
+- Methods now return symbols for keys instead of strings
+- Methods raise `PgHero::NotEnabled` error when a feature isn’t enabled
+- Requires pg_query 0.9.0+ for suggested indexes
+- Historical query stats require the `pghero_query_stats` table to have `query_hash` and `user` columns
+- Removed `with` option - use:
+
+```ruby
+PgHero.databases[:database2].running_queries
+```
+
+instead of
+
+```ruby
+PgHero.with(:database2) { PgHero.running_queries }
+```
+
+- Removed options from `connection_sources` method
+- Removed `locks` method
+
 ### 1.5.0
 
 For query stats grouping by user, create a migration with:

data/guides/Suggested-Indexes.md

@@ -1,6 +1,6 @@
 # How PgHero Suggests Indexes
 
-1. Get the most time-consuming queries from [pg_stat_statements](http://www.postgresql.org/docs/9.3/static/pgstatstatements.html).
+1. Get the most time-consuming queries from [pg_stat_statements](http://www.postgresql.org/docs/current/static/pgstatstatements.html).
 
 2. Parse queries with [pg_query](https://github.com/lfittl/pg_query).  Look for a single table with a `WHERE` clause that consists of only `=`, `IN`, `IS NULL` or `IS NOT NULL` and/or an `ORDER BY` clause.
 

data/lib/generators/pghero/config_generator.rb

@@ -0,0 +1,13 @@
+require "rails/generators"
+
+module Pghero
+  module Generators
+    class ConfigGenerator < Rails::Generators::Base
+      source_root File.expand_path("../templates", __FILE__)
+
+      def create_initializer
+        template "config.yml", "config/pghero.yml"
+      end
+    end
+  end
+end

data/lib/generators/pghero/query_stats_generator.rb

@@ -22,7 +22,13 @@ module Pghero
       end
 
       def copy_migration
-        migration_template "query_stats.rb", "db/migrate/create_pghero_query_stats.rb"
+        migration_template "query_stats.rb", "db/migrate/create_pghero_query_stats.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        if ActiveRecord::VERSION::MAJOR >= 5
+          "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+        end
       end
     end
   end

data/lib/generators/pghero/space_stats_generator.rb

@@ -22,7 +22,13 @@ module Pghero
       end
 
       def copy_migration
-        migration_template "space_stats.rb", "db/migrate/create_pghero_space_stats.rb"
+        migration_template "space_stats.rb", "db/migrate/create_pghero_space_stats.rb", migration_version: migration_version
+      end
+
+      def migration_version
+        if ActiveRecord::VERSION::MAJOR >= 5
+          "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+        end
       end
     end
   end

data/lib/generators/pghero/templates/config.yml

@@ -0,0 +1,23 @@
+databases:
+  main:
+    # Database URL (defaults to app database)
+    # url: <%= ENV["DATABASE_URL"] %>
+
+    # Minimum time for long running queries
+    # long_running_query_sec: 60
+
+    # Minimum average time for slow queries
+    # slow_query_ms: 20
+
+    # Minimum calls for slow queries
+    # slow_query_calls: 100
+
+    # Minimum connections for high connections warning
+    # total_connections_threshold: 100
+
+  # Add more databases
+  # other:
+  #   url: <%= ENV["OTHER_DATABASE_URL"] %>
+
+# Time zone (defaults to app time zone)
+# time_zone: "Pacific Time (US & Canada)"

data/lib/generators/pghero/templates/query_stats.rb

@@ -1,4 +1,4 @@
-class <%= migration_class_name %> < ActiveRecord::Migration
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :pghero_query_stats do |t|
       t.text :database

data/lib/generators/pghero/templates/space_stats.rb

@@ -1,4 +1,4 @@
-class <%= migration_class_name %> < ActiveRecord::Migration
+class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :pghero_space_stats do |t|
       t.text :database

data/lib/pghero/database.rb

@@ -8,7 +8,7 @@ module PgHero
     include Methods::Maintenance
     include Methods::Queries
     include Methods::QueryStats
-    include Methods::Replica
+    include Methods::Replication
     include Methods::Sequences
     include Methods::Space
     include Methods::SuggestedIndexes
@@ -20,7 +20,7 @@ module PgHero
 
     def initialize(id, config)
       @id = id
-      @config = config
+      @config = config || {}
     end
 
     def name
@@ -55,6 +55,10 @@ module PgHero
       (config["long_running_query_sec"] || PgHero.config["long_running_query_sec"] || PgHero.long_running_query_sec).to_i
     end
 
+    def index_bloat_bytes
+      (config["index_bloat_bytes"] || PgHero.config["index_bloat_bytes"] || 100.megabytes).to_i
+    end
+
     private
 
     def connection_model

data/lib/pghero/engine.rb

@@ -3,13 +3,16 @@ module PgHero
     isolate_namespace PgHero
 
     initializer "pghero", group: :all do |app|
-      if defined?(Sprockets) && Sprockets::VERSION >= "4"
-        app.config.assets.precompile << "pghero/application.js"
-        app.config.assets.precompile << "pghero/application.css"
-      else
-        # use a proc instead of a string
-        app.config.assets.precompile << proc { |path| path == "pghero/application.js" }
-        app.config.assets.precompile << proc { |path| path == "pghero/application.css" }
+      # check if Rails api mode
+      if app.config.respond_to?(:assets)
+        if defined?(Sprockets) && Sprockets::VERSION >= "4"
+          app.config.assets.precompile << "pghero/application.js"
+          app.config.assets.precompile << "pghero/application.css"
+        else
+          # use a proc instead of a string
+          app.config.assets.precompile << proc { |path| path == "pghero/application.js" }
+          app.config.assets.precompile << proc { |path| path == "pghero/application.css" }
+        end
       end
 
       PgHero.time_zone = PgHero.config["time_zone"] if PgHero.config["time_zone"]

data/lib/pghero/methods/basic.rb

@@ -4,44 +4,78 @@ module PgHero
       def settings
         names =
           if server_version_num >= 90500
-            %w(
+            %i(
               max_connections shared_buffers effective_cache_size work_mem
               maintenance_work_mem min_wal_size max_wal_size checkpoint_completion_target
               wal_buffers default_statistics_target
             )
           else
-            %w(
+            %i(
               max_connections shared_buffers effective_cache_size work_mem
               maintenance_work_mem checkpoint_segments checkpoint_completion_target
               wal_buffers default_statistics_target
             )
           end
-        Hash[names.map { |name| [name, select_all("SHOW #{name}").first[name]] }]
+        Hash[names.map { |name| [name, select_one("SHOW #{name}")] }]
       end
 
       def ssl_used?
         ssl_used = nil
-        connection_model.transaction do
-          execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
-          ssl_used = PgHero.truthy?(select_all("SELECT ssl_is_used()").first["ssl_is_used"])
-          raise ActiveRecord::Rollback
+        with_transaction(rollback: true) do
+          begin
+            execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
+          rescue ActiveRecord::StatementInvalid
+            # not superuser
+          end
+          ssl_used = select_one("SELECT ssl_is_used()")
         end
         ssl_used
       end
 
       def database_name
-        select_all("SELECT current_database()").first["current_database"]
+        select_one("SELECT current_database()")
       end
 
       def server_version
-        select_all("SHOW server_version").first["server_version"]
+        @server_version ||= select_one("SHOW server_version")
+      end
+
+      def server_version_num
+        @server_version_num ||= select_one("SHOW server_version_num").to_i
+      end
+
+      def quote_ident(value)
+        quote_table_name(value)
       end
 
       private
 
-      def select_all(sql)
+      def select_all(sql, conn = nil)
+        conn ||= connection
         # squish for logs
-        connection.select_all(squish(sql)).to_a
+        result = conn.select_all(squish(sql))
+        cast_method = ActiveRecord::VERSION::MAJOR < 5 ? :type_cast : :cast_value
+        result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(cast_method, val)] }] }
+      end
+
+      def select_all_stats(sql)
+        select_all(sql, stats_connection)
+      end
+
+      def select_all_size(sql)
+        result = select_all(sql)
+        result.each do |row|
+          row[:size] = PgHero.pretty_size(row[:size_bytes])
+        end
+        result
+      end
+
+      def select_one(sql, conn = nil)
+        select_all(sql, conn).first.values.first
+      end
+
+      def select_one_stats(sql)
+        select_one(sql, stats_connection)
       end
 
       def execute(sql)
@@ -52,6 +86,16 @@ module PgHero
         connection_model.connection
       end
 
+      def stats_connection
+        ::PgHero::QueryStats.connection
+      end
+
+      def insert_stats(table, columns, values)
+        values = values.map { |v| "(#{v.map { |v2| quote(v2) }.join(",")})" }.join(",")
+        columns = columns.map { |v| quote_table_name(v) }.join(",")
+        stats_connection.execute("INSERT INTO #{quote_table_name(table)} (#{columns}) VALUES #{values}")
+      end
+
       # from ActiveSupport
       def squish(str)
         str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
@@ -72,6 +116,34 @@ module PgHero
           part
         end
       end
+
+      def with_transaction(lock_timeout: nil, statement_timeout: nil, rollback: false)
+        connection_model.transaction do
+          select_all "SET LOCAL statement_timeout = #{statement_timeout.to_i}" if statement_timeout
+          select_all "SET LOCAL lock_timeout = #{lock_timeout.to_i}" if lock_timeout
+          yield
+          raise ActiveRecord::Rollback if rollback
+        end
+      end
+
+      def table_exists?(table)
+        ["PostgreSQL", "PostGIS"].include?(stats_connection.adapter_name) &&
+        select_one_stats(<<-SQL
+          SELECT EXISTS (
+            SELECT
+              1
+            FROM
+              pg_catalog.pg_class c
+            INNER JOIN
+              pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+            WHERE
+              n.nspname = 'public'
+              AND c.relname = #{quote(table)}
+              AND c.relkind = 'r'
+          )
+        SQL
+        )
+      end
     end
   end
 end