pghero 1.7.0 → 2.0.0

data/app/views/pg_hero/home/relation_space.html.erb

@@ -0,0 +1,9 @@
+<div class="content">
+  <h1><%= @relation %></h1>
+
+  <h1>Size <small>MB</small></h1>
+  <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+  <script>
+    new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false, min: null})
+  </script>
+</div>

data/app/views/pg_hero/home/show_query.html.erb

@@ -0,0 +1,107 @@
+<div class="content">
+  <pre><code style="max-height: 230px; overflow: hidden;" onclick="this.style.maxHeight = 'none';"><%= @query %></code></pre>
+  <script>
+    highlightQueries()
+  </script>
+
+  <% if @explainable_query %>
+    <p>
+      <% button_path, button_options = Rails.version >= "4.1" ? [explain_path, {params: {query: @explainable_query}}] : [explain_path(query: @explainable_query), {}] %>
+      <%= button_to "Explain", button_path, button_options.merge(form: {target: "_blank"}, class: "btn btn-info") %>
+    </p>
+  <% end %>
+
+  <% if @origins && @origins.keys.select { |k| k.length > 0 }.any? %>
+    <table style="table-layout: auto;">
+      <thead>
+        <tr>
+          <th colspan="2">
+            <div style="float: right;">Approx. Time</div>
+            Origin
+          </th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @origins.sort_by { |o, c| [-c, o.to_s] }.each do |origin, count| %>
+          <tr>
+            <td class="origin" style="width: 90%;">
+              <% if origin.length > 0 %>
+                <%= origin %>
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+            <td style="text-align: right; width: 10%;">
+              <% pct = (100.0 * count / @total_count).round %>
+              <% if pct == 0 %>
+                &lt; 1%
+              <% else %>
+                <%= pct %>%
+              <% end %>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+
+  <!-- chart -->
+  <% if @chart_data %>
+    <h1>Total Time <small>ms</small></h1>
+    <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-1", <%= json_escape(@chart_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+
+    <h1>Average Time <small>ms</small></h1>
+    <div id="chart-2" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-2", <%= json_escape(@chart2_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+
+    <h1>Calls</h1>
+    <div id="chart-3" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-3", <%= json_escape(@chart3_data.to_json).html_safe %>, {colors: ["#5bc0de"], legend: false})
+    </script>
+  <% else %>
+    <p>
+      Enable
+      <%= link_to "historical query stats", "https://github.com/ankane/pghero", target: "_blank" %>
+      to see more details
+    </p>
+  <% end %>
+
+  <!-- table info -->
+  <% if @tables.any? %>
+    <h1>Tables</h1>
+    <table>
+      <thead>
+        <tr>
+          <th style="width: 25%;">Name</th>
+          <th style="width: 25%;">Rows</th>
+          <th>Indexes</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @tables.each do |table| %>
+          <tr>
+            <td><%= table %></td>
+            <td><%= @row_counts[table] %></td>
+            <td>
+              <ul class="list-normal">
+                <% @indexes_by_table[table].to_a.sort_by { |i| [i[:primary] ? 0 : 1, i[:columns]] }.each do |i3| %>
+                  <li>
+                    <%= i3[:columns].join(", ") %><% if i3[:using] != "btree" %>
+                    <%= i3[:using].to_s.upcase %><% end %>
+                    <% if i3[:primary] %> PRIMARY<% elsif i3[:unique] %> UNIQUE<% end %>
+                  </li>
+                <% end %>
+              </ul>
+            </td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  <% end %>
+</div>

data/app/views/pg_hero/home/space.html.erb

@@ -3,25 +3,79 @@
 
   <p>Database Size: <%= @database_size %></p>
 
-  <table class="table">
+  <% if @system_stats_enabled %>
+    <div id="chart-1" class="chart" style="margin-bottom: 20px;">Loading...</div>
+    <script>
+      new Chartkick.LineChart("chart-1", <%= json_escape(free_space_stats_path.to_json).html_safe %>, {colors: ["#5bc0de"]})
+    </script>
+  <% end %>
+
+  <!--
+  <% if @index_bloat.any? %>
+    <p>Check out <%= link_to "index bloat", index_bloat_path %> for an easy way to reclaim space.</p>
+  <% end %>
+  -->
+
+  <% if @unused_indexes.any? %>
+    <p>
+      <%= pluralize(@unused_indexes.size, "unused index") %>. Remove them
+      <% if @show_migrations %>
+        <a href="javascript: void(0);" onclick="document.getElementById('migration').style.display = 'block';">with a migration</a>
+      <% end %>
+      for faster writes.
+
+      <% if @database.replicating? %>
+        Check they aren’t used on replicas.
+      <% end %>
+    </p>
+
+    <div id="migration" style="display: none;">
+      <pre>rails g migration remove_unused_indexes</pre>
+      <p>And paste</p>
+      <pre style="overflow: scroll; white-space: pre; word-break: normal;"><% @unused_indexes.sort_by { |q| q[:index] }.each do |query| %>
+remove_index <%= query[:table].to_sym.inspect %>, name: <%= query[:index].to_s.inspect %><% end %></pre>
+    </div>
+  <% end %>
+
+  <table class="table space-table">
     <thead>
       <tr>
-        <th>Relation</th>
-        <th style="width: 15%;"></th>
-        <th style="width: 15%;">Size</th>
+        <th><%= link_to "Relation", {sort: "name"} %></th>
+        <th style="width: 15%;"><%= link_to "Size", {} %></th>
+        <% if @space_stats_enabled %>
+          <th style="width: 15%;"><%= link_to "#{@days}d Growth", {sort: "growth"} %></th>
+        <% end %>
       </tr>
     </thead>
     <tbody>
       <% @relation_sizes.each do |query| %>
         <tr>
-          <td>
-            <%= query["name"] %>
-            <% if query["schema"] != "public" %>
-              <span class="text-muted"><%= query["schema"] %></span>
+          <td style="<%= query[:type] == "index" ? "font-style: italic;" : "" %>">
+            <span style="word-break: break-all;">
+              <% name = query[:relation] || query[:table] %>
+              <% if @space_stats_enabled %>
+                <%= link_to name, relation_space_path(name), target: "_blank", style: "color: inherit;" %>
+              <% else %>
+                <%= name %>
+              <% end %>
+            </span>
+            <% if query[:schema] != "public" %>
+              <span class="text-muted"><%= query[:schema] %></span>
+            <% end %>
+            <% if @unused_index_names.include?(query[:name]) %>
+              <span class="unused-index">UNUSED</span>
             <% end %>
           </td>
-          <td> <span class="text-muted"><%= query["type"] == "index" ? "index" : "" %></span></td>
-          <td><%= query["size"] %></td>
+          <td><%= query[:size] %></td>
+          <% if @space_stats_enabled %>
+            <td>
+              <% if @growth_bytes_by_relation[query[:relation]] %>
+                <% if @growth_bytes_by_relation[query[:relation]] < 0 %>-<% end %><%= PgHero.pretty_size(@growth_bytes_by_relation[query[:relation]].abs) %>
+              <% else %>
+                <span class="text-muted">Unknown</span>
+              <% end %>
+            </td>
+          <% end %>
         </tr>
       <% end %>
     </tbody>

data/config/routes.rb

@@ -1,14 +1,17 @@
 PgHero::Engine.routes.draw do
   scope "(:database)", constraints: proc { |req| (PgHero.config["databases"].keys + [nil]).include?(req.params[:database]) } do
-    get "index_usage", to: "home#index_usage"
     get "space", to: "home#space"
+    get "space/:relation", to: "home#relation_space", as: :relation_space
+    get "index_bloat", to: "home#index_bloat"
     get "live_queries", to: "home#live_queries"
     get "queries", to: "home#queries"
+    get "queries/:query_hash", to: "home#show_query", as: :show_query
     get "system", to: "home#system"
     get "cpu_usage", to: "home#cpu_usage"
     get "connection_stats", to: "home#connection_stats"
     get "replication_lag_stats", to: "home#replication_lag_stats"
     get "load_stats", to: "home#load_stats"
+    get "free_space_stats", to: "home#free_space_stats"
     get "explain", to: "home#explain"
     get "tune", to: "home#tune"
     get "connections", to: "home#connections"
@@ -23,7 +26,6 @@ PgHero::Engine.routes.draw do
     # legacy routes
     get "system_stats" => redirect("system")
     get "query_stats" => redirect("queries")
-    get "indexes" => redirect("index_usage")
 
     root to: "home#index"
   end

data/guides/Rails.md

@@ -19,7 +19,7 @@ Be sure to [secure the dashboard](#security) in production.
 PgHero can suggest indexes to add. To enable, add to your Gemfile:
 
 ```ruby
-gem 'pg_query'
+gem 'pg_query', '>= 0.9.0'
 ```
 
 and make sure [query stats](#query-stats) are enabled. Read about how it works [here](Suggested-Indexes.md).
@@ -241,6 +241,33 @@ PgHero.drop_user("ganondorf")
 
 ## Upgrading
 
+### 2.0.0
+
+New features
+
+- Query details page
+
+Breaking changes
+
+- Methods now return symbols for keys instead of strings
+- Methods raise `PgHero::NotEnabled` error when a feature isn’t enabled
+- Requires pg_query 0.9.0+ for suggested indexes
+- Historical query stats require the `pghero_query_stats` table to have `query_hash` and `user` columns
+- Removed `with` option - use:
+
+```ruby
+PgHero.databases[:database2].running_queries
+```
+
+instead of
+
+```ruby
+PgHero.with(:database2) { PgHero.running_queries }
+```
+
+- Removed options from `connection_sources` method
+- Removed `locks` method
+
 ### 1.5.0
 
 For query stats grouping by user, create a migration with:

data/guides/Suggested-Indexes.md

@@ -1,6 +1,6 @@
 # How PgHero Suggests Indexes
 
-1. Get the most time-consuming queries from [pg_stat_statements](http://www.postgresql.org/docs/9.3/static/pgstatstatements.html).
+1. Get the most time-consuming queries from [pg_stat_statements](http://www.postgresql.org/docs/current/static/pgstatstatements.html).
 
 2. Parse queries with [pg_query](https://github.com/lfittl/pg_query).  Look for a single table with a `WHERE` clause that consists of only `=`, `IN`, `IS NULL` or `IS NOT NULL` and/or an `ORDER BY` clause.
 

data/lib/pghero.rb

@@ -10,7 +10,7 @@ require "pghero/methods/kill"
 require "pghero/methods/maintenance"
 require "pghero/methods/queries"
 require "pghero/methods/query_stats"
-require "pghero/methods/replica"
+require "pghero/methods/replication"
 require "pghero/methods/sequences"
 require "pghero/methods/space"
 require "pghero/methods/suggested_indexes"
@@ -26,6 +26,8 @@ require "pghero/connection"
 require "pghero/query_stats"
 
 module PgHero
+  class NotEnabled < StandardError; end
+
   # settings
   class << self
     attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations
@@ -40,16 +42,16 @@ module PgHero
 
   class << self
     extend Forwardable
-    def_delegators :current_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autoindex_all, :autovacuum_danger,
+    def_delegators :primary_database, :access_key_id, :analyze, :analyze_tables, :autoindex, :autovacuum_danger,
       :best_index, :blocked_queries, :connection_sources, :connection_stats,
       :cpu_usage, :create_user, :database_size, :db_instance_identifier, :disable_query_stats, :drop_user,
       :duplicate_indexes, :enable_query_stats, :explain, :historical_query_stats_enabled?, :index_caching,
       :index_hit_rate, :index_usage, :indexes, :invalid_indexes, :kill, :kill_all, :kill_long_running_queries,
-      :locks, :long_running_queries, :maintenance_info, :missing_indexes, :query_stats,
+      :last_stats_reset_time, :long_running_queries, :maintenance_info, :missing_indexes, :query_stats,
       :query_stats_available?, :query_stats_enabled?, :query_stats_extension_enabled?, :query_stats_readable?,
       :rds_stats, :read_iops_stats, :region, :relation_sizes, :replica?, :replication_lag, :replication_lag_stats,
-      :reset_query_stats, :running_queries, :secret_access_key, :sequence_danger, :sequences, :settings,
-      :slow_queries, :ssl_used?, :stats_connection, :suggested_indexes, :suggested_indexes_by_query,
+      :reset_query_stats, :reset_stats, :running_queries, :secret_access_key, :sequence_danger, :sequences, :settings,
+      :slow_queries, :space_growth, :ssl_used?, :stats_connection, :suggested_indexes, :suggested_indexes_by_query,
       :suggested_indexes_enabled?, :system_stats_enabled?, :table_caching, :table_hit_rate, :table_stats,
       :total_connections, :transaction_id_danger, :unused_indexes, :unused_tables, :write_iops_stats
 
@@ -62,16 +64,20 @@ module PgHero
     end
 
     def config
-      Thread.current[:pghero_config] ||= begin
+      @config ||= begin
         path = "config/pghero.yml"
 
-        config = YAML.load(ERB.new(File.read(path)).result) if File.exist?(path)
+        config_file_exists = File.exist?(path)
+
+        config = YAML.load(ERB.new(File.read(path)).result) if config_file_exists
         config ||= {}
 
         if config[env]
           config[env]
         elsif config["databases"] # preferred format
           config
+        elsif config_file_exists
+          raise "Invalid config file"
         else
           {
             "databases" => {
@@ -89,7 +95,7 @@ module PgHero
       @databases ||= begin
         Hash[
           config["databases"].map do |id, c|
-            [id, PgHero::Database.new(id, c)]
+            [id.to_sym, PgHero::Database.new(id, c)]
           end
         ]
       end
@@ -99,26 +105,6 @@ module PgHero
       databases.values.first
     end
 
-    def current_database
-      Thread.current[:pghero_current_database] ||= primary_database
-    end
-
-    def current_database=(database)
-      raise "Database not found" unless databases[database.to_s]
-      Thread.current[:pghero_current_database] = databases[database.to_s]
-      database
-    end
-
-    def with(database)
-      previous_database = current_database
-      begin
-        self.current_database = database
-        yield
-      ensure
-        self.current_database = previous_database.id
-      end
-    end
-
     def capture_query_stats
       databases.each do |_, database|
         database.capture_query_stats
@@ -133,20 +119,23 @@ module PgHero
       true
     end
 
-    def analyze_all
-      databases.each do |_, database|
-        database.analyze_tables
+    def analyze_all(**options)
+      databases.reject { |_, d| d.replica? }.each do |_, database|
+        database.analyze_tables(**options)
       end
       true
     end
 
-    # Handles Rails 4 ('t') and Rails 5 (true) values.
-    def truthy?(value)
-      value == true || value == 't'
+    def autoindex_all(create: false)
+      databases.each do |_, database|
+        puts "Autoindexing #{database}..."
+        database.autoindex(create: create)
+      end
+      true
     end
 
-    def falsey?(value)
-      value == false || value == 'f'
+    def pretty_size(value)
+      ActiveSupport::NumberHelper.number_to_human_size(value, precision: 3)
     end
   end
 end

data/lib/pghero/database.rb

@@ -8,7 +8,7 @@ module PgHero
     include Methods::Maintenance
     include Methods::Queries
     include Methods::QueryStats
-    include Methods::Replica
+    include Methods::Replication
     include Methods::Sequences
     include Methods::Space
     include Methods::SuggestedIndexes
@@ -55,6 +55,10 @@ module PgHero
       (config["long_running_query_sec"] || PgHero.config["long_running_query_sec"] || PgHero.long_running_query_sec).to_i
     end
 
+    def index_bloat_bytes
+      (config["index_bloat_bytes"] || PgHero.config["index_bloat_bytes"] || 100.megabytes).to_i
+    end
+
     private
 
     def connection_model

data/lib/pghero/methods/basic.rb

@@ -4,44 +4,78 @@ module PgHero
       def settings
         names =
           if server_version_num >= 90500
-            %w(
+            %i(
               max_connections shared_buffers effective_cache_size work_mem
               maintenance_work_mem min_wal_size max_wal_size checkpoint_completion_target
               wal_buffers default_statistics_target
             )
           else
-            %w(
+            %i(
               max_connections shared_buffers effective_cache_size work_mem
               maintenance_work_mem checkpoint_segments checkpoint_completion_target
               wal_buffers default_statistics_target
             )
           end
-        Hash[names.map { |name| [name, select_all("SHOW #{name}").first[name]] }]
+        Hash[names.map { |name| [name, select_one("SHOW #{name}")] }]
       end
 
       def ssl_used?
         ssl_used = nil
-        connection_model.transaction do
-          execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
-          ssl_used = PgHero.truthy?(select_all("SELECT ssl_is_used()").first["ssl_is_used"])
-          raise ActiveRecord::Rollback
+        with_transaction(rollback: true) do
+          begin
+            execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
+          rescue ActiveRecord::StatementInvalid
+            # not superuser
+          end
+          ssl_used = select_one("SELECT ssl_is_used()")
         end
         ssl_used
       end
 
       def database_name
-        select_all("SELECT current_database()").first["current_database"]
+        select_one("SELECT current_database()")
       end
 
       def server_version
-        select_all("SHOW server_version").first["server_version"]
+        @server_version ||= select_one("SHOW server_version")
+      end
+
+      def server_version_num
+        @server_version_num ||= select_one("SHOW server_version_num").to_i
+      end
+
+      def quote_ident(value)
+        quote_table_name(value)
       end
 
       private
 
-      def select_all(sql)
+      def select_all(sql, conn = nil)
+        conn ||= connection
         # squish for logs
-        connection.select_all(squish(sql)).to_a
+        result = conn.select_all(squish(sql))
+        cast_method = ActiveRecord::VERSION::MAJOR < 5 ? :type_cast : :cast_value
+        result.map { |row| Hash[row.map { |col, val| [col.to_sym, result.column_types[col].send(cast_method, val)] }] }
+      end
+
+      def select_all_stats(sql)
+        select_all(sql, stats_connection)
+      end
+
+      def select_all_size(sql)
+        result = select_all(sql)
+        result.each do |row|
+          row[:size] = PgHero.pretty_size(row[:size_bytes])
+        end
+        result
+      end
+
+      def select_one(sql, conn = nil)
+        select_all(sql, conn).first.values.first
+      end
+
+      def select_one_stats(sql)
+        select_one(sql, stats_connection)
       end
 
       def execute(sql)
@@ -52,6 +86,16 @@ module PgHero
         connection_model.connection
       end
 
+      def stats_connection
+        ::PgHero::QueryStats.connection
+      end
+
+      def insert_stats(table, columns, values)
+        values = values.map { |v| "(#{v.map { |v2| quote(v2) }.join(",")})" }.join(",")
+        columns = columns.map { |v| quote_table_name(v) }.join(",")
+        stats_connection.execute("INSERT INTO #{quote_table_name(table)} (#{columns}) VALUES #{values}")
+      end
+
       # from ActiveSupport
       def squish(str)
         str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
@@ -73,12 +117,33 @@ module PgHero
         end
       end
 
-      def with_lock_timeout(timeout)
+      def with_transaction(lock_timeout: nil, statement_timeout: nil, rollback: false)
         connection_model.transaction do
-          select_all "SET LOCAL lock_timeout = #{timeout.to_i}"
+          select_all "SET LOCAL statement_timeout = #{statement_timeout.to_i}" if statement_timeout
+          select_all "SET LOCAL lock_timeout = #{lock_timeout.to_i}" if lock_timeout
           yield
+          raise ActiveRecord::Rollback if rollback
         end
       end
+
+      def table_exists?(table)
+        ["PostgreSQL", "PostGIS"].include?(stats_connection.adapter_name) &&
+        select_one_stats(<<-SQL
+          SELECT EXISTS (
+            SELECT
+              1
+            FROM
+              pg_catalog.pg_class c
+            INNER JOIN
+              pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+            WHERE
+              n.nspname = 'public'
+              AND c.relname = #{quote(table)}
+              AND c.relkind = 'r'
+          )
+        SQL
+        )
+      end
     end
   end
 end