pghero 1.4.2 → 1.5.0

data/lib/pghero/methods/connections.rb

@@ -6,7 +6,24 @@ module PgHero
       end
 
       def connection_sources(options = {})
-        if options[:by_database]
+        if options[:by_database_and_user]
+          select_all <<-SQL
+            SELECT
+              datname AS database,
+              usename AS user,
+              application_name AS source,
+              client_addr AS ip,
+              COUNT(*) AS total_connections
+            FROM
+              pg_stat_activity
+            WHERE
+              pid <> pg_backend_pid()
+            GROUP BY
+              1, 2, 3, 4
+            ORDER BY
+              5 DESC, 1, 2, 3, 4
+          SQL
+        elsif options[:by_database]
           select_all <<-SQL
             SELECT
               application_name AS source,

data/lib/pghero/methods/explain.rb

@@ -18,6 +18,8 @@ module PgHero
         explanation
       end
 
+      private
+
       def explain_safe?
         select_all("SELECT 1; SELECT 1")
         false

data/lib/pghero/methods/indexes.rb

@@ -140,8 +140,8 @@ module PgHero
         indexes = []
 
         indexes_by_table = self.indexes.group_by { |i| i["table"] }
-        indexes_by_table.values.flatten.select { |i| falsey?(i["primary"]) && falsey?(i["unique"]) && !i["indexprs"] && !i["indpred"] && truthy?(i["valid"]) }.each do |index|
-          covering_index = indexes_by_table[index["table"]].find { |i| index_covers?(i["columns"], index["columns"]) && i["using"] == index["using"] && i["name"] != index["name"] && truthy?(i["valid"]) }
+        indexes_by_table.values.flatten.select { |i| PgHero.falsey?(i["primary"]) && PgHero.falsey?(i["unique"]) && !i["indexprs"] && !i["indpred"] && PgHero.truthy?(i["valid"]) }.each do |index|
+          covering_index = indexes_by_table[index["table"]].find { |i| index_covers?(i["columns"], index["columns"]) && i["using"] == index["using"] && i["name"] != index["name"] && PgHero.truthy?(i["valid"]) }
           if covering_index
             indexes << {"unneeded_index" => index, "covering_index" => covering_index}
           end

data/lib/pghero/methods/kill.rb

@@ -2,7 +2,7 @@ module PgHero
   module Methods
     module Kill
       def kill(pid)
-        truthy? execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"]
+        PgHero.truthy? execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"]
       end
 
       def kill_long_running_queries

data/lib/pghero/methods/queries.rb

@@ -41,7 +41,7 @@ module PgHero
             query <> '<insufficient privilege>'
             AND state <> 'idle'
             AND pid <> pg_backend_pid()
-            AND now() - query_start > interval '#{long_running_query_sec.to_i} seconds'
+            AND now() - query_start > interval '#{PgHero.long_running_query_sec.to_i} seconds'
             AND datname = current_database()
           ORDER BY
             query_start DESC
@@ -50,7 +50,7 @@ module PgHero
 
       def slow_queries(options = {})
         query_stats = options[:query_stats] || self.query_stats(options.except(:query_stats))
-        query_stats.select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
+        query_stats.select { |q| q["calls"].to_i >= PgHero.slow_query_calls.to_i && q["average_time"].to_i >= PgHero.slow_query_ms.to_i }
       end
 
       def locks

data/lib/pghero/methods/query_stats.rb

@@ -5,8 +5,8 @@ module PgHero
         current_query_stats = options[:historical] && options[:end_at] && options[:end_at] < Time.now ? [] : current_query_stats(options)
         historical_query_stats = options[:historical] ? historical_query_stats(options) : []
 
-        query_stats = combine_query_stats((current_query_stats + historical_query_stats).group_by { |q| q["query_hash"] })
-        query_stats = combine_query_stats(query_stats.group_by { |q| normalize_query(q["query"]) })
+        query_stats = combine_query_stats((current_query_stats + historical_query_stats).group_by { |q| [q["query_hash"], q["user"]] })
+        query_stats = combine_query_stats(query_stats.group_by { |q| [normalize_query(q["query"]), q["user"]] })
 
         # add percentages
         all_queries_total_minutes = [current_query_stats, historical_query_stats].sum { |s| (s.first || {})["all_queries_total_minutes"].to_f }
@@ -63,65 +63,10 @@ module PgHero
         end
       end
 
-      # resetting query stats will reset across the entire Postgres instance
-      # this is problematic if multiple PgHero databases use the same Postgres instance
-      #
-      # to get around this, we capture queries for every Postgres database before we
-      # reset query stats for the Postgres instance with the `capture_query_stats` option
-      def capture_query_stats
-        # get database names
-        pg_databases = {}
-        supports_query_hash = {}
-        config["databases"].each do |k, _|
-          with(k) do
-            pg_databases[k] = execute("SELECT current_database()").first["current_database"]
-            supports_query_hash[k] = supports_query_hash?
-          end
-        end
-
-        config["databases"].reject { |_, v| v["capture_query_stats"] && v["capture_query_stats"] != true }.each do |database, _|
-          with(database) do
-            mapping = {database => pg_databases[database]}
-            config["databases"].select { |_, v| v["capture_query_stats"] == database }.each do |k, _|
-              mapping[k] = pg_databases[k]
-            end
-
-            now = Time.now
-            query_stats = {}
-            mapping.each do |db, pg_database|
-              query_stats[db] = self.query_stats(limit: 1000000, database: pg_database)
-            end
-
-            if query_stats.any? { |_, v| v.any? } && reset_query_stats
-              query_stats.each do |db, db_query_stats|
-                if db_query_stats.any?
-                  values =
-                    db_query_stats.map do |qs|
-                      values = [
-                        db,
-                        qs["query"],
-                        qs["total_minutes"].to_f * 60 * 1000,
-                        qs["calls"],
-                        now
-                      ]
-                      values << qs["query_hash"] if supports_query_hash[db]
-                      values.map { |v| quote(v) }.join(",")
-                    end.map { |v| "(#{v})" }.join(",")
-
-                  columns = %w[database query total_time calls captured_at]
-                  columns << "query_hash" if supports_query_hash[db]
-                  stats_connection.execute("INSERT INTO pghero_query_stats (#{columns.join(", ")}) VALUES #{values}")
-                end
-              end
-            end
-          end
-        end
-      end
-
       # http://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
       def historical_query_stats_enabled?
         # TODO use schema from config
-        truthy? stats_connection.select_all(squish <<-SQL
+        PgHero.truthy?(stats_connection.select_all(squish <<-SQL
           SELECT EXISTS (
             SELECT
               1
@@ -135,15 +80,80 @@ module PgHero
               AND c.relkind = 'r'
           )
         SQL
-        ).to_a.first["exists"]
+        ).to_a.first["exists"]) && capture_query_stats?
       end
 
-      def stats_connection
-        ::PgHero::QueryStats.connection
+      def supports_query_hash?
+        @supports_query_hash ||= server_version >= 90400 && historical_query_stats_enabled? && PgHero::QueryStats.column_names.include?("query_hash")
+      end
+
+      def supports_query_stats_user?
+        @supports_query_stats_user ||= historical_query_stats_enabled? && PgHero::QueryStats.column_names.include?("user")
+      end
+
+      def insert_stats(table, columns, values)
+        values = values.map { |v| "(#{v.map { |v2| quote(v2) }.join(",")})" }.join(",")
+        columns = columns.map { |v| quote_table_name(v) }.join(",")
+        stats_connection.execute("INSERT INTO #{quote_table_name(table)} (#{columns}) VALUES #{values}")
+      end
+
+      # resetting query stats will reset across the entire Postgres instance
+      # this is problematic if multiple PgHero databases use the same Postgres instance
+      #
+      # to get around this, we capture queries for every Postgres database before we
+      # reset query stats for the Postgres instance with the `capture_query_stats` option
+      def capture_query_stats
+        return if config["capture_query_stats"] && config["capture_query_stats"] != true
+
+        # get all databases that use same query stats and build mapping
+        mapping = {id => database_name}
+        PgHero.databases.select { |_, d| d.config["capture_query_stats"] == id }.each do |_, d|
+          mapping[d.id] = d.database_name
+        end
+
+        now = Time.now
+
+        query_stats = {}
+        mapping.each do |database_id, database_name|
+          query_stats[database_id] = query_stats(limit: 1000000, database: database_name)
+        end
+
+        if query_stats.any? { |_, v| v.any? } && reset_query_stats
+          query_stats.each do |db_id, db_query_stats|
+            if db_query_stats.any?
+              supports_query_hash = PgHero.databases[db_id].supports_query_hash?
+              supports_query_stats_user = PgHero.databases[db_id].supports_query_stats_user?
+
+              values =
+                db_query_stats.map do |qs|
+                  values = [
+                    db_id,
+                    qs["query"],
+                    qs["total_minutes"].to_f * 60 * 1000,
+                    qs["calls"],
+                    now
+                  ]
+                  values << qs["query_hash"] if supports_query_hash
+                  values << qs["user"] if supports_query_stats_user
+                  values
+                end
+
+              columns = %w[database query total_time calls captured_at]
+              columns << "query_hash" if supports_query_hash
+              columns << "user" if supports_query_stats_user
+
+              insert_stats("pghero_query_stats", columns, values)
+            end
+          end
+        end
       end
 
       private
 
+      def stats_connection
+        ::PgHero::QueryStats.connection
+      end
+
       # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
       def current_query_stats(options = {})
         if query_stats_enabled?
@@ -155,6 +165,7 @@ module PgHero
               SELECT
                 LEFT(query, 10000) AS query,
                 #{supports_query_hash? ? "queryid" : "md5(query)"} AS query_hash,
+                #{supports_query_stats_user? ? "rolname" : "NULL"} AS user,
                 (total_time / 1000 / 60) AS total_minutes,
                 (total_time / calls) AS average_time,
                 calls
@@ -162,12 +173,15 @@ module PgHero
                 pg_stat_statements
               INNER JOIN
                 pg_database ON pg_database.oid = pg_stat_statements.dbid
+              INNER JOIN
+                pg_roles ON pg_roles.oid = pg_stat_statements.userid
               WHERE
                 pg_database.datname = #{database}
             )
             SELECT
               query,
               query_hash,
+              query_stats.user,
               total_minutes,
               average_time,
               calls,
@@ -191,6 +205,7 @@ module PgHero
             WITH query_stats AS (
               SELECT
                 #{supports_query_hash? ? "query_hash" : "md5(query)"} AS query_hash,
+                #{supports_query_stats_user? ? "pghero_query_stats.user" : "NULL"} AS user,
                 array_agg(LEFT(query, 10000)) AS query,
                 (SUM(total_time) / 1000 / 60) AS total_minutes,
                 (SUM(total_time) / SUM(calls)) AS average_time,
@@ -198,15 +213,16 @@ module PgHero
               FROM
                 pghero_query_stats
               WHERE
-                database = #{quote(current_database)}
+                database = #{quote(id)}
                 #{supports_query_hash? ? "AND query_hash IS NOT NULL" : ""}
                 #{options[:start_at] ? "AND captured_at >= #{quote(options[:start_at])}" : ""}
                 #{options[:end_at] ? "AND captured_at <= #{quote(options[:end_at])}" : ""}
               GROUP BY
-                1
+                1, 2
             )
             SELECT
               query_hash,
+              query_stats.user,
               query[1],
               total_minutes,
               average_time,
@@ -224,26 +240,16 @@ module PgHero
         end
       end
 
-      def supports_query_hash?
-        @supports_query_hash ||= {}
-        if @supports_query_hash[current_database].nil?
-          @supports_query_hash[current_database] = server_version >= 90400 && historical_query_stats_enabled? && PgHero::QueryStats.column_names.include?("query_hash")
-        end
-        @supports_query_hash[current_database]
-      end
-
       def server_version
-        @server_version ||= {}
-        @server_version[current_database] ||= select_all("SHOW server_version_num").first["server_version_num"].to_i
+        @server_version ||= select_all("SHOW server_version_num").first["server_version_num"].to_i
       end
 
-      private
-
       def combine_query_stats(grouped_stats)
         query_stats = []
         grouped_stats.each do |_, stats2|
           value = {
             "query" => (stats2.find { |s| s["query"] } || {})["query"],
+            "user" => (stats2.find { |s| s["user"] } || {})["user"],
             "query_hash" => (stats2.find { |s| s["query"] } || {})["query_hash"],
             "total_minutes" => stats2.sum { |s| s["total_minutes"].to_f },
             "calls" => stats2.sum { |s| s["calls"].to_i },

data/lib/pghero/methods/space.rb

@@ -11,7 +11,8 @@ module PgHero
             n.nspname AS schema,
             c.relname AS name,
             CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
-            pg_size_pretty(pg_table_size(c.oid)) AS size
+            pg_size_pretty(pg_table_size(c.oid)) AS size,
+            pg_table_size(c.oid) AS size_bytes
           FROM
             pg_class c
           LEFT JOIN
@@ -25,6 +26,16 @@ module PgHero
             name ASC
         SQL
       end
+
+      def capture_space_stats
+        now = Time.now
+        columns = %w[database schema relation size captured_at]
+        values = []
+        relation_sizes.each do |rs|
+          values << [id, rs["schema"], rs["name"], rs["size_bytes"].to_i, now]
+        end
+        insert_stats("pghero_space_stats", columns, values)
+      end
     end
   end
 end

data/lib/pghero/methods/suggested_indexes.rb

@@ -188,7 +188,9 @@ module PgHero
         return {error: "Too large"} if statement.to_s.length > 10000
 
         begin
-          tree = PgQuery.parse(statement).parsetree
+          parsed_statement = PgQuery.parse(statement)
+          v2 = parsed_statement.respond_to?(:tree)
+          tree = v2 ? parsed_statement.tree : parsed_statement.parsetree
         rescue PgQuery::ParseError
           return {error: "Parse error"}
         end
@@ -199,10 +201,14 @@ module PgHero
         unless table
           error =
             case tree.keys.first
-            when "INSERT INTO"
+            when "InsertStmt", "INSERT INTO"
               "INSERT statement"
-            when "SET"
+            when "VariableSetStmt", "SET"
               "SET statement"
+            when "SelectStmt"
+              if (tree["SelectStmt"]["fromClause"].first["JoinExpr"] rescue false)
+                "JOIN not supported yet"
+              end
             when "SELECT"
               if (tree["SELECT"]["fromClause"].first["JOINEXPR"] rescue false)
                 "JOIN not supported yet"
@@ -211,11 +217,11 @@ module PgHero
           return {error: error || "Unknown structure"}
         end
 
-        select = tree["SELECT"] || tree["DELETE FROM"] || tree["UPDATE"]
-        where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
+        select = tree.values.first
+        where = (select["whereClause"] ? parse_where(select["whereClause"], v2) : []) rescue nil
         return {error: "Unknown structure"} unless where
 
-        sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue []
+        sort = (select["sortClause"] ? parse_sort(select["sortClause"], v2) : []) rescue []
 
         {table: table, where: where, sort: sort}
       end
@@ -260,6 +266,12 @@ module PgHero
 
       def parse_table(tree)
         case tree.keys.first
+        when "SelectStmt"
+          tree["SelectStmt"]["fromClause"].first["RangeVar"]["relname"]
+        when "DeleteStmt"
+          tree["DeleteStmt"]["relation"]["RangeVar"]["relname"]
+        when "UpdateStmt"
+          tree["UpdateStmt"]["relation"]["RangeVar"]["relname"]
         when "SELECT"
           tree["SELECT"]["fromClause"].first["RANGEVAR"]["relname"]
         when "DELETE FROM"
@@ -270,41 +282,69 @@ module PgHero
       end
 
       # TODO capture values
-      def parse_where(tree)
-        aexpr = tree["AEXPR"] || tree[nil]
+      def parse_where(tree, v2 = false)
+        if v2
+          aexpr = tree["A_Expr"]
 
-        if tree["BOOLEXPR"]
-          if tree["BOOLEXPR"]["boolop"] == 0
-            tree["BOOLEXPR"]["args"].flat_map { |v| parse_where(v) }
+          if tree["BoolExpr"]
+            if tree["BoolExpr"]["boolop"] == 0
+              tree["BoolExpr"]["args"].flat_map { |v| parse_where(v, v2) }
+            else
+              raise "Not Implemented"
+            end
+          elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr["name"].first["String"]["str"])
+            [{column: aexpr["lexpr"]["ColumnRef"]["fields"].last["String"]["str"], op: aexpr["name"].first["String"]["str"]}]
+          elsif tree["NullTest"]
+            op = tree["NullTest"]["nulltesttype"] == 1 ? "not_null" : "null"
+            [{column: tree["NullTest"]["arg"]["ColumnRef"]["fields"].last["String"]["str"], op: op}]
           else
             raise "Not Implemented"
           end
-        elsif tree["AEXPR AND"]
-          left = parse_where(tree["AEXPR AND"]["lexpr"])
-          right = parse_where(tree["AEXPR AND"]["rexpr"])
-          if left && right
-            left + right
+        else
+          aexpr = tree["AEXPR"] || tree[nil]
+
+          if tree["BOOLEXPR"]
+            if tree["BOOLEXPR"]["boolop"] == 0
+              tree["BOOLEXPR"]["args"].flat_map { |v| parse_where(v) }
+            else
+              raise "Not Implemented"
+            end
+          elsif tree["AEXPR AND"]
+            left = parse_where(tree["AEXPR AND"]["lexpr"])
+            right = parse_where(tree["AEXPR AND"]["rexpr"])
+            if left && right
+              left + right
+            else
+              raise "Not Implemented"
+            end
+          elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr["name"].first)
+            [{column: aexpr["lexpr"]["COLUMNREF"]["fields"].last, op: aexpr["name"].first}]
+          elsif tree["AEXPR IN"] && ["=", "<>"].include?(tree["AEXPR IN"]["name"].first)
+            [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR IN"]["name"].first}]
+          elsif tree["NULLTEST"]
+            op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
+            [{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]
           else
             raise "Not Implemented"
           end
-        elsif aexpr && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*", "BETWEEN"].include?(aexpr["name"].first)
-          [{column: aexpr["lexpr"]["COLUMNREF"]["fields"].last, op: aexpr["name"].first}]
-        elsif tree["AEXPR IN"] && ["=", "<>"].include?(tree["AEXPR IN"]["name"].first)
-          [{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR IN"]["name"].first}]
-        elsif tree["NULLTEST"]
-          op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
-          [{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]
-        else
-          raise "Not Implemented"
         end
       end
 
-      def parse_sort(sort_clause)
-        sort_clause.map do |v|
-          {
-            column: v["SORTBY"]["node"]["COLUMNREF"]["fields"].last,
-            direction: v["SORTBY"]["sortby_dir"] == 2 ? "desc" : "asc"
-          }
+      def parse_sort(sort_clause, v2)
+        if v2
+          sort_clause.map do |v|
+            {
+              column: v["SortBy"]["node"]["ColumnRef"]["fields"].last["String"]["str"],
+              direction: v["SortBy"]["sortby_dir"] == 2 ? "desc" : "asc"
+            }
+          end
+        else
+          sort_clause.map do |v|
+            {
+              column: v["SORTBY"]["node"]["COLUMNREF"]["fields"].last,
+              direction: v["SORTBY"]["sortby_dir"] == 2 ? "desc" : "asc"
+            }
+          end
         end
       end