pghero 3.7.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +22 -2
  3. data/LICENSE.txt +1 -1
  4. data/README.md +4 -3
  5. data/app/assets/javascripts/pghero/Chart.bundle.js +2228 -5320
  6. data/app/assets/javascripts/pghero/application.js +104 -68
  7. data/app/assets/javascripts/pghero/highlight.min.js +330 -341
  8. data/app/assets/javascripts/pghero/nouislider.js +64 -5
  9. data/app/assets/stylesheets/pghero/application.css +7 -1
  10. data/app/controllers/pg_hero/home_controller.rb +75 -41
  11. data/app/views/layouts/pg_hero/application.html.erb +1 -1
  12. data/app/views/pg_hero/home/_live_queries_table.html.erb +4 -2
  13. data/app/views/pg_hero/home/_queries_table.html.erb +8 -7
  14. data/app/views/pg_hero/home/_query_stats_slider.html.erb +1 -0
  15. data/app/views/pg_hero/home/_suggested_index.html.erb +3 -3
  16. data/app/views/pg_hero/home/explain.html.erb +2 -1
  17. data/app/views/pg_hero/home/index.html.erb +17 -18
  18. data/app/views/pg_hero/home/index_bloat.html.erb +2 -2
  19. data/app/views/pg_hero/home/live_queries.html.erb +7 -3
  20. data/app/views/pg_hero/home/queries.html.erb +31 -6
  21. data/app/views/pg_hero/home/show_query.html.erb +5 -5
  22. data/app/views/pg_hero/home/space.html.erb +5 -4
  23. data/app/views/pg_hero/home/tune.html.erb +1 -1
  24. data/lib/generators/pghero/templates/config.yml.tt +6 -1
  25. data/lib/generators/pghero/templates/query_stats.rb.tt +7 -1
  26. data/lib/generators/pghero/templates/upgrade_query_stats.rb.tt +23 -0
  27. data/lib/generators/pghero/upgrade_query_stats_generator.rb +19 -0
  28. data/lib/pghero/database.rb +6 -9
  29. data/lib/pghero/engine.rb +3 -10
  30. data/lib/pghero/methods/basic.rb +28 -27
  31. data/lib/pghero/methods/connections.rb +16 -32
  32. data/lib/pghero/methods/explain.rb +13 -20
  33. data/lib/pghero/methods/indexes.rb +8 -5
  34. data/lib/pghero/methods/kill.rb +4 -1
  35. data/lib/pghero/methods/maintenance.rb +14 -20
  36. data/lib/pghero/methods/queries.rb +7 -5
  37. data/lib/pghero/methods/query_stats.rb +264 -238
  38. data/lib/pghero/methods/replication.rb +9 -20
  39. data/lib/pghero/methods/sequences.rb +1 -1
  40. data/lib/pghero/methods/settings.rb +6 -11
  41. data/lib/pghero/methods/space.rb +11 -8
  42. data/lib/pghero/methods/suggested_indexes.rb +15 -16
  43. data/lib/pghero/methods/system.rb +6 -106
  44. data/lib/pghero/methods/tables.rb +7 -3
  45. data/lib/pghero/query.rb +8 -0
  46. data/lib/pghero/query_stats.rb +3 -0
  47. data/lib/pghero/version.rb +1 -1
  48. data/lib/pghero.rb +10 -42
  49. data/licenses/LICENSE-chart.js.txt +1 -1
  50. metadata +8 -8
  51. data/app/assets/javascripts/pghero/jquery.js +0 -10993
  52. data/lib/pghero/methods/users.rb +0 -95
  53. data/licenses/LICENSE-jquery.txt +0 -20
@@ -8,25 +8,27 @@ module PgHero
8
8
  state,
9
9
  application_name AS source,
10
10
  age(NOW(), COALESCE(query_start, xact_start)) AS duration,
11
- #{server_version_num >= 90600 ? "(wait_event IS NOT NULL) AS waiting" : "waiting"},
11
+ wait_event IS NOT NULL AS waiting,
12
12
  query,
13
13
  COALESCE(query_start, xact_start) AS started_at,
14
14
  EXTRACT(EPOCH FROM NOW() - COALESCE(query_start, xact_start)) * 1000.0 AS duration_ms,
15
15
  usename AS user,
16
- #{server_version_num >= 100000 ? "backend_type" : "NULL AS backend_type"}
16
+ backend_type
17
17
  FROM
18
18
  pg_stat_activity
19
19
  WHERE
20
20
  state <> 'idle'
21
21
  AND pid <> pg_backend_pid()
22
22
  AND datname = current_database()
23
- #{min_duration ? "AND NOW() - COALESCE(query_start, xact_start) > interval '#{min_duration.to_i} seconds'" : nil}
24
- #{all ? nil : "AND query <> '<insufficient privilege>'"}
23
+ #{"AND NOW() - COALESCE(query_start, xact_start) > interval :min_duration" if min_duration}
24
+ #{"AND query <> '<insufficient privilege>'" if !all}
25
25
  ORDER BY
26
26
  COALESCE(query_start, xact_start) DESC
27
27
  SQL
28
28
 
29
- select_all(query, query_columns: [:query])
29
+ binds = {}
30
+ binds[:min_duration] = "#{min_duration.to_i} seconds" if min_duration
31
+ select_all(query, binds, query_columns: [:query])
30
32
  end
31
33
 
32
34
  def long_running_queries
@@ -1,22 +1,56 @@
1
1
  module PgHero
2
2
  module Methods
3
3
  module QueryStats
4
- def query_stats(historical: false, start_at: nil, end_at: nil, min_average_time: nil, min_calls: nil, **options)
5
- current_query_stats = historical && end_at && end_at < Time.now ? [] : current_query_stats(**options)
6
- historical_query_stats = historical && historical_query_stats_enabled? ? historical_query_stats(start_at: start_at, end_at: end_at, **options) : []
4
+ def query_stats(
5
+ current: true,
6
+ historical: false,
7
+ limit: nil,
8
+ sort: nil,
9
+ user: nil,
10
+ query_hash: nil,
11
+ start_at: nil,
12
+ end_at: nil,
13
+ min_average_time: nil,
14
+ min_calls: nil
15
+ )
16
+ limit ||= 100
17
+
18
+ sort ||= "total_time"
19
+ unless ["total_time", "average_time", "calls"].include?(sort)
20
+ raise ArgumentError, "Invalid sort"
21
+ end
22
+
23
+ current_query_stats, current_total_time =
24
+ if !current || (historical && end_at && end_at < Time.now)
25
+ [[], 0]
26
+ else
27
+ current_query_stats(limit: limit, sort: sort, user: user, query_hash: query_hash)
28
+ end
7
29
 
8
- query_stats = combine_query_stats((current_query_stats + historical_query_stats).group_by { |q| [q[:query_hash], q[:user]] })
9
- query_stats = combine_query_stats(query_stats.group_by { |q| [normalize_query(q[:query]), q[:user]] })
30
+ historical_query_stats, historical_total_time =
31
+ if historical && historical_query_stats_enabled?
32
+ historical_query_stats(limit: limit, sort: sort, user: user, query_hash: query_hash, start_at: start_at, end_at: end_at)
33
+ else
34
+ [[], 0]
35
+ end
10
36
 
11
- # add percentages
12
- all_queries_total_minutes = [current_query_stats, historical_query_stats].sum { |s| (s.first || {})[:all_queries_total_minutes] || 0 }
37
+ query_stats = current_query_stats + historical_query_stats
38
+ query_stats = combine_query_stats(query_stats.group_by { |q| [q[:query_hash], q[:user]] })
13
39
  query_stats.each do |query|
14
- query[:average_time] = query[:total_minutes] * 1000 * 60 / query[:calls]
15
- query[:total_percent] = query[:total_minutes] * 100.0 / all_queries_total_minutes
40
+ query[:average_time] = query[:total_time] / query[:calls]
16
41
  end
17
42
 
18
- sort = options[:sort] || "total_minutes"
19
- query_stats = query_stats.sort_by { |q| -q[sort.to_sym] }.first(100)
43
+ # add total percent when not filtering by user or query hash
44
+ # could make accurate for these by changing location of filters in queries
45
+ # but not needed at the moment
46
+ if user.nil? && query_hash.nil?
47
+ all_queries_total_time = current_total_time + historical_total_time
48
+ query_stats.each do |query|
49
+ query[:total_percent] = query[:total_time] * 100.0 / all_queries_total_time
50
+ end
51
+ end
52
+
53
+ query_stats = query_stats.sort_by { |q| -q[sort.to_sym] }.first(limit)
20
54
  if min_average_time
21
55
  query_stats.reject! { |q| q[:average_time] < min_average_time }
22
56
  end
@@ -56,45 +90,30 @@ module PgHero
56
90
  true
57
91
  end
58
92
 
59
- def reset_query_stats(**options)
60
- raise PgHero::Error, "Use reset_instance_query_stats to pass database" if options.delete(:database)
61
- reset_instance_query_stats(**options, database: database_name)
62
- end
63
-
64
- # resets query stats for the entire instance
65
- # it's possible to reset stats for a specific
66
- # database, user or query hash in Postgres 12+
67
- def reset_instance_query_stats(database: nil, user: nil, query_hash: nil, raise_errors: false)
68
- if database || user || query_hash
69
- raise PgHero::Error, "Requires PostgreSQL 12+" if server_version_num < 120000
93
+ def reset_query_stats(user: nil, query_hash: nil, raise_errors: false)
94
+ database = database_name
95
+ database_id = select_one("SELECT oid FROM pg_database WHERE datname = :database", {database: database})
96
+ raise Error, "Database not found: #{database}" unless database_id
70
97
 
71
- if database
72
- database_id = execute("SELECT oid FROM pg_database WHERE datname = #{quote(database)}").first.try(:[], "oid")
73
- raise PgHero::Error, "Database not found: #{database}" unless database_id
74
- else
75
- database_id = 0
76
- end
77
-
78
- if user
79
- user_id = execute("SELECT usesysid FROM pg_user WHERE usename = #{quote(user)}").first.try(:[], "usesysid")
80
- raise PgHero::Error, "User not found: #{user}" unless user_id
81
- else
82
- user_id = 0
83
- end
84
-
85
- if query_hash
86
- query_id = query_hash.to_i
87
- # may not be needed
88
- # but not intuitive that all query hashes are reset with 0
89
- raise PgHero::Error, "Invalid query hash: #{query_hash}" if query_id == 0
90
- else
91
- query_id = 0
92
- end
98
+ if user
99
+ user_id = select_one("SELECT usesysid FROM pg_user WHERE usename = :user", {user: user})
100
+ raise Error, "User not found: #{user}" unless user_id
101
+ else
102
+ user_id = 0
103
+ end
93
104
 
94
- execute("SELECT pg_stat_statements_reset(#{quote(user_id.to_i)}, #{quote(database_id.to_i)}, #{quote(query_id.to_i)})")
105
+ if query_hash
106
+ query_id = query_hash.to_i
107
+ # may not be needed
108
+ # but not intuitive that all query hashes are reset with 0
109
+ raise Error, "Invalid query hash: #{query_hash}" if query_id == 0
95
110
  else
96
- execute("SELECT pg_stat_statements_reset()")
111
+ query_id = 0
97
112
  end
113
+
114
+ binds = {user_id: user_id, database_id: database_id, query_id: query_id}
115
+ # use execute to prevent "unknown OID 2278" warning
116
+ execute("SELECT pg_stat_statements_reset(:user_id, :database_id, :query_id)", binds)
98
117
  true
99
118
  rescue ActiveRecord::StatementInvalid => e
100
119
  raise e if raise_errors
@@ -105,60 +124,22 @@ module PgHero
105
124
  def historical_query_stats_enabled?
106
125
  # TODO use schema from config
107
126
  # make sure primary database is PostgreSQL first
108
- query_stats_table_exists? && capture_query_stats? && !missing_query_stats_columns.any?
127
+ queries_table_exists? && query_stats_table_exists? && capture_query_stats?
109
128
  end
110
129
 
111
- def query_stats_table_exists?
112
- table_exists?("pghero_query_stats")
113
- end
114
-
115
- def missing_query_stats_columns
116
- %w(query_hash user) - PgHero::QueryStats.column_names
130
+ def queries_table_exists?
131
+ table_exists?("pghero_queries")
117
132
  end
118
133
 
119
- def supports_query_hash?
120
- server_version_num >= 90400
134
+ def query_stats_table_exists?
135
+ table_exists?("pghero_query_stats")
121
136
  end
122
137
 
123
- # resetting query stats will reset across the entire Postgres instance in Postgres < 12
124
- # this is problematic if multiple PgHero databases use the same Postgres instance
125
- #
126
- # to get around this, we capture queries for every Postgres database before we
127
- # reset query stats for the Postgres instance with the `capture_query_stats` option
128
138
  def capture_query_stats(raise_errors: false)
129
- return if config["capture_query_stats"] && config["capture_query_stats"] != true
130
-
131
- # get all databases that use same query stats and build mapping
132
- mapping = {id => database_name}
133
- PgHero.databases.select { |_, d| d.config["capture_query_stats"] == id }.each do |_, d|
134
- mapping[d.id] = d.database_name
135
- end
136
-
137
- now = Time.now
138
-
139
- query_stats = {}
140
- mapping.each do |database_id, database_name|
141
- query_stats[database_id] = query_stats(limit: 1000000, database: database_name)
142
- end
143
-
144
- query_stats = query_stats.select { |_, v| v.any? }
145
-
146
- # nothing to do
147
- return if query_stats.empty?
148
-
149
- # reset individual databases for Postgres 12+ instance
150
- if server_version_num >= 120000
151
- query_stats.each do |db_id, db_query_stats|
152
- if reset_instance_query_stats(database: mapping[db_id], raise_errors: raise_errors)
153
- insert_query_stats(db_id, db_query_stats, now)
154
- end
155
- end
156
- else
157
- if reset_instance_query_stats(raise_errors: raise_errors)
158
- query_stats.each do |db_id, db_query_stats|
159
- insert_query_stats(db_id, db_query_stats, now)
160
- end
161
- end
139
+ captured_at = Time.now
140
+ db_query_stats = query_stats(limit: 100)
141
+ if db_query_stats.any? && reset_query_stats(raise_errors: raise_errors)
142
+ insert_query_stats(db_query_stats, captured_at)
162
143
  end
163
144
  end
164
145
 
@@ -172,186 +153,231 @@ module PgHero
172
153
  query_stats.select { |q| q[:calls].to_i >= slow_query_calls.to_i && q[:average_time].to_f >= slow_query_ms.to_f }
173
154
  end
174
155
 
175
- def query_hash_stats(query_hash, user: nil, current: false)
176
- if historical_query_stats_enabled? && supports_query_hash?
177
- start_at = 24.hours.ago
178
- stats = select_all_stats <<~SQL
179
- SELECT
180
- captured_at,
181
- total_time / 1000 / 60 AS total_minutes,
182
- (total_time / calls) AS average_time,
183
- calls,
184
- (SELECT regexp_matches(query, '.*/\\*(.+?)\\*/'))[1] AS origin
185
- FROM
186
- pghero_query_stats
187
- WHERE
188
- database = #{quote(id)}
189
- AND captured_at >= #{quote(start_at)}
190
- AND query_hash = #{quote(query_hash)}
191
- #{user ? "AND \"user\" = #{quote(user)}" : ""}
192
- ORDER BY
193
- 1 ASC
194
- SQL
195
- if current
196
- captured_at = Time.current
197
- current_stats = current_query_stats(query_hash: query_hash, user: user, origin: true)
198
- current_stats.each do |r|
199
- r[:captured_at] = captured_at
200
- end
201
- stats += current_stats
202
- end
203
- stats
204
- else
156
+ def query_hash_stats(query_hash, user: nil, current: true)
157
+ if !historical_query_stats_enabled?
205
158
  raise NotEnabled, "Query hash stats not enabled"
206
159
  end
160
+
161
+ start_at = 24.hours.ago
162
+ # specify pghero_queries.query in case pghero_query_stats.query exists
163
+ sql = <<~SQL
164
+ SELECT
165
+ captured_at,
166
+ total_time,
167
+ calls,
168
+ (SELECT regexp_matches(pghero_queries.query, '.*/\\*(.+?)\\*/'))[1] AS origin
169
+ FROM
170
+ pghero_query_stats
171
+ INNER JOIN
172
+ pghero_queries ON pghero_queries.id = pghero_query_stats.query_id
173
+ WHERE
174
+ database = :id
175
+ AND captured_at >= :start_at
176
+ AND query_hash = :query_hash
177
+ #{"AND \"user\" = :user" if user}
178
+ ORDER BY
179
+ 1 ASC
180
+ SQL
181
+ binds = {id: id, start_at: start_at, query_hash: query_hash}
182
+ binds[:user] = user if user
183
+ stats = select_all_stats(sql, binds)
184
+ if current
185
+ captured_at = Time.now
186
+ current_stats, _ = current_query_stats(query_hash: query_hash, user: user, origin: true)
187
+ current_stats.each do |r|
188
+ stats << {
189
+ captured_at: captured_at,
190
+ total_time: r[:total_time],
191
+ calls: r[:calls],
192
+ origin: r[:origin]
193
+ }
194
+ end
195
+ end
196
+ stats.each do |query|
197
+ query[:average_time] = query[:total_time] / query[:calls]
198
+ end
199
+ stats
207
200
  end
208
201
 
209
202
  private
210
203
 
211
- # http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
212
- def current_query_stats(limit: nil, sort: nil, database: nil, query_hash: nil, user: nil, origin: false)
213
- if query_stats_enabled?
214
- limit ||= 100
215
- sort ||= "total_minutes"
216
- total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
217
- query = <<~SQL
218
- WITH query_stats AS (
219
- SELECT
220
- LEFT(query, 10000) AS query,
221
- #{supports_query_hash? ? "queryid" : "md5(query)"} AS query_hash,
222
- rolname AS user,
223
- (#{total_time} / 1000 / 60) AS total_minutes,
224
- (#{total_time} / calls) AS average_time,
225
- calls
226
- FROM
227
- pg_stat_statements
228
- INNER JOIN
229
- pg_database ON pg_database.oid = pg_stat_statements.dbid
230
- INNER JOIN
231
- pg_roles ON pg_roles.oid = pg_stat_statements.userid
232
- WHERE
233
- calls > 0 AND
234
- pg_database.datname = #{database ? quote(database) : "current_database()"}
235
- #{query_hash ? "AND queryid = #{quote(query_hash)}" : nil}
236
- #{user ? "AND rolname = #{quote(user)}" : nil}
237
- )
204
+ # https://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
205
+ def current_query_stats(limit: nil, sort: nil, user: nil, query_hash: nil, origin: false)
206
+ if !query_stats_enabled?
207
+ raise NotEnabled, "Query stats not enabled"
208
+ end
209
+
210
+ limit ||= 100
211
+ sort ||= "total_time"
212
+ query = <<~SQL
213
+ WITH query_stats AS (
214
+ SELECT
215
+ LEFT(query, 10000) AS query,
216
+ queryid AS query_hash,
217
+ rolname AS user,
218
+ total_plan_time + total_exec_time AS total_time,
219
+ #{"(total_plan_time + total_exec_time) / calls AS average_time," if sort == "average_time"}
220
+ calls
221
+ FROM
222
+ pg_stat_statements
223
+ INNER JOIN
224
+ pg_database ON pg_database.oid = pg_stat_statements.dbid
225
+ INNER JOIN
226
+ pg_roles ON pg_roles.oid = pg_stat_statements.userid
227
+ WHERE
228
+ calls > 0 AND
229
+ pg_database.datname = current_database()
230
+ #{"AND rolname = :user" if user}
231
+ #{"AND queryid = :query_hash" if query_hash}
232
+ )
233
+ (
238
234
  SELECT
239
235
  query,
240
- query AS explainable_query,
241
- #{origin ? "(SELECT regexp_matches(query, '.*/\\*(.+?)\\*/'))[1] AS origin," : nil}
236
+ #{"(SELECT regexp_matches(query, '.*/\\*(.+?)\\*/'))[1] AS origin," if origin}
242
237
  query_hash,
243
238
  query_stats.user,
244
- total_minutes,
245
- average_time,
246
- calls,
247
- total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
248
- (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
239
+ total_time,
240
+ calls
249
241
  FROM
250
242
  query_stats
251
243
  ORDER BY
252
244
  #{quote_column_name(sort)} DESC
253
- LIMIT #{limit.to_i}
254
- SQL
245
+ LIMIT :limit
246
+ ) UNION ALL (
247
+ SELECT NULL, #{"NULL, " if origin}NULL, NULL, SUM(total_time), NULL FROM query_stats
248
+ )
249
+ SQL
250
+
251
+ binds = {limit: limit.to_i}
252
+ binds[:user] = user if user
253
+ binds[:query_hash] = query_hash if query_hash
254
+
255
+ # we may be able to skip query_columns
256
+ # in more recent versions of Postgres
257
+ # as pg_stat_statements should be already normalized
258
+ result = select_all(query, binds, query_columns: [:query])
259
+ total = result.pop
260
+ [result, total[:total_time] || 0]
261
+ end
255
262
 
256
- # we may be able to skip query_columns
257
- # in more recent versions of Postgres
258
- # as pg_stat_statements should be already normalized
259
- select_all(query, query_columns: [:query, :explainable_query])
260
- else
261
- raise NotEnabled, "Query stats not enabled"
263
+ def historical_query_stats(limit: nil, sort: nil, user: nil, query_hash: nil, start_at: nil, end_at: nil)
264
+ if !historical_query_stats_enabled?
265
+ raise NotEnabled, "Historical query stats not enabled"
262
266
  end
263
- end
264
267
 
265
- def historical_query_stats(sort: nil, start_at: nil, end_at: nil, query_hash: nil)
266
- if historical_query_stats_enabled?
267
- sort ||= "total_minutes"
268
- query = <<~SQL
269
- WITH query_stats AS (
270
- SELECT
271
- #{supports_query_hash? ? "query_hash" : "md5(query)"} AS query_hash,
272
- pghero_query_stats.user AS user,
273
- array_agg(LEFT(query, 10000) ORDER BY REPLACE(LEFT(query, 1000), '?', '!') COLLATE "C" ASC) AS query,
274
- (SUM(total_time) / 1000 / 60) AS total_minutes,
275
- (SUM(total_time) / SUM(calls)) AS average_time,
276
- SUM(calls) AS calls
277
- FROM
278
- pghero_query_stats
279
- WHERE
280
- database = #{quote(id)}
281
- #{supports_query_hash? ? "AND query_hash IS NOT NULL" : ""}
282
- #{start_at ? "AND captured_at >= #{quote(start_at)}" : ""}
283
- #{end_at ? "AND captured_at <= #{quote(end_at)}" : ""}
284
- #{query_hash ? "AND query_hash = #{quote(query_hash)}" : ""}
285
- GROUP BY
286
- 1, 2
287
- )
268
+ limit ||= 100
269
+ sort ||= "total_time"
270
+ query = <<~SQL
271
+ WITH query_stats AS (
288
272
  SELECT
289
273
  query_hash,
290
- query_stats.user,
291
- query[1] AS query,
292
- query[array_length(query, 1)] AS explainable_query,
293
- total_minutes,
294
- average_time,
295
- calls,
296
- total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
297
- (SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
274
+ "user",
275
+ query_id,
276
+ SUM(total_time) AS total_time,
277
+ SUM(calls) AS calls
278
+ FROM
279
+ pghero_query_stats
280
+ WHERE
281
+ database = :id
282
+ #{"AND captured_at >= :start_at" if start_at}
283
+ #{"AND captured_at <= :end_at" if end_at}
284
+ #{"AND \"user\" = :user" if user}
285
+ #{query_hash ? "AND query_hash = :query_hash" : "AND query_hash IS NOT NULL"}
286
+ GROUP BY
287
+ 1, 2, 3
288
+ ),
289
+ grouped_query_stats AS (
290
+ SELECT
291
+ query_hash,
292
+ "user",
293
+ (array_agg(query_id ORDER BY total_time DESC))[1] AS query_id,
294
+ SUM(total_time) AS total_time,
295
+ #{"SUM(total_time) / SUM(calls) AS average_time," if sort == "average_time"}
296
+ SUM(calls) AS calls
298
297
  FROM
299
298
  query_stats
299
+ GROUP BY
300
+ 1, 2
300
301
  ORDER BY
301
302
  #{quote_column_name(sort)} DESC
302
- LIMIT 100
303
- SQL
304
-
305
- # we can skip query_columns if all stored data is normalized
306
- # for now, assume it's not
307
- select_all_stats(query, query_columns: [:query, :explainable_query])
308
- else
309
- raise NotEnabled, "Historical query stats not enabled"
310
- end
303
+ LIMIT :limit
304
+ )
305
+ (
306
+ SELECT
307
+ query_hash,
308
+ "user",
309
+ query,
310
+ total_time,
311
+ calls
312
+ FROM
313
+ grouped_query_stats
314
+ INNER JOIN
315
+ pghero_queries ON pghero_queries.id = grouped_query_stats.query_id
316
+ ORDER BY
317
+ #{quote_column_name(sort)} DESC
318
+ ) UNION ALL (
319
+ SELECT NULL, NULL, NULL, SUM(total_time), NULL FROM query_stats
320
+ )
321
+ SQL
322
+
323
+ binds = {id: id, limit: limit.to_i}
324
+ binds[:start_at] = start_at if start_at
325
+ binds[:end_at] = end_at if end_at
326
+ binds[:user] = user if user
327
+ binds[:query_hash] = query_hash if query_hash
328
+
329
+ # we can skip query_columns if all stored data is normalized
330
+ # for now, assume it's not
331
+ result = select_all_stats(query, binds, query_columns: [:query])
332
+ total = result.pop
333
+ [result, total[:total_time] || 0]
311
334
  end
312
335
 
313
336
  def combine_query_stats(grouped_stats)
314
- query_stats = []
315
- grouped_stats.each do |_, stats2|
316
- value = {
317
- query: (stats2.find { |s| s[:query] } || {})[:query],
318
- user: (stats2.find { |s| s[:user] } || {})[:user],
319
- query_hash: (stats2.find { |s| s[:query_hash] } || {})[:query_hash],
320
- total_minutes: stats2.sum { |s| s[:total_minutes] },
321
- calls: stats2.sum { |s| s[:calls] }.to_i,
322
- all_queries_total_minutes: stats2.sum { |s| s[:all_queries_total_minutes] }
337
+ grouped_stats.map do |_, stats|
338
+ {
339
+ query: stats.filter_map { |v| v[:query] }.first,
340
+ user: stats.filter_map { |v| v[:user] }.first,
341
+ query_hash: stats.filter_map { |v| v[:query_hash] }.first,
342
+ total_time: stats.sum { |s| s[:total_time] },
343
+ calls: stats.sum { |s| s[:calls] }.to_i
323
344
  }
324
- value[:total_percent] = value[:total_minutes] * 100.0 / value[:all_queries_total_minutes]
325
- value[:explainable_query] = stats2.map { |s| s[:explainable_query] }.find { |q| q && explainable?(q) }
326
- query_stats << value
327
345
  end
328
- query_stats
329
346
  end
330
347
 
331
348
  def explainable?(query)
332
- query =~ /select/i && (server_version_num >= 160000 || (!query.include?("?)") && !query.include?("= ?") && !query.include?("$1") && query !~ /limit \?/i))
349
+ query =~ /select/i && (server_version_num >= 160000 || !query.include?("$1"))
333
350
  end
334
351
 
335
- # removes comments
336
- # combines ?, ?, ? => ?
337
- def normalize_query(query)
338
- squish(query.to_s.gsub(/\?(, ?\?)+/, "?").gsub(/\/\*.+?\*\//, ""))
352
+ def insert_query_stats(query_stats, captured_at)
353
+ PgHero::QueryStats.transaction do
354
+ query_ids = add_queries(query_stats.map { |qs| qs[:query] })
355
+ values =
356
+ query_stats.map do |qs, query_id|
357
+ {
358
+ database: id,
359
+ user: qs[:user],
360
+ query_id: query_ids.fetch(qs[:query]),
361
+ query_hash: qs[:query_hash],
362
+ total_time: qs[:total_time],
363
+ calls: qs[:calls],
364
+ captured_at: captured_at
365
+ }
366
+ end
367
+ PgHero::QueryStats.insert_all!(values)
368
+ end
339
369
  end
340
370
 
341
- def insert_query_stats(db_id, db_query_stats, now)
342
- values =
343
- db_query_stats.map do |qs|
344
- {
345
- database: db_id,
346
- query: qs[:query],
347
- total_time: qs[:total_minutes] * 60 * 1000,
348
- calls: qs[:calls],
349
- captured_at: now,
350
- query_hash: supports_query_hash? ? qs[:query_hash] : nil,
351
- user: qs[:user]
352
- }
353
- end
354
- PgHero::QueryStats.insert_all!(values)
371
+ # duplicates are fine, but could use advisory lock to prevent them
372
+ def add_queries(queries)
373
+ queries = queries.uniq
374
+ query_ids = PgHero::Query.where(query: queries).to_h { |q| [q.query, q.id] }
375
+ new_queries = queries - query_ids.keys
376
+ new_ids = PgHero::Query.insert_all!(new_queries.map { |q| {query: q} }).rows.map(&:first)
377
+ new_queries.zip(new_ids) do |query, id|
378
+ query_ids[query] = id
379
+ end
380
+ query_ids
355
381
  end
356
382
  end
357
383
  end
@@ -11,17 +11,10 @@ module PgHero
11
11
  # https://www.postgresql.org/message-id/CADKbJJWz9M0swPT3oqe8f9+tfD4-F54uE6Xtkh4nERpVsQnjnw@mail.gmail.com
12
12
  def replication_lag
13
13
  with_feature_support(:replication_lag) do
14
- lag_condition =
15
- if server_version_num >= 100000
16
- "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
17
- else
18
- "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
19
- end
20
-
21
14
  select_one <<~SQL
22
15
  SELECT
23
16
  CASE
24
- WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
17
+ WHEN NOT pg_is_in_recovery() OR pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn() THEN 0
25
18
  ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
26
19
  END
27
20
  AS replication_lag
@@ -30,18 +23,14 @@ module PgHero
30
23
  end
31
24
 
32
25
  def replication_slots
33
- if server_version_num >= 90400
34
- with_feature_support(:replication_slots, []) do
35
- select_all <<~SQL
36
- SELECT
37
- slot_name,
38
- database,
39
- active
40
- FROM pg_replication_slots
41
- SQL
42
- end
43
- else
44
- []
26
+ with_feature_support(:replication_slots, []) do
27
+ select_all <<~SQL
28
+ SELECT
29
+ slot_name,
30
+ database,
31
+ active
32
+ FROM pg_replication_slots
33
+ SQL
45
34
  end
46
35
  end
47
36