pghero 1.2.2 → 1.2.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of pghero might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +11 -0
- data/CHANGELOG.md +6 -0
- data/README.md +1 -1
- data/app/controllers/pg_hero/home_controller.rb +15 -4
- data/app/views/layouts/pg_hero/application.html.erb +4 -4
- data/app/views/pg_hero/home/explain.html.erb +1 -1
- data/app/views/pg_hero/home/index_usage.html.erb +6 -1
- data/app/views/pg_hero/home/maintenance.html.erb +6 -1
- data/app/views/pg_hero/home/space.html.erb +5 -3
- data/lib/pghero.rb +35 -1243
- data/lib/pghero/connection.rb +5 -0
- data/lib/pghero/database.rb +12 -3
- data/lib/pghero/methods/basic.rb +104 -0
- data/lib/pghero/methods/connections.rb +49 -0
- data/lib/pghero/methods/databases.rb +39 -0
- data/lib/pghero/methods/explain.rb +29 -0
- data/lib/pghero/methods/indexes.rb +154 -0
- data/lib/pghero/methods/kill.rb +27 -0
- data/lib/pghero/methods/maintenance.rb +61 -0
- data/lib/pghero/methods/queries.rb +73 -0
- data/lib/pghero/methods/query_stats.rb +188 -0
- data/lib/pghero/methods/replica.rb +22 -0
- data/lib/pghero/methods/space.rb +30 -0
- data/lib/pghero/methods/suggested_indexes.rb +322 -0
- data/lib/pghero/methods/system.rb +70 -0
- data/lib/pghero/methods/tables.rb +68 -0
- data/lib/pghero/methods/users.rb +85 -0
- data/lib/pghero/query_stats.rb +7 -0
- data/lib/pghero/version.rb +1 -1
- data/lib/{pghero/tasks.rb → tasks/pghero.rake} +0 -2
- data/test/suggested_indexes_test.rb +3 -2
- data/test/test_helper.rb +1 -1
- metadata +22 -10
- data/test/gemfiles/activerecord31.gemfile +0 -6
- data/test/gemfiles/activerecord32.gemfile +0 -6
- data/test/gemfiles/activerecord40.gemfile +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63488be445a29436ffa0451acef24bc12e517f31
|
4
|
+
data.tar.gz: 31a1973c5f7324b119544ab9f54a34161238a0b4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7233f1a2a560983a00086c76349493a4433870e9f29394dbff41bfb39991bf589458777bad9b212fac3f654d1ed245953b6b6f7c38c9eb8569c7fdb9d7d52b6
|
7
|
+
data.tar.gz: 13c5da9a191fd4c2c7d3e450a07dea36e41f812c1abf67da1c04896dc495ba10675794f149da74627f05188008784a7b16f0dd615d8e2eab941d4038d1276453
|
data/.travis.yml
ADDED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -4,7 +4,7 @@ The missing dashboard for Postgres
|
|
4
4
|
|
5
5
|
[See it in action](https://pghero.herokuapp.com/)
|
6
6
|
|
7
|
-
[![Screenshot](https://pghero.herokuapp.com/assets/screenshot-
|
7
|
+
[![Screenshot](https://pghero.herokuapp.com/assets/screenshot-a54dead9c9bfc4c1176b184c5bd97ca1.png)](https://pghero.herokuapp.com/)
|
8
8
|
|
9
9
|
:speech_balloon: Get [handcrafted updates](http://chartkick.us7.list-manage.com/subscribe?u=952c861f99eb43084e0a49f98&id=6ea6541e8e&group[0][16]=true) for new features
|
10
10
|
|
@@ -6,8 +6,15 @@ module PgHero
|
|
6
6
|
|
7
7
|
http_basic_authenticate_with name: ENV["PGHERO_USERNAME"], password: ENV["PGHERO_PASSWORD"] if ENV["PGHERO_PASSWORD"]
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
if respond_to?(:before_action)
|
10
|
+
around_action :set_database
|
11
|
+
before_action :set_current_database
|
12
|
+
before_action :set_query_stats_enabled
|
13
|
+
else
|
14
|
+
around_filter :set_database
|
15
|
+
before_filter :set_current_database
|
16
|
+
before_filter :set_query_stats_enabled
|
17
|
+
end
|
11
18
|
|
12
19
|
def index
|
13
20
|
@title = "Overview"
|
@@ -56,7 +63,7 @@ module PgHero
|
|
56
63
|
def queries
|
57
64
|
@title = "Queries"
|
58
65
|
@historical_query_stats_enabled = PgHero.historical_query_stats_enabled?
|
59
|
-
@sort = %w
|
66
|
+
@sort = %w(average_time calls).include?(params[:sort]) ? params[:sort] : nil
|
60
67
|
@min_average_time = params[:min_average_time] ? params[:min_average_time].to_i : nil
|
61
68
|
@min_calls = params[:min_calls] ? params[:min_calls].to_i : nil
|
62
69
|
|
@@ -179,7 +186,7 @@ module PgHero
|
|
179
186
|
protected
|
180
187
|
|
181
188
|
def set_database
|
182
|
-
@databases = PgHero.
|
189
|
+
@databases = PgHero.databases.values
|
183
190
|
if params[:database]
|
184
191
|
PgHero.with(params[:database]) do
|
185
192
|
yield
|
@@ -191,6 +198,10 @@ module PgHero
|
|
191
198
|
end
|
192
199
|
end
|
193
200
|
|
201
|
+
def set_current_database
|
202
|
+
@current_database = PgHero.databases[PgHero.current_database]
|
203
|
+
end
|
204
|
+
|
194
205
|
def default_url_options
|
195
206
|
{database: params[:database]}
|
196
207
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
<!DOCTYPE html>
|
2
2
|
<html>
|
3
3
|
<head>
|
4
|
-
<title><%= [@databases.size > 1 ?
|
4
|
+
<title><%= [@databases.size > 1 ? @current_database.name : "PgHero", @title].compact.join(" / ") %></title>
|
5
5
|
|
6
6
|
<meta charset="utf-8" />
|
7
7
|
|
@@ -406,7 +406,7 @@
|
|
406
406
|
<div class="grid">
|
407
407
|
<div class="col-3-12">
|
408
408
|
<% if @databases.size > 1 %>
|
409
|
-
<p class="nav-header"><%=
|
409
|
+
<p class="nav-header"><%= @current_database.name %></p>
|
410
410
|
<% end %>
|
411
411
|
|
412
412
|
<ul class="nav">
|
@@ -430,8 +430,8 @@
|
|
430
430
|
<p class="nav-header">Databases</p>
|
431
431
|
<ul class="nav">
|
432
432
|
<% @databases.each do |database| %>
|
433
|
-
<li class="<%= ("active-database" if PgHero.current_database == database) %>">
|
434
|
-
<%= link_to database.
|
433
|
+
<li class="<%= ("active-database" if PgHero.current_database == database.id) %>">
|
434
|
+
<%= link_to database.name, database: database.id %>
|
435
435
|
</li>
|
436
436
|
<% end %>
|
437
437
|
</ul>
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<pre><%= @explanation %></pre>
|
11
11
|
<p><%= link_to "See how to interpret this", "http://www.postgresql.org/docs/current/static/using-explain.html", target: "_blank" %></p>
|
12
12
|
<% if (index = @suggested_index) %>
|
13
|
-
<%= render partial: "suggested_index", locals: {index: index} %>
|
13
|
+
<%= render partial: "suggested_index", locals: {index: index, details: index[:details]} %>
|
14
14
|
<% end %>
|
15
15
|
<% elsif @error %>
|
16
16
|
<div class="alert alert-danger"><%= @error %></div>
|
@@ -12,7 +12,12 @@
|
|
12
12
|
<tbody>
|
13
13
|
<% @index_usage.each do |query| %>
|
14
14
|
<tr>
|
15
|
-
<td
|
15
|
+
<td>
|
16
|
+
<%= query["table"] %>
|
17
|
+
<% if query["schema"] != "public" %>
|
18
|
+
<span class="text-muted"><%= query["schema"] %></span>
|
19
|
+
<% end %>
|
20
|
+
</td>
|
16
21
|
<td><%= query["percent_of_times_index_used"] %></td>
|
17
22
|
<td><%= number_with_delimiter(query["rows_in_table"]) %></td>
|
18
23
|
</tr>
|
@@ -12,7 +12,12 @@
|
|
12
12
|
<tbody>
|
13
13
|
<% @maintenance_info.each do |table| %>
|
14
14
|
<tr>
|
15
|
-
<td
|
15
|
+
<td>
|
16
|
+
<%= table["table"] %>
|
17
|
+
<% if table["schema"] != "public" %>
|
18
|
+
<span class="text-muted"><%= table["schema"] %></span>
|
19
|
+
<% end %>
|
20
|
+
</td>
|
16
21
|
<td>
|
17
22
|
<% time = [table["last_autovacuum"], table["last_vacuum"]].compact.max %>
|
18
23
|
<% if time %>
|
@@ -7,7 +7,8 @@
|
|
7
7
|
<thead>
|
8
8
|
<tr>
|
9
9
|
<th>Relation</th>
|
10
|
-
<th style="width:
|
10
|
+
<th style="width: 15%;"></th>
|
11
|
+
<th style="width: 15%;">Size</th>
|
11
12
|
</tr>
|
12
13
|
</thead>
|
13
14
|
<tbody>
|
@@ -15,10 +16,11 @@
|
|
15
16
|
<tr>
|
16
17
|
<td>
|
17
18
|
<%= query["name"] %>
|
18
|
-
<% if query["
|
19
|
-
<span class="text-muted"><%= query["
|
19
|
+
<% if query["schema"] != "public" %>
|
20
|
+
<span class="text-muted"><%= query["schema"] %></span>
|
20
21
|
<% end %>
|
21
22
|
</td>
|
23
|
+
<td> <span class="text-muted"><%= query["type"] == "index" ? "index" : "" %></span></td>
|
22
24
|
<td><%= query["size"] %></td>
|
23
25
|
</tr>
|
24
26
|
<% end %>
|
data/lib/pghero.rb
CHANGED
@@ -2,20 +2,29 @@ require "pghero/version"
|
|
2
2
|
require "active_record"
|
3
3
|
require "pghero/database"
|
4
4
|
require "pghero/engine" if defined?(Rails)
|
5
|
-
require "pghero/tasks"
|
6
5
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
6
|
+
# models
|
7
|
+
require "pghero/connection"
|
8
|
+
require "pghero/query_stats"
|
9
|
+
|
10
|
+
# methods
|
11
|
+
require "pghero/methods/basic"
|
12
|
+
require "pghero/methods/connections"
|
13
|
+
require "pghero/methods/databases"
|
14
|
+
require "pghero/methods/explain"
|
15
|
+
require "pghero/methods/indexes"
|
16
|
+
require "pghero/methods/kill"
|
17
|
+
require "pghero/methods/maintenance"
|
18
|
+
require "pghero/methods/queries"
|
19
|
+
require "pghero/methods/query_stats"
|
20
|
+
require "pghero/methods/replica"
|
21
|
+
require "pghero/methods/space"
|
22
|
+
require "pghero/methods/suggested_indexes"
|
23
|
+
require "pghero/methods/system"
|
24
|
+
require "pghero/methods/tables"
|
18
25
|
|
26
|
+
module PgHero
|
27
|
+
# settings
|
19
28
|
class << self
|
20
29
|
attr_accessor :long_running_query_sec, :slow_query_ms, :slow_query_calls, :total_connections_threshold, :cache_hit_rate_threshold, :env, :show_migrations
|
21
30
|
end
|
@@ -27,1235 +36,18 @@ module PgHero
|
|
27
36
|
self.env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
28
37
|
self.show_migrations = true
|
29
38
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
if File.exist?(path)
|
45
|
-
YAML.load(ERB.new(File.read(path)).result)[env]
|
46
|
-
end
|
47
|
-
|
48
|
-
if config
|
49
|
-
config
|
50
|
-
else
|
51
|
-
{
|
52
|
-
"databases" => {
|
53
|
-
"primary" => {
|
54
|
-
"url" => ENV["PGHERO_DATABASE_URL"] || ActiveRecord::Base.connection_config,
|
55
|
-
"db_instance_identifier" => ENV["PGHERO_DB_INSTANCE_IDENTIFIER"]
|
56
|
-
}
|
57
|
-
}
|
58
|
-
}
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def databases
|
64
|
-
@databases ||= begin
|
65
|
-
Hash[
|
66
|
-
config["databases"].map do |id, c|
|
67
|
-
[id, PgHero::Database.new(id, c)]
|
68
|
-
end
|
69
|
-
]
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
def primary_database
|
74
|
-
databases.keys.first
|
75
|
-
end
|
76
|
-
|
77
|
-
def current_database
|
78
|
-
Thread.current[:pghero_current_database] ||= primary_database
|
79
|
-
end
|
80
|
-
|
81
|
-
def current_database=(database)
|
82
|
-
raise "Database not found" unless databases[database]
|
83
|
-
Thread.current[:pghero_current_database] = database.to_s
|
84
|
-
database
|
85
|
-
end
|
86
|
-
|
87
|
-
def with(database)
|
88
|
-
previous_database = current_database
|
89
|
-
begin
|
90
|
-
self.current_database = database
|
91
|
-
yield
|
92
|
-
ensure
|
93
|
-
self.current_database = previous_database
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
def running_queries
|
98
|
-
select_all <<-SQL
|
99
|
-
SELECT
|
100
|
-
pid,
|
101
|
-
state,
|
102
|
-
application_name AS source,
|
103
|
-
age(now(), xact_start) AS duration,
|
104
|
-
waiting,
|
105
|
-
query,
|
106
|
-
xact_start AS started_at
|
107
|
-
FROM
|
108
|
-
pg_stat_activity
|
109
|
-
WHERE
|
110
|
-
query <> '<insufficient privilege>'
|
111
|
-
AND state <> 'idle'
|
112
|
-
AND pid <> pg_backend_pid()
|
113
|
-
ORDER BY
|
114
|
-
query_start DESC
|
115
|
-
SQL
|
116
|
-
end
|
117
|
-
|
118
|
-
def long_running_queries
|
119
|
-
select_all <<-SQL
|
120
|
-
SELECT
|
121
|
-
pid,
|
122
|
-
state,
|
123
|
-
application_name AS source,
|
124
|
-
age(now(), xact_start) AS duration,
|
125
|
-
waiting,
|
126
|
-
query,
|
127
|
-
xact_start AS started_at
|
128
|
-
FROM
|
129
|
-
pg_stat_activity
|
130
|
-
WHERE
|
131
|
-
query <> '<insufficient privilege>'
|
132
|
-
AND state <> 'idle'
|
133
|
-
AND pid <> pg_backend_pid()
|
134
|
-
AND now() - query_start > interval '#{long_running_query_sec.to_i} seconds'
|
135
|
-
ORDER BY
|
136
|
-
query_start DESC
|
137
|
-
SQL
|
138
|
-
end
|
139
|
-
|
140
|
-
def locks
|
141
|
-
select_all <<-SQL
|
142
|
-
SELECT DISTINCT ON (pid)
|
143
|
-
pg_stat_activity.pid,
|
144
|
-
pg_stat_activity.query,
|
145
|
-
age(now(), pg_stat_activity.query_start) AS age
|
146
|
-
FROM
|
147
|
-
pg_stat_activity
|
148
|
-
INNER JOIN
|
149
|
-
pg_locks ON pg_locks.pid = pg_stat_activity.pid
|
150
|
-
WHERE
|
151
|
-
pg_stat_activity.query <> '<insufficient privilege>'
|
152
|
-
AND pg_locks.mode = 'ExclusiveLock'
|
153
|
-
AND pg_stat_activity.pid <> pg_backend_pid()
|
154
|
-
ORDER BY
|
155
|
-
pid,
|
156
|
-
query_start
|
157
|
-
SQL
|
158
|
-
end
|
159
|
-
|
160
|
-
def index_hit_rate
|
161
|
-
select_all(<<-SQL
|
162
|
-
SELECT
|
163
|
-
(sum(idx_blks_hit)) / nullif(sum(idx_blks_hit + idx_blks_read), 0) AS rate
|
164
|
-
FROM
|
165
|
-
pg_statio_user_indexes
|
166
|
-
SQL
|
167
|
-
).first["rate"].to_f
|
168
|
-
end
|
169
|
-
|
170
|
-
def table_hit_rate
|
171
|
-
select_all(<<-SQL
|
172
|
-
SELECT
|
173
|
-
sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS rate
|
174
|
-
FROM
|
175
|
-
pg_statio_user_tables
|
176
|
-
SQL
|
177
|
-
).first["rate"].to_f
|
178
|
-
end
|
179
|
-
|
180
|
-
def table_caching
|
181
|
-
select_all <<-SQL
|
182
|
-
SELECT
|
183
|
-
relname AS table,
|
184
|
-
CASE WHEN heap_blks_hit + heap_blks_read = 0 THEN
|
185
|
-
0
|
186
|
-
ELSE
|
187
|
-
ROUND(1.0 * heap_blks_hit / (heap_blks_hit + heap_blks_read), 2)
|
188
|
-
END AS hit_rate
|
189
|
-
FROM
|
190
|
-
pg_statio_user_tables
|
191
|
-
ORDER BY
|
192
|
-
2 DESC, 1
|
193
|
-
SQL
|
194
|
-
end
|
195
|
-
|
196
|
-
def index_caching
|
197
|
-
select_all <<-SQL
|
198
|
-
SELECT
|
199
|
-
indexrelname AS index,
|
200
|
-
relname AS table,
|
201
|
-
CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN
|
202
|
-
0
|
203
|
-
ELSE
|
204
|
-
ROUND(1.0 * idx_blks_hit / (idx_blks_hit + idx_blks_read), 2)
|
205
|
-
END AS hit_rate
|
206
|
-
FROM
|
207
|
-
pg_statio_user_indexes
|
208
|
-
ORDER BY
|
209
|
-
3 DESC, 1
|
210
|
-
SQL
|
211
|
-
end
|
212
|
-
|
213
|
-
def index_usage
|
214
|
-
select_all <<-SQL
|
215
|
-
SELECT
|
216
|
-
relname AS table,
|
217
|
-
CASE idx_scan
|
218
|
-
WHEN 0 THEN 'Insufficient data'
|
219
|
-
ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
|
220
|
-
END percent_of_times_index_used,
|
221
|
-
n_live_tup rows_in_table
|
222
|
-
FROM
|
223
|
-
pg_stat_user_tables
|
224
|
-
ORDER BY
|
225
|
-
n_live_tup DESC,
|
226
|
-
relname ASC
|
227
|
-
SQL
|
228
|
-
end
|
229
|
-
|
230
|
-
def missing_indexes
|
231
|
-
select_all <<-SQL
|
232
|
-
SELECT
|
233
|
-
relname AS table,
|
234
|
-
CASE idx_scan
|
235
|
-
WHEN 0 THEN 'Insufficient data'
|
236
|
-
ELSE (100 * idx_scan / (seq_scan + idx_scan))::text
|
237
|
-
END percent_of_times_index_used,
|
238
|
-
n_live_tup rows_in_table
|
239
|
-
FROM
|
240
|
-
pg_stat_user_tables
|
241
|
-
WHERE
|
242
|
-
idx_scan > 0
|
243
|
-
AND (100 * idx_scan / (seq_scan + idx_scan)) < 95
|
244
|
-
AND n_live_tup >= 10000
|
245
|
-
ORDER BY
|
246
|
-
n_live_tup DESC,
|
247
|
-
relname ASC
|
248
|
-
SQL
|
249
|
-
end
|
250
|
-
|
251
|
-
def unused_tables
|
252
|
-
select_all <<-SQL
|
253
|
-
SELECT
|
254
|
-
relname AS table,
|
255
|
-
n_live_tup rows_in_table
|
256
|
-
FROM
|
257
|
-
pg_stat_user_tables
|
258
|
-
WHERE
|
259
|
-
idx_scan = 0
|
260
|
-
ORDER BY
|
261
|
-
n_live_tup DESC,
|
262
|
-
relname ASC
|
263
|
-
SQL
|
264
|
-
end
|
265
|
-
|
266
|
-
def unused_indexes
|
267
|
-
select_all <<-SQL
|
268
|
-
SELECT
|
269
|
-
relname AS table,
|
270
|
-
indexrelname AS index,
|
271
|
-
pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size,
|
272
|
-
idx_scan as index_scans
|
273
|
-
FROM
|
274
|
-
pg_stat_user_indexes ui
|
275
|
-
INNER JOIN
|
276
|
-
pg_index i ON ui.indexrelid = i.indexrelid
|
277
|
-
WHERE
|
278
|
-
NOT indisunique
|
279
|
-
AND idx_scan < 50
|
280
|
-
ORDER BY
|
281
|
-
pg_relation_size(i.indexrelid) DESC,
|
282
|
-
relname ASC
|
283
|
-
SQL
|
284
|
-
end
|
285
|
-
|
286
|
-
def invalid_indexes
|
287
|
-
select_all <<-SQL
|
288
|
-
SELECT
|
289
|
-
c.relname AS index
|
290
|
-
FROM
|
291
|
-
pg_catalog.pg_class c,
|
292
|
-
pg_catalog.pg_namespace n,
|
293
|
-
pg_catalog.pg_index i
|
294
|
-
WHERE
|
295
|
-
i.indisvalid = false
|
296
|
-
AND i.indexrelid = c.oid
|
297
|
-
AND c.relnamespace = n.oid
|
298
|
-
AND n.nspname != 'pg_catalog'
|
299
|
-
AND n.nspname != 'information_schema'
|
300
|
-
AND n.nspname != 'pg_toast'
|
301
|
-
ORDER BY
|
302
|
-
c.relname
|
303
|
-
SQL
|
304
|
-
end
|
305
|
-
|
306
|
-
def relation_sizes
|
307
|
-
select_all <<-SQL
|
308
|
-
SELECT
|
309
|
-
c.relname AS name,
|
310
|
-
CASE WHEN c.relkind = 'r' THEN 'table' ELSE 'index' END AS type,
|
311
|
-
pg_size_pretty(pg_table_size(c.oid)) AS size
|
312
|
-
FROM
|
313
|
-
pg_class c
|
314
|
-
LEFT JOIN
|
315
|
-
pg_namespace n ON (n.oid = c.relnamespace)
|
316
|
-
WHERE
|
317
|
-
n.nspname NOT IN ('pg_catalog', 'information_schema')
|
318
|
-
AND n.nspname !~ '^pg_toast'
|
319
|
-
AND c.relkind IN ('r', 'i')
|
320
|
-
ORDER BY
|
321
|
-
pg_table_size(c.oid) DESC,
|
322
|
-
name ASC
|
323
|
-
SQL
|
324
|
-
end
|
325
|
-
|
326
|
-
def database_size
|
327
|
-
select_all("SELECT pg_size_pretty(pg_database_size(current_database()))").first["pg_size_pretty"]
|
328
|
-
end
|
329
|
-
|
330
|
-
def total_connections
|
331
|
-
select_all("SELECT COUNT(*) FROM pg_stat_activity WHERE pid <> pg_backend_pid()").first["count"].to_i
|
332
|
-
end
|
333
|
-
|
334
|
-
def connection_sources(options = {})
|
335
|
-
if options[:by_database]
|
336
|
-
select_all <<-SQL
|
337
|
-
SELECT
|
338
|
-
application_name AS source,
|
339
|
-
client_addr AS ip,
|
340
|
-
datname AS database,
|
341
|
-
COUNT(*) AS total_connections
|
342
|
-
FROM
|
343
|
-
pg_stat_activity
|
344
|
-
WHERE
|
345
|
-
pid <> pg_backend_pid()
|
346
|
-
GROUP BY
|
347
|
-
1, 2, 3
|
348
|
-
ORDER BY
|
349
|
-
COUNT(*) DESC,
|
350
|
-
application_name ASC,
|
351
|
-
client_addr ASC
|
352
|
-
SQL
|
353
|
-
else
|
354
|
-
select_all <<-SQL
|
355
|
-
SELECT
|
356
|
-
application_name AS source,
|
357
|
-
client_addr AS ip,
|
358
|
-
COUNT(*) AS total_connections
|
359
|
-
FROM
|
360
|
-
pg_stat_activity
|
361
|
-
WHERE
|
362
|
-
pid <> pg_backend_pid()
|
363
|
-
GROUP BY
|
364
|
-
application_name,
|
365
|
-
ip
|
366
|
-
ORDER BY
|
367
|
-
COUNT(*) DESC,
|
368
|
-
application_name ASC,
|
369
|
-
client_addr ASC
|
370
|
-
SQL
|
371
|
-
end
|
372
|
-
end
|
373
|
-
|
374
|
-
# http://www.postgresql.org/docs/9.1/static/routine-vacuuming.html#VACUUM-FOR-WRAPAROUND
|
375
|
-
# "the system will shut down and refuse to start any new transactions
|
376
|
-
# once there are fewer than 1 million transactions left until wraparound"
|
377
|
-
# warn when 10,000,000 transactions left
|
378
|
-
def transaction_id_danger(options = {})
|
379
|
-
threshold = options[:threshold] || 10000000
|
380
|
-
select_all <<-SQL
|
381
|
-
SELECT
|
382
|
-
c.oid::regclass::text AS table,
|
383
|
-
2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_shutdown
|
384
|
-
FROM
|
385
|
-
pg_class c
|
386
|
-
LEFT JOIN
|
387
|
-
pg_class t ON c.reltoastrelid = t.oid
|
388
|
-
WHERE
|
389
|
-
c.relkind = 'r'
|
390
|
-
AND (2146483648 - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid))) < #{threshold}
|
391
|
-
ORDER BY
|
392
|
-
2, 1
|
393
|
-
SQL
|
394
|
-
end
|
395
|
-
|
396
|
-
def autovacuum_danger
|
397
|
-
select_all <<-SQL
|
398
|
-
SELECT
|
399
|
-
c.oid::regclass::text as table,
|
400
|
-
(SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int -
|
401
|
-
GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) AS transactions_before_autovacuum
|
402
|
-
FROM
|
403
|
-
pg_class c
|
404
|
-
LEFT JOIN
|
405
|
-
pg_class t ON c.reltoastrelid = t.oid
|
406
|
-
WHERE
|
407
|
-
c.relkind = 'r'
|
408
|
-
AND (SELECT setting FROM pg_settings WHERE name = 'autovacuum_freeze_max_age')::int - GREATEST(AGE(c.relfrozenxid), AGE(t.relfrozenxid)) < 2000000
|
409
|
-
ORDER BY
|
410
|
-
transactions_before_autovacuum
|
411
|
-
SQL
|
412
|
-
end
|
413
|
-
|
414
|
-
def maintenance_info
|
415
|
-
select_all <<-SQL
|
416
|
-
SELECT
|
417
|
-
relname AS table,
|
418
|
-
last_vacuum,
|
419
|
-
last_autovacuum,
|
420
|
-
last_analyze,
|
421
|
-
last_autoanalyze
|
422
|
-
FROM
|
423
|
-
pg_stat_user_tables
|
424
|
-
ORDER BY
|
425
|
-
relname ASC
|
426
|
-
SQL
|
427
|
-
end
|
428
|
-
|
429
|
-
def kill(pid)
|
430
|
-
execute("SELECT pg_terminate_backend(#{pid.to_i})").first["pg_terminate_backend"] == "t"
|
431
|
-
end
|
432
|
-
|
433
|
-
def kill_long_running_queries
|
434
|
-
long_running_queries.each { |query| kill(query["pid"]) }
|
435
|
-
true
|
436
|
-
end
|
437
|
-
|
438
|
-
def kill_all
|
439
|
-
select_all <<-SQL
|
440
|
-
SELECT
|
441
|
-
pg_terminate_backend(pid)
|
442
|
-
FROM
|
443
|
-
pg_stat_activity
|
444
|
-
WHERE
|
445
|
-
pid <> pg_backend_pid()
|
446
|
-
AND query <> '<insufficient privilege>'
|
447
|
-
SQL
|
448
|
-
true
|
449
|
-
end
|
450
|
-
|
451
|
-
def query_stats(options = {})
|
452
|
-
current_query_stats = (options[:historical] && options[:end_at] && options[:end_at] < Time.now ? [] : current_query_stats(options)).index_by { |q| q["query"] }
|
453
|
-
historical_query_stats = (options[:historical] ? historical_query_stats(options) : []).index_by { |q| q["query"] }
|
454
|
-
current_query_stats.default = {}
|
455
|
-
historical_query_stats.default = {}
|
456
|
-
|
457
|
-
query_stats = []
|
458
|
-
(current_query_stats.keys + historical_query_stats.keys).uniq.each do |query|
|
459
|
-
value = {
|
460
|
-
"query" => query,
|
461
|
-
"total_minutes" => current_query_stats[query]["total_minutes"].to_f + historical_query_stats[query]["total_minutes"].to_f,
|
462
|
-
"calls" => current_query_stats[query]["calls"].to_i + historical_query_stats[query]["calls"].to_i
|
463
|
-
}
|
464
|
-
value["average_time"] = value["total_minutes"] * 1000 * 60 / value["calls"]
|
465
|
-
value["total_percent"] = value["total_minutes"] * 100.0 / (current_query_stats[query]["all_queries_total_minutes"].to_f + historical_query_stats[query]["all_queries_total_minutes"].to_f)
|
466
|
-
query_stats << value
|
467
|
-
end
|
468
|
-
sort = options[:sort] || "total_minutes"
|
469
|
-
query_stats = query_stats.sort_by { |q| -q[sort] }.first(100)
|
470
|
-
if options[:min_average_time]
|
471
|
-
query_stats.reject! { |q| q["average_time"].to_f < options[:min_average_time] }
|
472
|
-
end
|
473
|
-
if options[:min_calls]
|
474
|
-
query_stats.reject! { |q| q["calls"].to_i < options[:min_calls] }
|
475
|
-
end
|
476
|
-
query_stats
|
477
|
-
end
|
478
|
-
|
479
|
-
def slow_queries(options = {})
|
480
|
-
query_stats = options[:query_stats] || self.query_stats(options.except(:query_stats))
|
481
|
-
query_stats.select { |q| q["calls"].to_i >= slow_query_calls.to_i && q["average_time"].to_i >= slow_query_ms.to_i }
|
482
|
-
end
|
483
|
-
|
484
|
-
def query_stats_available?
|
485
|
-
select_all("SELECT COUNT(*) AS count FROM pg_available_extensions WHERE name = 'pg_stat_statements'").first["count"].to_i > 0
|
486
|
-
end
|
487
|
-
|
488
|
-
def query_stats_enabled?
|
489
|
-
select_all("SELECT COUNT(*) AS count FROM pg_extension WHERE extname = 'pg_stat_statements'").first["count"].to_i > 0 && query_stats_readable?
|
490
|
-
end
|
491
|
-
|
492
|
-
def query_stats_readable?
|
493
|
-
select_all("SELECT has_table_privilege(current_user, 'pg_stat_statements', 'SELECT')").first["has_table_privilege"] == "t"
|
494
|
-
rescue ActiveRecord::StatementInvalid
|
495
|
-
false
|
496
|
-
end
|
497
|
-
|
498
|
-
def enable_query_stats
|
499
|
-
execute("CREATE EXTENSION pg_stat_statements")
|
500
|
-
end
|
501
|
-
|
502
|
-
def disable_query_stats
|
503
|
-
execute("DROP EXTENSION IF EXISTS pg_stat_statements")
|
504
|
-
true
|
505
|
-
end
|
506
|
-
|
507
|
-
def reset_query_stats
|
508
|
-
if query_stats_enabled?
|
509
|
-
execute("SELECT pg_stat_statements_reset()")
|
510
|
-
true
|
511
|
-
else
|
512
|
-
false
|
513
|
-
end
|
514
|
-
end
|
515
|
-
|
516
|
-
def capture_query_stats
|
517
|
-
config["databases"].keys.each do |database|
|
518
|
-
with(database) do
|
519
|
-
now = Time.now
|
520
|
-
query_stats = self.query_stats(limit: 1000000)
|
521
|
-
if query_stats.any? && reset_query_stats
|
522
|
-
values =
|
523
|
-
query_stats.map do |qs|
|
524
|
-
[
|
525
|
-
database,
|
526
|
-
qs["query"],
|
527
|
-
qs["total_minutes"].to_f * 60 * 1000,
|
528
|
-
qs["calls"],
|
529
|
-
now
|
530
|
-
].map { |v| quote(v) }.join(",")
|
531
|
-
end.map { |v| "(#{v})" }.join(",")
|
532
|
-
|
533
|
-
stats_connection.execute("INSERT INTO pghero_query_stats (database, query, total_time, calls, captured_at) VALUES #{values}")
|
534
|
-
end
|
535
|
-
end
|
536
|
-
end
|
537
|
-
end
|
538
|
-
|
539
|
-
# http://stackoverflow.com/questions/20582500/how-to-check-if-a-table-exists-in-a-given-schema
|
540
|
-
def historical_query_stats_enabled?
|
541
|
-
# TODO use schema from config
|
542
|
-
stats_connection.select_all( squish <<-SQL
|
543
|
-
SELECT EXISTS (
|
544
|
-
SELECT
|
545
|
-
1
|
546
|
-
FROM
|
547
|
-
pg_catalog.pg_class c
|
548
|
-
INNER JOIN
|
549
|
-
pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
550
|
-
WHERE
|
551
|
-
n.nspname = 'public'
|
552
|
-
AND c.relname = 'pghero_query_stats'
|
553
|
-
AND c.relkind = 'r'
|
554
|
-
)
|
555
|
-
SQL
|
556
|
-
).to_a.first["exists"] == "t"
|
557
|
-
end
|
558
|
-
|
559
|
-
def stats_connection
|
560
|
-
QueryStats.connection
|
561
|
-
end
|
562
|
-
|
563
|
-
def ssl_used?
|
564
|
-
ssl_used = nil
|
565
|
-
connection_model.transaction do
|
566
|
-
execute("CREATE EXTENSION IF NOT EXISTS sslinfo")
|
567
|
-
ssl_used = select_all("SELECT ssl_is_used()").first["ssl_is_used"] == "t"
|
568
|
-
raise ActiveRecord::Rollback
|
569
|
-
end
|
570
|
-
ssl_used
|
571
|
-
end
|
572
|
-
|
573
|
-
def cpu_usage
|
574
|
-
rds_stats("CPUUtilization")
|
575
|
-
end
|
576
|
-
|
577
|
-
def connection_stats
|
578
|
-
rds_stats("DatabaseConnections")
|
579
|
-
end
|
580
|
-
|
581
|
-
def replication_lag_stats
|
582
|
-
rds_stats("ReplicaLag")
|
583
|
-
end
|
584
|
-
|
585
|
-
def read_iops_stats
|
586
|
-
rds_stats("ReadIOPS")
|
587
|
-
end
|
588
|
-
|
589
|
-
def write_iops_stats
|
590
|
-
rds_stats("WriteIOPS")
|
591
|
-
end
|
592
|
-
|
593
|
-
def rds_stats(metric_name)
|
594
|
-
if system_stats_enabled?
|
595
|
-
client =
|
596
|
-
if defined?(Aws)
|
597
|
-
Aws::CloudWatch::Client.new(access_key_id: access_key_id, secret_access_key: secret_access_key)
|
598
|
-
else
|
599
|
-
AWS::CloudWatch.new(access_key_id: access_key_id, secret_access_key: secret_access_key).client
|
600
|
-
end
|
601
|
-
|
602
|
-
now = Time.now
|
603
|
-
resp = client.get_metric_statistics(
|
604
|
-
namespace: "AWS/RDS",
|
605
|
-
metric_name: metric_name,
|
606
|
-
dimensions: [{name: "DBInstanceIdentifier", value: db_instance_identifier}],
|
607
|
-
start_time: (now - 1 * 3600).iso8601,
|
608
|
-
end_time: now.iso8601,
|
609
|
-
period: 60,
|
610
|
-
statistics: ["Average"]
|
611
|
-
)
|
612
|
-
data = {}
|
613
|
-
resp[:datapoints].sort_by { |d| d[:timestamp] }.each do |d|
|
614
|
-
data[d[:timestamp]] = d[:average]
|
615
|
-
end
|
616
|
-
data
|
617
|
-
else
|
618
|
-
{}
|
619
|
-
end
|
620
|
-
end
|
621
|
-
|
622
|
-
def system_stats_enabled?
|
623
|
-
!!((defined?(Aws) || defined?(AWS)) && access_key_id && secret_access_key && db_instance_identifier)
|
624
|
-
end
|
625
|
-
|
626
|
-
def random_password
|
627
|
-
require "securerandom"
|
628
|
-
SecureRandom.base64(40).delete("+/=")[0...24]
|
629
|
-
end
|
630
|
-
|
631
|
-
def create_user(user, options = {})
|
632
|
-
password = options[:password] || random_password
|
633
|
-
schema = options[:schema] || "public"
|
634
|
-
database = options[:database] || connection_model.connection_config[:database]
|
635
|
-
|
636
|
-
commands =
|
637
|
-
[
|
638
|
-
"CREATE ROLE #{user} LOGIN PASSWORD #{quote(password)}",
|
639
|
-
"GRANT CONNECT ON DATABASE #{database} TO #{user}",
|
640
|
-
"GRANT USAGE ON SCHEMA #{schema} TO #{user}"
|
641
|
-
]
|
642
|
-
if options[:readonly]
|
643
|
-
if options[:tables]
|
644
|
-
commands.concat table_grant_commands("SELECT", options[:tables], user)
|
645
|
-
else
|
646
|
-
commands << "GRANT SELECT ON ALL TABLES IN SCHEMA #{schema} TO #{user}"
|
647
|
-
commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT SELECT ON TABLES TO #{user}"
|
648
|
-
end
|
649
|
-
else
|
650
|
-
if options[:tables]
|
651
|
-
commands.concat table_grant_commands("ALL PRIVILEGES", options[:tables], user)
|
652
|
-
else
|
653
|
-
commands << "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA #{schema} TO #{user}"
|
654
|
-
commands << "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA #{schema} TO #{user}"
|
655
|
-
commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT ALL PRIVILEGES ON TABLES TO #{user}"
|
656
|
-
commands << "ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} GRANT ALL PRIVILEGES ON SEQUENCES TO #{user}"
|
657
|
-
end
|
658
|
-
end
|
659
|
-
|
660
|
-
# run commands
|
661
|
-
connection_model.transaction do
|
662
|
-
commands.each do |command|
|
663
|
-
execute command
|
664
|
-
end
|
665
|
-
end
|
666
|
-
|
667
|
-
{password: password}
|
668
|
-
end
|
669
|
-
|
670
|
-
def drop_user(user, options = {})
|
671
|
-
schema = options[:schema] || "public"
|
672
|
-
database = options[:database] || connection_model.connection_config[:database]
|
673
|
-
|
674
|
-
# thanks shiftb
|
675
|
-
commands =
|
676
|
-
[
|
677
|
-
"REVOKE CONNECT ON DATABASE #{database} FROM #{user}",
|
678
|
-
"REVOKE USAGE ON SCHEMA #{schema} FROM #{user}",
|
679
|
-
"REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA #{schema} FROM #{user}",
|
680
|
-
"REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA #{schema} FROM #{user}",
|
681
|
-
"ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE SELECT ON TABLES FROM #{user}",
|
682
|
-
"ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE SELECT ON SEQUENCES FROM #{user}",
|
683
|
-
"ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE ALL ON SEQUENCES FROM #{user}",
|
684
|
-
"ALTER DEFAULT PRIVILEGES IN SCHEMA #{schema} REVOKE ALL ON TABLES FROM #{user}",
|
685
|
-
"DROP ROLE #{user}"
|
686
|
-
]
|
687
|
-
|
688
|
-
# run commands
|
689
|
-
connection_model.transaction do
|
690
|
-
commands.each do |command|
|
691
|
-
execute command
|
692
|
-
end
|
693
|
-
end
|
694
|
-
|
695
|
-
true
|
696
|
-
end
|
697
|
-
|
698
|
-
def access_key_id
|
699
|
-
ENV["PGHERO_ACCESS_KEY_ID"] || ENV["AWS_ACCESS_KEY_ID"]
|
700
|
-
end
|
701
|
-
|
702
|
-
def secret_access_key
|
703
|
-
ENV["PGHERO_SECRET_ACCESS_KEY"] || ENV["AWS_SECRET_ACCESS_KEY"]
|
704
|
-
end
|
705
|
-
|
706
|
-
def db_instance_identifier
|
707
|
-
databases[current_database].db_instance_identifier
|
708
|
-
end
|
709
|
-
|
710
|
-
def explain(sql)
|
711
|
-
sql = squish(sql)
|
712
|
-
explanation = nil
|
713
|
-
explain_safe = explain_safe?
|
714
|
-
|
715
|
-
# use transaction for safety
|
716
|
-
connection_model.transaction do
|
717
|
-
if !explain_safe && (sql.sub(/;\z/, "").include?(";") || sql.upcase.include?("COMMIT"))
|
718
|
-
raise ActiveRecord::StatementInvalid, "Unsafe statement"
|
719
|
-
end
|
720
|
-
explanation = select_all("EXPLAIN #{sql}").map { |v| v["QUERY PLAN"] }.join("\n")
|
721
|
-
raise ActiveRecord::Rollback
|
722
|
-
end
|
723
|
-
|
724
|
-
explanation
|
725
|
-
end
|
726
|
-
|
727
|
-
def explain_safe?
|
728
|
-
select_all("SELECT 1; SELECT 1")
|
729
|
-
false
|
730
|
-
rescue ActiveRecord::StatementInvalid
|
731
|
-
true
|
732
|
-
end
|
733
|
-
|
734
|
-
def settings
|
735
|
-
names = %w(
|
736
|
-
max_connections shared_buffers effective_cache_size work_mem
|
737
|
-
maintenance_work_mem checkpoint_segments checkpoint_completion_target
|
738
|
-
wal_buffers default_statistics_target
|
739
|
-
)
|
740
|
-
values = Hash[select_all(connection_model.send(:sanitize_sql_array, ["SELECT name, setting, unit FROM pg_settings WHERE name IN (?)", names])).sort_by { |row| names.index(row["name"]) }.map { |row| [row["name"], friendly_value(row["setting"], row["unit"])] }]
|
741
|
-
Hash[names.map { |name| [name, values[name]] }]
|
742
|
-
end
|
743
|
-
|
744
|
-
def replica?
|
745
|
-
select_all("SELECT setting FROM pg_settings WHERE name = 'hot_standby'").first["setting"] == "on"
|
746
|
-
end
|
747
|
-
|
748
|
-
# http://www.niwi.be/2013/02/16/replication-status-in-postgresql/
|
749
|
-
def replication_lag
|
750
|
-
select_all("SELECT EXTRACT(EPOCH FROM NOW() - pg_last_xact_replay_timestamp()) AS replication_lag").first["replication_lag"].to_f
|
751
|
-
end
|
752
|
-
|
753
|
-
# TODO parse array properly
|
754
|
-
# http://stackoverflow.com/questions/2204058/list-columns-with-indexes-in-postgresql
|
755
|
-
def indexes
|
756
|
-
select_all( <<-SQL
|
757
|
-
SELECT
|
758
|
-
t.relname AS table,
|
759
|
-
ix.relname AS name,
|
760
|
-
regexp_replace(pg_get_indexdef(indexrelid), '^[^\\(]*\\((.*)\\)$', '\\1') AS columns,
|
761
|
-
regexp_replace(pg_get_indexdef(indexrelid), '.* USING ([^ ]*) \\(.*', '\\1') AS using,
|
762
|
-
indisunique AS unique,
|
763
|
-
indisprimary AS primary,
|
764
|
-
indisvalid AS valid,
|
765
|
-
indexprs::text,
|
766
|
-
indpred::text,
|
767
|
-
pg_get_indexdef(indexrelid) AS definition
|
768
|
-
FROM
|
769
|
-
pg_index i
|
770
|
-
INNER JOIN
|
771
|
-
pg_class t ON t.oid = i.indrelid
|
772
|
-
INNER JOIN
|
773
|
-
pg_class ix ON ix.oid = i.indexrelid
|
774
|
-
ORDER BY
|
775
|
-
1, 2
|
776
|
-
SQL
|
777
|
-
).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", "); v }
|
778
|
-
end
|
779
|
-
|
780
|
-
def duplicate_indexes
|
781
|
-
indexes = []
|
782
|
-
|
783
|
-
indexes_by_table = self.indexes.group_by { |i| i["table"] }
|
784
|
-
indexes_by_table.values.flatten.select { |i| i["primary"] == "f" && i["unique"] == "f" && !i["indexprs"] && !i["indpred"] && i["valid"] == "t" }.each do |index|
|
785
|
-
covering_index = indexes_by_table[index["table"]].find { |i| index_covers?(i["columns"], index["columns"]) && i["using"] == index["using"] && i["name"] != index["name"] && i["valid"] == "t" }
|
786
|
-
if covering_index
|
787
|
-
indexes << {"unneeded_index" => index, "covering_index" => covering_index}
|
788
|
-
end
|
789
|
-
end
|
790
|
-
|
791
|
-
indexes.sort_by { |i| ui = i["unneeded_index"]; [ui["table"], ui["columns"]] }
|
792
|
-
end
|
793
|
-
|
794
|
-
def suggested_indexes_enabled?
|
795
|
-
defined?(PgQuery) && query_stats_enabled?
|
796
|
-
end
|
797
|
-
|
798
|
-
# TODO clean this mess
|
799
|
-
def suggested_indexes_by_query(options = {})
|
800
|
-
best_indexes = {}
|
801
|
-
|
802
|
-
if suggested_indexes_enabled?
|
803
|
-
# get most time-consuming queries
|
804
|
-
queries = options[:queries] || (options[:query_stats] || self.query_stats(historical: true, start_at: 24.hours.ago)).map { |qs| qs["query"] }
|
805
|
-
|
806
|
-
# get best indexes for queries
|
807
|
-
best_indexes = best_index_helper(queries)
|
808
|
-
|
809
|
-
if best_indexes.any?
|
810
|
-
existing_columns = Hash.new { |hash, key| hash[key] = Hash.new { |hash2, key2| hash2[key2] = [] } }
|
811
|
-
indexes = self.indexes
|
812
|
-
indexes.group_by { |g| g["using"] }.each do |group, inds|
|
813
|
-
inds.each do |i|
|
814
|
-
existing_columns[group][i["table"]] << i["columns"]
|
815
|
-
end
|
816
|
-
end
|
817
|
-
indexes_by_table = indexes.group_by { |i| i["table"] }
|
818
|
-
|
819
|
-
best_indexes.each do |query, best_index|
|
820
|
-
if best_index[:found]
|
821
|
-
index = best_index[:index]
|
822
|
-
best_index[:table_indexes] = indexes_by_table[index[:table]].to_a
|
823
|
-
covering_index = existing_columns[index[:using] || "btree"][index[:table]].find { |e| index_covers?(e, index[:columns]) }
|
824
|
-
if covering_index
|
825
|
-
best_index[:covering_index] = covering_index
|
826
|
-
best_index[:explanation] = "Covered by index on (#{covering_index.join(", ")})"
|
827
|
-
end
|
828
|
-
end
|
829
|
-
end
|
830
|
-
end
|
831
|
-
end
|
832
|
-
|
833
|
-
best_indexes
|
834
|
-
end
|
835
|
-
|
836
|
-
def suggested_indexes(options = {})
|
837
|
-
indexes = []
|
838
|
-
|
839
|
-
(options[:suggested_indexes_by_query] || suggested_indexes_by_query(options)).select { |s, i| i[:found] && !i[:covering_index] }.group_by { |s, i| i[:index] }.each do |index, group|
|
840
|
-
details = {}
|
841
|
-
group.map(&:second).each do |g|
|
842
|
-
details = details.except(:index).deep_merge(g)
|
843
|
-
end
|
844
|
-
indexes << index.merge(queries: group.map(&:first), details: details)
|
845
|
-
end
|
846
|
-
|
847
|
-
indexes.sort_by { |i| [i[:table], i[:columns]] }
|
848
|
-
end
|
849
|
-
|
850
|
-
def autoindex(options = {})
|
851
|
-
suggested_indexes.each do |index|
|
852
|
-
p index
|
853
|
-
if options[:create]
|
854
|
-
connection.execute("CREATE INDEX CONCURRENTLY ON #{quote_table_name(index[:table])} (#{index[:columns].map { |c| quote_table_name(c) }.join(",")})")
|
855
|
-
end
|
856
|
-
end
|
857
|
-
end
|
858
|
-
|
859
|
-
def autoindex_all(options = {})
|
860
|
-
config["databases"].keys.each do |database|
|
861
|
-
with(database) do
|
862
|
-
puts "Autoindexing #{database}..."
|
863
|
-
autoindex(options)
|
864
|
-
end
|
865
|
-
end
|
866
|
-
end
|
867
|
-
|
868
|
-
def best_index(statement, options = {})
|
869
|
-
best_index_helper([statement])[statement]
|
870
|
-
end
|
871
|
-
|
872
|
-
def column_stats(options = {})
|
873
|
-
schema = options[:schema]
|
874
|
-
tables = options[:table] ? Array(options[:table]) : nil
|
875
|
-
select_all <<-SQL
|
876
|
-
SELECT
|
877
|
-
schemaname AS schema,
|
878
|
-
tablename AS table,
|
879
|
-
attname AS column,
|
880
|
-
null_frac,
|
881
|
-
n_distinct
|
882
|
-
FROM
|
883
|
-
pg_stats
|
884
|
-
WHERE
|
885
|
-
#{tables ? "tablename IN (#{tables.map { |t| quote(t) }.join(", ")})" : "1 = 1"}
|
886
|
-
AND schemaname = #{quote(schema)}
|
887
|
-
ORDER BY
|
888
|
-
1, 2, 3
|
889
|
-
SQL
|
890
|
-
end
|
891
|
-
|
892
|
-
def table_stats(options = {})
|
893
|
-
schema = options[:schema]
|
894
|
-
tables = options[:table] ? Array(options[:table]) : nil
|
895
|
-
select_all <<-SQL
|
896
|
-
SELECT
|
897
|
-
nspname AS schema,
|
898
|
-
relname AS table,
|
899
|
-
reltuples::bigint
|
900
|
-
FROM
|
901
|
-
pg_class
|
902
|
-
INNER JOIN
|
903
|
-
pg_namespace ON pg_namespace.oid = pg_class.relnamespace
|
904
|
-
WHERE
|
905
|
-
relkind = 'r'
|
906
|
-
AND nspname = #{quote(schema)}
|
907
|
-
#{tables ? "AND relname IN (#{tables.map { |t| quote(t) }.join(", ")})" : nil}
|
908
|
-
ORDER BY
|
909
|
-
1, 2
|
910
|
-
SQL
|
911
|
-
end
|
912
|
-
|
913
|
-
private
|
914
|
-
|
915
|
-
def best_index_helper(statements)
|
916
|
-
indexes = {}
|
917
|
-
|
918
|
-
# see if this is a query we understand and can use
|
919
|
-
parts = {}
|
920
|
-
statements.each do |statement|
|
921
|
-
parts[statement] = best_index_structure(statement)
|
922
|
-
end
|
923
|
-
|
924
|
-
# get stats about columns for relevant tables
|
925
|
-
tables = parts.values.map { |t| t[:table] }.uniq
|
926
|
-
# TODO get schema from query structure, then try search path
|
927
|
-
schema = connection_model.connection_config[:schema] || "public"
|
928
|
-
if tables.any?
|
929
|
-
row_stats = Hash[self.table_stats(table: tables, schema: schema).map { |i| [i["table"], i["reltuples"]] }]
|
930
|
-
column_stats = self.column_stats(table: tables, schema: schema).group_by { |i| i["table"] }
|
931
|
-
end
|
932
|
-
|
933
|
-
# find best index based on query structure and column stats
|
934
|
-
parts.each do |statement, structure|
|
935
|
-
index = {found: false}
|
936
|
-
|
937
|
-
if structure[:error]
|
938
|
-
index[:explanation] = structure[:error]
|
939
|
-
elsif structure[:table].start_with?("pg_")
|
940
|
-
index[:explanation] = "System table"
|
941
|
-
else
|
942
|
-
index[:structure] = structure
|
943
|
-
|
944
|
-
table = structure[:table]
|
945
|
-
where = structure[:where].uniq
|
946
|
-
sort = structure[:sort]
|
947
|
-
|
948
|
-
total_rows = row_stats[table].to_i
|
949
|
-
index[:rows] = total_rows
|
950
|
-
|
951
|
-
ranks = Hash[column_stats[table].to_a.map { |r| [r["column"], r] }]
|
952
|
-
columns = (where + sort).map { |c| c[:column] }.uniq
|
953
|
-
|
954
|
-
if columns.any?
|
955
|
-
if columns.all? { |c| ranks[c] }
|
956
|
-
first_desc = sort.index { |c| c[:direction] == "desc" }
|
957
|
-
if first_desc
|
958
|
-
sort = sort.first(first_desc + 1)
|
959
|
-
end
|
960
|
-
where = where.sort_by { |c| [row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]), c[:column]] } + sort
|
961
|
-
|
962
|
-
index[:row_estimates] = Hash[where.map { |c| ["#{c[:column]} (#{c[:op] || "sort"})", row_estimates(ranks[c[:column]], total_rows, total_rows, c[:op]).round] }]
|
963
|
-
|
964
|
-
# no index needed if less than 500 rows
|
965
|
-
if total_rows >= 500
|
966
|
-
|
967
|
-
if ["~~", "~~*"].include?(where.first[:op])
|
968
|
-
index[:found] = true
|
969
|
-
index[:row_progression] = [total_rows, index[:row_estimates].values.first]
|
970
|
-
index[:index] = {table: table, columns: ["#{where.first[:column]} gist_trgm_ops"], using: "gist"}
|
971
|
-
else
|
972
|
-
# if most values are unique, no need to index others
|
973
|
-
rows_left = total_rows
|
974
|
-
final_where = []
|
975
|
-
prev_rows_left = [rows_left]
|
976
|
-
where.reject { |c| ["~~", "~~*"].include?(c[:op]) }.each do |c|
|
977
|
-
next if final_where.include?(c[:column])
|
978
|
-
final_where << c[:column]
|
979
|
-
rows_left = row_estimates(ranks[c[:column]], total_rows, rows_left, c[:op])
|
980
|
-
prev_rows_left << rows_left
|
981
|
-
if rows_left < 50 || final_where.size >= 2 || [">", ">=", "<", "<=", "~~", "~~*"].include?(c[:op])
|
982
|
-
break
|
983
|
-
end
|
984
|
-
end
|
985
|
-
|
986
|
-
index[:row_progression] = prev_rows_left.map(&:round)
|
987
|
-
|
988
|
-
# if the last indexes don't give us much, don't include
|
989
|
-
prev_rows_left.reverse!
|
990
|
-
(prev_rows_left.size - 1).times do |i|
|
991
|
-
if prev_rows_left[i] > prev_rows_left[i + 1] * 0.3
|
992
|
-
final_where.pop
|
993
|
-
else
|
994
|
-
break
|
995
|
-
end
|
996
|
-
end
|
997
|
-
|
998
|
-
if final_where.any?
|
999
|
-
index[:found] = true
|
1000
|
-
index[:index] = {table: table, columns: final_where}
|
1001
|
-
end
|
1002
|
-
end
|
1003
|
-
else
|
1004
|
-
index[:explanation] = "No index needed if less than 500 rows"
|
1005
|
-
end
|
1006
|
-
else
|
1007
|
-
index[:explanation] = "Stats not found"
|
1008
|
-
end
|
1009
|
-
else
|
1010
|
-
index[:explanation] = "No columns to index"
|
1011
|
-
end
|
1012
|
-
end
|
1013
|
-
|
1014
|
-
indexes[statement] = index
|
1015
|
-
end
|
1016
|
-
|
1017
|
-
indexes
|
1018
|
-
end
|
1019
|
-
|
1020
|
-
def best_index_structure(statement)
|
1021
|
-
begin
|
1022
|
-
tree = PgQuery.parse(statement).parsetree
|
1023
|
-
rescue PgQuery::ParseError
|
1024
|
-
return {error: "Parse error"}
|
1025
|
-
end
|
1026
|
-
return {error: "Unknown structure"} unless tree.size == 1
|
1027
|
-
|
1028
|
-
tree = tree.first
|
1029
|
-
table = parse_table(tree) rescue nil
|
1030
|
-
unless table
|
1031
|
-
error =
|
1032
|
-
case tree.keys.first
|
1033
|
-
when "INSERT INTO"
|
1034
|
-
"INSERT statement"
|
1035
|
-
when "SET"
|
1036
|
-
"SET statement"
|
1037
|
-
when "SELECT"
|
1038
|
-
if (tree["SELECT"]["fromClause"].first["JOINEXPR"] rescue false)
|
1039
|
-
"JOIN not supported yet"
|
1040
|
-
end
|
1041
|
-
end
|
1042
|
-
return {error: error || "Unknown structure"}
|
1043
|
-
end
|
1044
|
-
|
1045
|
-
select = tree["SELECT"] || tree["DELETE FROM"] || tree["UPDATE"]
|
1046
|
-
where = (select["whereClause"] ? parse_where(select["whereClause"]) : []) rescue nil
|
1047
|
-
return {error: "Unknown structure"} unless where
|
1048
|
-
|
1049
|
-
sort = (select["sortClause"] ? parse_sort(select["sortClause"]) : []) rescue []
|
1050
|
-
|
1051
|
-
{table: table, where: where, sort: sort}
|
1052
|
-
end
|
1053
|
-
|
1054
|
-
def index_covers?(indexed_columns, columns)
|
1055
|
-
indexed_columns.first(columns.size) == columns
|
1056
|
-
end
|
1057
|
-
|
1058
|
-
# TODO better row estimation
|
1059
|
-
# http://www.postgresql.org/docs/current/static/row-estimation-examples.html
|
1060
|
-
def row_estimates(stats, total_rows, rows_left, op)
|
1061
|
-
case op
|
1062
|
-
when "null"
|
1063
|
-
rows_left * stats["null_frac"].to_f
|
1064
|
-
when "not_null"
|
1065
|
-
rows_left * (1 - stats["null_frac"].to_f)
|
1066
|
-
else
|
1067
|
-
rows_left *= (1 - stats["null_frac"].to_f)
|
1068
|
-
ret =
|
1069
|
-
if stats["n_distinct"].to_f == 0
|
1070
|
-
0
|
1071
|
-
elsif stats["n_distinct"].to_f < 0
|
1072
|
-
if total_rows > 0
|
1073
|
-
(-1 / stats["n_distinct"].to_f) * (rows_left / total_rows.to_f)
|
1074
|
-
else
|
1075
|
-
0
|
1076
|
-
end
|
1077
|
-
else
|
1078
|
-
rows_left / stats["n_distinct"].to_f
|
1079
|
-
end
|
1080
|
-
|
1081
|
-
case op
|
1082
|
-
when ">", ">=", "<", "<=", "~~", "~~*"
|
1083
|
-
(rows_left + ret) / 10.0 # TODO better approximation
|
1084
|
-
when "<>"
|
1085
|
-
rows_left - ret
|
1086
|
-
else
|
1087
|
-
ret
|
1088
|
-
end
|
1089
|
-
end
|
1090
|
-
end
|
1091
|
-
|
1092
|
-
def parse_table(tree)
|
1093
|
-
case tree.keys.first
|
1094
|
-
when "SELECT"
|
1095
|
-
tree["SELECT"]["fromClause"].first["RANGEVAR"]["relname"]
|
1096
|
-
when "DELETE FROM"
|
1097
|
-
tree["DELETE FROM"]["relation"]["RANGEVAR"]["relname"]
|
1098
|
-
when "UPDATE"
|
1099
|
-
tree["UPDATE"]["relation"]["RANGEVAR"]["relname"]
|
1100
|
-
else
|
1101
|
-
nil
|
1102
|
-
end
|
1103
|
-
end
|
1104
|
-
|
1105
|
-
# TODO capture values
|
1106
|
-
def parse_where(tree)
|
1107
|
-
if tree["AEXPR AND"]
|
1108
|
-
left = parse_where(tree["AEXPR AND"]["lexpr"])
|
1109
|
-
right = parse_where(tree["AEXPR AND"]["rexpr"])
|
1110
|
-
if left && right
|
1111
|
-
left + right
|
1112
|
-
end
|
1113
|
-
elsif tree["AEXPR"] && ["=", "<>", ">", ">=", "<", "<=", "~~", "~~*"].include?(tree["AEXPR"]["name"].first)
|
1114
|
-
[{column: tree["AEXPR"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR"]["name"].first}]
|
1115
|
-
elsif tree["AEXPR IN"] && ["=", "<>"].include?(tree["AEXPR IN"]["name"].first)
|
1116
|
-
[{column: tree["AEXPR IN"]["lexpr"]["COLUMNREF"]["fields"].last, op: tree["AEXPR IN"]["name"].first}]
|
1117
|
-
elsif tree["NULLTEST"]
|
1118
|
-
op = tree["NULLTEST"]["nulltesttype"] == 1 ? "not_null" : "null"
|
1119
|
-
[{column: tree["NULLTEST"]["arg"]["COLUMNREF"]["fields"].last, op: op}]
|
1120
|
-
else
|
1121
|
-
nil
|
1122
|
-
end
|
1123
|
-
end
|
1124
|
-
|
1125
|
-
def parse_sort(sort_clause)
|
1126
|
-
sort_clause.map do |v|
|
1127
|
-
{
|
1128
|
-
column: v["SORTBY"]["node"]["COLUMNREF"]["fields"].last,
|
1129
|
-
direction: v["SORTBY"]["sortby_dir"] == 2 ? "desc" : "asc"
|
1130
|
-
}
|
1131
|
-
end
|
1132
|
-
end
|
1133
|
-
|
1134
|
-
def table_grant_commands(privilege, tables, user)
|
1135
|
-
tables.map do |table|
|
1136
|
-
"GRANT #{privilege} ON TABLE #{table} TO #{user}"
|
1137
|
-
end
|
1138
|
-
end
|
1139
|
-
|
1140
|
-
# http://www.craigkerstiens.com/2013/01/10/more-on-postgres-performance/
|
1141
|
-
def current_query_stats(options = {})
|
1142
|
-
if query_stats_enabled?
|
1143
|
-
limit = options[:limit] || 100
|
1144
|
-
sort = options[:sort] || "total_minutes"
|
1145
|
-
select_all <<-SQL
|
1146
|
-
WITH query_stats AS (
|
1147
|
-
SELECT
|
1148
|
-
query,
|
1149
|
-
(total_time / 1000 / 60) as total_minutes,
|
1150
|
-
(total_time / calls) as average_time,
|
1151
|
-
calls
|
1152
|
-
FROM
|
1153
|
-
pg_stat_statements
|
1154
|
-
INNER JOIN
|
1155
|
-
pg_database ON pg_database.oid = pg_stat_statements.dbid
|
1156
|
-
WHERE
|
1157
|
-
pg_database.datname = current_database()
|
1158
|
-
)
|
1159
|
-
SELECT
|
1160
|
-
query,
|
1161
|
-
total_minutes,
|
1162
|
-
average_time,
|
1163
|
-
calls,
|
1164
|
-
total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
|
1165
|
-
(SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
|
1166
|
-
FROM
|
1167
|
-
query_stats
|
1168
|
-
ORDER BY
|
1169
|
-
#{quote_table_name(sort)} DESC
|
1170
|
-
LIMIT #{limit.to_i}
|
1171
|
-
SQL
|
1172
|
-
else
|
1173
|
-
[]
|
1174
|
-
end
|
1175
|
-
end
|
1176
|
-
|
1177
|
-
def historical_query_stats(options = {})
|
1178
|
-
if historical_query_stats_enabled?
|
1179
|
-
sort = options[:sort] || "total_minutes"
|
1180
|
-
stats_connection.select_all squish <<-SQL
|
1181
|
-
WITH query_stats AS (
|
1182
|
-
SELECT
|
1183
|
-
query,
|
1184
|
-
(SUM(total_time) / 1000 / 60) as total_minutes,
|
1185
|
-
(SUM(total_time) / SUM(calls)) as average_time,
|
1186
|
-
SUM(calls) as calls
|
1187
|
-
FROM
|
1188
|
-
pghero_query_stats
|
1189
|
-
WHERE
|
1190
|
-
database = #{quote(current_database)}
|
1191
|
-
#{options[:start_at] ? "AND captured_at >= #{quote(options[:start_at])}" : ""}
|
1192
|
-
#{options[:end_at] ? "AND captured_at <= #{quote(options[:end_at])}" : ""}
|
1193
|
-
GROUP BY
|
1194
|
-
query
|
1195
|
-
)
|
1196
|
-
SELECT
|
1197
|
-
query,
|
1198
|
-
total_minutes,
|
1199
|
-
average_time,
|
1200
|
-
calls,
|
1201
|
-
total_minutes * 100.0 / (SELECT SUM(total_minutes) FROM query_stats) AS total_percent,
|
1202
|
-
(SELECT SUM(total_minutes) FROM query_stats) AS all_queries_total_minutes
|
1203
|
-
FROM
|
1204
|
-
query_stats
|
1205
|
-
ORDER BY
|
1206
|
-
#{quote_table_name(sort)} DESC
|
1207
|
-
LIMIT 100
|
1208
|
-
SQL
|
1209
|
-
else
|
1210
|
-
[]
|
1211
|
-
end
|
1212
|
-
end
|
1213
|
-
|
1214
|
-
def friendly_value(setting, unit)
|
1215
|
-
if %w(kB 8kB).include?(unit)
|
1216
|
-
value = setting.to_i
|
1217
|
-
value *= 8 if unit == "8kB"
|
1218
|
-
|
1219
|
-
if value % (1024 * 1024) == 0
|
1220
|
-
"#{value / (1024 * 1024)}GB"
|
1221
|
-
elsif value % 1024 == 0
|
1222
|
-
"#{value / 1024}MB"
|
1223
|
-
else
|
1224
|
-
"#{value}kB"
|
1225
|
-
end
|
1226
|
-
else
|
1227
|
-
"#{setting}#{unit}".strip
|
1228
|
-
end
|
1229
|
-
end
|
1230
|
-
|
1231
|
-
def select_all(sql)
|
1232
|
-
# squish for logs
|
1233
|
-
connection.select_all(squish(sql)).to_a
|
1234
|
-
end
|
1235
|
-
|
1236
|
-
def execute(sql)
|
1237
|
-
connection.execute(sql)
|
1238
|
-
end
|
1239
|
-
|
1240
|
-
def connection_model
|
1241
|
-
databases[current_database].connection_model
|
1242
|
-
end
|
1243
|
-
|
1244
|
-
def connection
|
1245
|
-
connection_model.connection
|
1246
|
-
end
|
1247
|
-
|
1248
|
-
# from ActiveSupport
|
1249
|
-
def squish(str)
|
1250
|
-
str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
|
1251
|
-
end
|
1252
|
-
|
1253
|
-
def quote(value)
|
1254
|
-
connection.quote(value)
|
1255
|
-
end
|
1256
|
-
|
1257
|
-
def quote_table_name(value)
|
1258
|
-
connection.quote_table_name(value)
|
1259
|
-
end
|
1260
|
-
end
|
39
|
+
extend Methods::Basic
|
40
|
+
extend Methods::Connections
|
41
|
+
extend Methods::Databases
|
42
|
+
extend Methods::Explain
|
43
|
+
extend Methods::Indexes
|
44
|
+
extend Methods::Kill
|
45
|
+
extend Methods::Maintenance
|
46
|
+
extend Methods::Queries
|
47
|
+
extend Methods::QueryStats
|
48
|
+
extend Methods::Replica
|
49
|
+
extend Methods::Space
|
50
|
+
extend Methods::SuggestedIndexes
|
51
|
+
extend Methods::System
|
52
|
+
extend Methods::Tables
|
1261
53
|
end
|