pgdexter 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +13 -3
- data/guides/Linux.md +59 -0
- data/lib/dexter/client.rb +1 -0
- data/lib/dexter/indexer.rb +77 -37
- data/lib/dexter/query.rb +15 -2
- data/lib/dexter/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d08be58d7de049df2c11f33211c0ada165f4a363
|
4
|
+
data.tar.gz: ce17e302b12f3b7babe38de9618b76bab982b9d9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e0974f0f8c39ec8039749a75a3fb61f7613c6931db15f191eba2dda2a794cf57e8875f6aff4465474a9c608727deef08fd7a5833e16ba96e2d1d5a02e3e7f7b
|
7
|
+
data.tar.gz: 73668e8c819b4334888bec707db491453ab2cc03a39e189e451738bb7606b8fe329912e2520d6e2b6b7aeb9634a8e2200b267887257c26428bf453fde74cc54e
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -13,12 +13,12 @@ wget https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz
|
|
13
13
|
tar xf 1.0.0.tar.gz
|
14
14
|
cd hypopg-1.0.0
|
15
15
|
make
|
16
|
-
make install
|
16
|
+
make install # may need sudo
|
17
17
|
```
|
18
18
|
|
19
19
|
> Note: If you have issues, make sure `postgresql-server-dev-*` is installed.
|
20
20
|
|
21
|
-
Enable logging for slow queries.
|
21
|
+
Enable logging for slow queries in your Postgres config file.
|
22
22
|
|
23
23
|
```ini
|
24
24
|
log_min_duration_statement = 10 # ms
|
@@ -30,6 +30,8 @@ And install the command line tool with:
|
|
30
30
|
gem install pgdexter
|
31
31
|
```
|
32
32
|
|
33
|
+
The command line tool is also available as a [Linux package](guides/Linux.md).
|
34
|
+
|
33
35
|
## How to Use
|
34
36
|
|
35
37
|
Dexter needs a connection to your database and a log file to process.
|
@@ -60,13 +62,21 @@ To be safe, Dexter will not create indexes unless you pass the `--create` flag.
|
|
60
62
|
2017-06-25T17:52:37+00:00 Index created: 15243 ms
|
61
63
|
```
|
62
64
|
|
65
|
+
## Single Statement Mode
|
66
|
+
|
67
|
+
You can also pass a single statement with:
|
68
|
+
|
69
|
+
```sh
|
70
|
+
dexter <database-url> -s "SELECT * FROM ..."
|
71
|
+
```
|
72
|
+
|
63
73
|
## Options
|
64
74
|
|
65
75
|
Name | Description | Default
|
66
76
|
--- | --- | ---
|
67
77
|
exclude | prevent specific tables from being indexed | None
|
68
78
|
interval | time to wait between processing queries, in seconds | 60
|
69
|
-
log-level | `debug` gives additional info for suggested indexes<br />`debug2` gives additional info for
|
79
|
+
log-level | `debug` gives additional info for suggested indexes<br />`debug2` gives additional info for processed queries | info
|
70
80
|
log-sql | log SQL statements executed | false
|
71
81
|
min-time | only process queries consuming a min amount of DB time, in minutes | 0
|
72
82
|
|
data/guides/Linux.md
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# Linux Packages
|
2
|
+
|
3
|
+
Distributions
|
4
|
+
|
5
|
+
- [Ubuntu 16.04 (Xenial)](#ubuntu-1604-xenial)
|
6
|
+
- [Ubuntu 14.04 (Trusty)](#ubuntu-1404-trusty)
|
7
|
+
- [Debian 8 (Jesse)](#debian-8-jesse)
|
8
|
+
- [CentOS / RHEL 7](#centos--rhel-7)
|
9
|
+
- [SUSE Linux Enterprise Server 12](#suse-linux-enterprise-server-12)
|
10
|
+
|
11
|
+
### Ubuntu 16.04 (Xenial)
|
12
|
+
|
13
|
+
```sh
|
14
|
+
wget -qO - https://deb.packager.io/key | sudo apt-key add -
|
15
|
+
echo "deb https://deb.packager.io/gh/pghero/dexter xenial master" | sudo tee /etc/apt/sources.list.d/dexter.list
|
16
|
+
sudo apt-get update
|
17
|
+
sudo apt-get -y install dexter
|
18
|
+
```
|
19
|
+
|
20
|
+
### Ubuntu 14.04 (Trusty)
|
21
|
+
|
22
|
+
```sh
|
23
|
+
wget -qO - https://deb.packager.io/key | sudo apt-key add -
|
24
|
+
echo "deb https://deb.packager.io/gh/pghero/dexter trusty master" | sudo tee /etc/apt/sources.list.d/dexter.list
|
25
|
+
sudo apt-get update
|
26
|
+
sudo apt-get install dexter
|
27
|
+
```
|
28
|
+
|
29
|
+
### Debian 8 (Jesse)
|
30
|
+
|
31
|
+
```sh
|
32
|
+
wget -qO - https://deb.packager.io/key | sudo apt-key add -
|
33
|
+
echo "deb https://deb.packager.io/gh/pghero/dexter jessie master" | sudo tee /etc/apt/sources.list.d/dexter.list
|
34
|
+
sudo apt-get update
|
35
|
+
sudo apt-get install dexter
|
36
|
+
```
|
37
|
+
|
38
|
+
### CentOS / RHEL 7
|
39
|
+
|
40
|
+
```sh
|
41
|
+
sudo rpm --import https://rpm.packager.io/key
|
42
|
+
echo "[dexter]
|
43
|
+
name=Repository for pghero/dexter application.
|
44
|
+
baseurl=https://rpm.packager.io/gh/pghero/dexter/centos7/master
|
45
|
+
enabled=1" | sudo tee /etc/yum.repos.d/dexter.repo
|
46
|
+
sudo yum install dexter
|
47
|
+
```
|
48
|
+
|
49
|
+
### SUSE Linux Enterprise Server 12
|
50
|
+
|
51
|
+
```sh
|
52
|
+
sudo rpm --import https://rpm.packager.io/key
|
53
|
+
sudo zypper addrepo "https://rpm.packager.io/gh/pghero/dexter/sles12/master" "dexter"
|
54
|
+
sudo zypper install dexter
|
55
|
+
```
|
56
|
+
|
57
|
+
## Credits
|
58
|
+
|
59
|
+
:heart: Made possible by [Packager](https://packager.io/)
|
data/lib/dexter/client.rb
CHANGED
@@ -31,6 +31,7 @@ Options:)
|
|
31
31
|
o.array "--exclude", "prevent specific tables from being indexed"
|
32
32
|
o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
|
33
33
|
o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
|
34
|
+
o.boolean "--log-explain", "log explain", default: false, help: false
|
34
35
|
o.string "--log-level", "log level", default: "info"
|
35
36
|
o.boolean "--log-sql", "log sql", default: false
|
36
37
|
o.string "-s", "--statement", "process a single statement"
|
data/lib/dexter/indexer.rb
CHANGED
@@ -8,6 +8,7 @@ module Dexter
|
|
8
8
|
@log_level = options[:log_level]
|
9
9
|
@exclude_tables = options[:exclude]
|
10
10
|
@log_sql = options[:log_sql]
|
11
|
+
@log_explain = options[:log_explain]
|
11
12
|
|
12
13
|
create_extension
|
13
14
|
end
|
@@ -31,13 +32,10 @@ module Dexter
|
|
31
32
|
# analyze tables if needed
|
32
33
|
analyze_tables(tables) if tables.any?
|
33
34
|
|
34
|
-
#
|
35
|
-
|
36
|
-
|
37
|
-
# create hypothetical indexes
|
38
|
-
candidates = tables.any? ? create_hypothetical_indexes(tables) : {}
|
35
|
+
# create hypothetical indexes and explain queries
|
36
|
+
candidates = tables.any? ? create_hypothetical_indexes(queries.reject(&:missing_tables), tables) : {}
|
39
37
|
|
40
|
-
#
|
38
|
+
# see if new indexes were used and meet bar
|
41
39
|
new_indexes = determine_indexes(queries, candidates)
|
42
40
|
|
43
41
|
# display and create new indexes
|
@@ -47,18 +45,18 @@ module Dexter
|
|
47
45
|
private
|
48
46
|
|
49
47
|
def create_extension
|
50
|
-
|
51
|
-
|
48
|
+
execute("SET client_min_messages = warning")
|
49
|
+
execute("CREATE EXTENSION IF NOT EXISTS hypopg")
|
52
50
|
end
|
53
51
|
|
54
52
|
def reset_hypothetical_indexes
|
55
|
-
|
53
|
+
execute("SELECT hypopg_reset()")
|
56
54
|
end
|
57
55
|
|
58
56
|
def analyze_tables(tables)
|
59
57
|
tables = tables.to_a.sort
|
60
58
|
|
61
|
-
analyze_stats =
|
59
|
+
analyze_stats = execute <<-SQL
|
62
60
|
SELECT
|
63
61
|
schemaname AS schema,
|
64
62
|
relname AS table,
|
@@ -79,22 +77,33 @@ module Dexter
|
|
79
77
|
if !last_analyzed[table] || last_analyzed[table] < Time.now - 3600
|
80
78
|
statement = "ANALYZE #{quote_ident(table)}"
|
81
79
|
log "Running analyze: #{statement}"
|
82
|
-
|
80
|
+
execute(statement)
|
83
81
|
end
|
84
82
|
end
|
85
83
|
end
|
86
84
|
|
87
|
-
def
|
85
|
+
def calculate_plan(queries)
|
88
86
|
queries.each do |query|
|
89
87
|
begin
|
90
|
-
query.
|
88
|
+
query.plans << plan(query.statement)
|
89
|
+
if @log_explain
|
90
|
+
log "Explaining query"
|
91
|
+
puts
|
92
|
+
# Pass format to prevent ANALYZE
|
93
|
+
puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
|
94
|
+
puts
|
95
|
+
end
|
91
96
|
rescue PG::Error
|
92
97
|
# do nothing
|
93
98
|
end
|
94
99
|
end
|
95
100
|
end
|
96
101
|
|
97
|
-
def create_hypothetical_indexes(tables)
|
102
|
+
def create_hypothetical_indexes(queries, tables)
|
103
|
+
# get initial costs for queries
|
104
|
+
calculate_plan(queries)
|
105
|
+
explainable_queries = queries.select(&:explainable?)
|
106
|
+
|
98
107
|
# get existing indexes
|
99
108
|
index_set = Set.new
|
100
109
|
indexes(tables).each do |index|
|
@@ -104,13 +113,20 @@ module Dexter
|
|
104
113
|
|
105
114
|
# create hypothetical indexes
|
106
115
|
candidates = {}
|
107
|
-
columns(tables).
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
116
|
+
columns_by_table = columns(tables).group_by { |c| c[:table] }
|
117
|
+
|
118
|
+
# create single column indexes
|
119
|
+
create_hypothetical_indexes_helper(columns_by_table, 1, index_set, candidates)
|
120
|
+
|
121
|
+
# get next round of costs
|
122
|
+
calculate_plan(explainable_queries)
|
123
|
+
|
124
|
+
# create multicolumn indexes
|
125
|
+
create_hypothetical_indexes_helper(columns_by_table, 2, index_set, candidates)
|
126
|
+
|
127
|
+
# get next round of costs
|
128
|
+
calculate_plan(explainable_queries)
|
129
|
+
|
114
130
|
candidates
|
115
131
|
end
|
116
132
|
|
@@ -118,17 +134,23 @@ module Dexter
|
|
118
134
|
new_indexes = {}
|
119
135
|
|
120
136
|
queries.each do |query|
|
121
|
-
if query.
|
122
|
-
|
123
|
-
|
124
|
-
cost_savings =
|
137
|
+
if query.explainable?
|
138
|
+
new_cost, new_cost2 = query.costs[1..2]
|
139
|
+
|
140
|
+
cost_savings = new_cost < query.initial_cost * 0.5
|
141
|
+
# set high bar for multicolumn indexes
|
142
|
+
cost_savings2 = new_cost > 100 && new_cost2 < new_cost * 0.5
|
143
|
+
|
144
|
+
query.new_cost = cost_savings2 ? new_cost2 : new_cost
|
125
145
|
|
126
146
|
query_indexes = []
|
127
|
-
candidates.each do |
|
128
|
-
|
147
|
+
candidates.each do |col_set, index_name|
|
148
|
+
key = cost_savings2 ? 2 : 1
|
149
|
+
|
150
|
+
if query.plans[key].inspect.include?(index_name)
|
129
151
|
index = {
|
130
|
-
table:
|
131
|
-
columns: [
|
152
|
+
table: col_set[0][:table],
|
153
|
+
columns: col_set.map { |c| c[:column] }
|
132
154
|
}
|
133
155
|
query_indexes << index
|
134
156
|
|
@@ -142,12 +164,12 @@ module Dexter
|
|
142
164
|
|
143
165
|
if @log_level == "debug2"
|
144
166
|
log "Processed #{query.fingerprint}"
|
145
|
-
if query.
|
167
|
+
if query.explainable?
|
146
168
|
log "Cost: #{query.initial_cost} -> #{query.new_cost}"
|
147
169
|
|
148
170
|
if query_indexes.any?
|
149
171
|
log "Indexes: #{query_indexes.map { |i| "#{i[:table]} (#{i[:columns].join(", ")})" }.join(", ")}"
|
150
|
-
log "Need 50% cost savings to suggest index" unless cost_savings
|
172
|
+
log "Need 50% cost savings to suggest index" unless cost_savings || cost_savings2
|
151
173
|
else
|
152
174
|
log "Indexes: None"
|
153
175
|
end
|
@@ -195,7 +217,7 @@ module Dexter
|
|
195
217
|
statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
|
196
218
|
log "Creating index: #{statement}"
|
197
219
|
started_at = Time.now
|
198
|
-
|
220
|
+
execute(statement)
|
199
221
|
log "Index created: #{((Time.now - started_at) * 1000).to_i} ms"
|
200
222
|
end
|
201
223
|
end
|
@@ -223,7 +245,7 @@ module Dexter
|
|
223
245
|
abort "Bad database url"
|
224
246
|
end
|
225
247
|
|
226
|
-
def
|
248
|
+
def execute(query)
|
227
249
|
# use exec_params instead of exec for security
|
228
250
|
#
|
229
251
|
# Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
|
@@ -238,11 +260,23 @@ module Dexter
|
|
238
260
|
|
239
261
|
def plan(query)
|
240
262
|
# strip semi-colons as another measure of defense
|
241
|
-
JSON.parse(
|
263
|
+
JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"]).first["Plan"]
|
264
|
+
end
|
265
|
+
|
266
|
+
# TODO for multicolumn indexes, use ordering
|
267
|
+
def create_hypothetical_indexes_helper(columns_by_table, n, index_set, candidates)
|
268
|
+
columns_by_table.each do |table, cols|
|
269
|
+
# no reason to use btree index for json columns
|
270
|
+
cols.reject { |c| ["json", "jsonb"].include?(c[:type]) }.permutation(n) do |col_set|
|
271
|
+
if !index_set.include?([table, col_set.map { |col| col[:column] }])
|
272
|
+
candidates[col_set] = execute("SELECT * FROM hypopg_create_index('CREATE INDEX ON #{quote_ident(table)} (#{col_set.map { |c| quote_ident(c[:column]) }.join(", ")})')").first["indexname"]
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
242
276
|
end
|
243
277
|
|
244
278
|
def database_tables
|
245
|
-
result =
|
279
|
+
result = execute <<-SQL
|
246
280
|
SELECT
|
247
281
|
table_name
|
248
282
|
FROM
|
@@ -259,7 +293,7 @@ module Dexter
|
|
259
293
|
end
|
260
294
|
|
261
295
|
def columns(tables)
|
262
|
-
columns =
|
296
|
+
columns = execute <<-SQL
|
263
297
|
SELECT
|
264
298
|
table_name,
|
265
299
|
column_name,
|
@@ -269,13 +303,15 @@ module Dexter
|
|
269
303
|
WHERE
|
270
304
|
table_schema = 'public' AND
|
271
305
|
table_name IN (#{tables.map { |t| quote(t) }.join(", ")})
|
306
|
+
ORDER BY
|
307
|
+
1, 2
|
272
308
|
SQL
|
273
309
|
|
274
310
|
columns.map { |v| {table: v["table_name"], column: v["column_name"], type: v["data_type"]} }
|
275
311
|
end
|
276
312
|
|
277
313
|
def indexes(tables)
|
278
|
-
|
314
|
+
execute(<<-SQL
|
279
315
|
SELECT
|
280
316
|
schemaname AS schema,
|
281
317
|
t.relname AS table,
|
@@ -331,5 +367,9 @@ module Dexter
|
|
331
367
|
def squish(str)
|
332
368
|
str.to_s.gsub(/\A[[:space:]]+/, "").gsub(/[[:space:]]+\z/, "").gsub(/[[:space:]]+/, " ")
|
333
369
|
end
|
370
|
+
|
371
|
+
def safe_statement(statement)
|
372
|
+
statement.gsub(";", "")
|
373
|
+
end
|
334
374
|
end
|
335
375
|
end
|
data/lib/dexter/query.rb
CHANGED
@@ -1,15 +1,28 @@
|
|
1
1
|
module Dexter
|
2
2
|
class Query
|
3
|
-
attr_reader :statement, :fingerprint
|
4
|
-
attr_accessor :
|
3
|
+
attr_reader :statement, :fingerprint, :plans
|
4
|
+
attr_accessor :missing_tables, :new_cost
|
5
5
|
|
6
6
|
def initialize(statement, fingerprint)
|
7
7
|
@statement = statement
|
8
8
|
@fingerprint = fingerprint
|
9
|
+
@plans = []
|
9
10
|
end
|
10
11
|
|
11
12
|
def tables
|
12
13
|
@tables ||= PgQuery.parse(statement).tables rescue []
|
13
14
|
end
|
15
|
+
|
16
|
+
def explainable?
|
17
|
+
plans.any?
|
18
|
+
end
|
19
|
+
|
20
|
+
def costs
|
21
|
+
plans.map { |plan| plan["Total Cost"] }
|
22
|
+
end
|
23
|
+
|
24
|
+
def initial_cost
|
25
|
+
costs[0]
|
26
|
+
end
|
14
27
|
end
|
15
28
|
end
|
data/lib/dexter/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgdexter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-07-
|
11
|
+
date: 2017-07-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: slop
|
@@ -109,6 +109,7 @@ files:
|
|
109
109
|
- README.md
|
110
110
|
- Rakefile
|
111
111
|
- exe/dexter
|
112
|
+
- guides/Linux.md
|
112
113
|
- lib/dexter.rb
|
113
114
|
- lib/dexter/client.rb
|
114
115
|
- lib/dexter/collector.rb
|