pg_reindex 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +47 -0
- data/MIT-LICENSE +20 -0
- data/README.md +63 -0
- data/Rakefile +8 -0
- data/bin/pgre +209 -0
- data/lib/pg-reindex.rb +193 -0
- data/pg_reindex.gemspec +26 -0
- data/spec/reindex_spec.rb +135 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/spec_support.rb +25 -0
- metadata +145 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
pg_reindex (0.1.1)
|
5
|
+
pg
|
6
|
+
thor
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: http://rubygems.org/
|
10
|
+
specs:
|
11
|
+
activemodel (3.2.5)
|
12
|
+
activesupport (= 3.2.5)
|
13
|
+
builder (~> 3.0.0)
|
14
|
+
activerecord (3.2.5)
|
15
|
+
activemodel (= 3.2.5)
|
16
|
+
activesupport (= 3.2.5)
|
17
|
+
arel (~> 3.0.2)
|
18
|
+
tzinfo (~> 0.3.29)
|
19
|
+
activesupport (3.2.5)
|
20
|
+
i18n (~> 0.6)
|
21
|
+
multi_json (~> 1.0)
|
22
|
+
arel (3.0.2)
|
23
|
+
builder (3.0.0)
|
24
|
+
diff-lcs (1.1.3)
|
25
|
+
i18n (0.6.0)
|
26
|
+
multi_json (1.3.6)
|
27
|
+
pg (0.13.2)
|
28
|
+
rake (0.9.2.2)
|
29
|
+
rspec (2.10.0)
|
30
|
+
rspec-core (~> 2.10.0)
|
31
|
+
rspec-expectations (~> 2.10.0)
|
32
|
+
rspec-mocks (~> 2.10.0)
|
33
|
+
rspec-core (2.10.1)
|
34
|
+
rspec-expectations (2.10.0)
|
35
|
+
diff-lcs (~> 1.1.3)
|
36
|
+
rspec-mocks (2.10.1)
|
37
|
+
thor (0.15.2)
|
38
|
+
tzinfo (0.3.33)
|
39
|
+
|
40
|
+
PLATFORMS
|
41
|
+
ruby
|
42
|
+
|
43
|
+
DEPENDENCIES
|
44
|
+
activerecord
|
45
|
+
pg_reindex!
|
46
|
+
rake
|
47
|
+
rspec
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Makarchev K
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
Pg Reindex
|
2
|
+
==========
|
3
|
+
|
4
|
+
Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
5
|
+
|
6
|
+
Install:
|
7
|
+
|
8
|
+
$ gem install pg_reindex
|
9
|
+
|
10
|
+
Using:
|
11
|
+
|
12
|
+
export PGRE_CFG=/some/path/database.yml
|
13
|
+
pgre --help
|
14
|
+
|
15
|
+
User in connection should be an owner of relations. And for rebuild pkey, should be superuser.
|
16
|
+
|
17
|
+
Tasks:
|
18
|
+
|
19
|
+
pgre dbs # Show list of databases from database.yml
|
20
|
+
pgre install ENV # Install function swap_for_pkey to database
|
21
|
+
pgre rebuild ENV (table|index)[,(table|index)] # rebuild tables,indexes,pkeys
|
22
|
+
pgre tables ENV # Show tables of database
|
23
|
+
|
24
|
+
Examples:
|
25
|
+
|
26
|
+
# Show tables with indexes, which full size more than 1Gb
|
27
|
+
pgre tables production -s 1000
|
28
|
+
|
29
|
+
# Show process without really rebuild
|
30
|
+
pgre rebuild production users,some_index1,some_index2
|
31
|
+
|
32
|
+
# Rebuild indexes
|
33
|
+
pgre rebuild production users --write
|
34
|
+
|
35
|
+
# Rebuild indexes and no ask confirmation (not recommended)
|
36
|
+
pgre rebuild production users --write --no-ask
|
37
|
+
|
38
|
+
|
39
|
+
Explanation/Warning:
|
40
|
+
--------------------
|
41
|
+
|
42
|
+
Rebuild index produces sqls:
|
43
|
+
|
44
|
+
1. CREATE INDEX CONCURRENTLY bla2 on some_table USING btree (some_field);
|
45
|
+
2. ANALYZE some_table;
|
46
|
+
3. DROP INDEX bla;
|
47
|
+
4. ALTER INDEX bla2 RENAME TO bla;
|
48
|
+
|
49
|
+
2 can be blocked by long running query(LRQ), or autovacuum (in this case just kill autovacuum or wait LRQ).
|
50
|
+
By careful, if between 2 and (3,4) started LRQ or autovacuum, (3,4) can blocks all queries on this table. If this happens, and (3,4) not quit after < 30s, should stop (3,4) by cancel query in PostgreSQL. And execute (3,4) manually.
|
51
|
+
|
52
|
+
|
53
|
+
Rebuild pkey produces sqls:
|
54
|
+
|
55
|
+
1. CREATE UNIQUE INDEX CONCURRENTLY some_table_pkey2 on some_table USING btree (ID);
|
56
|
+
2. ANALYZE some_table;
|
57
|
+
3. SELECT swap_for_pkey('public', 'some_table_pkey', 'some_table_pkey2');
|
58
|
+
|
59
|
+
Same issue with 2 and 3.
|
60
|
+
|
61
|
+
|
62
|
+
|
63
|
+
MIT-LICENCE
|
data/Rakefile
ADDED
data/bin/pgre
ADDED
@@ -0,0 +1,209 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'thor'
|
5
|
+
require 'yaml'
|
6
|
+
require File.dirname(__FILE__) + '/../lib/pg-reindex'
|
7
|
+
|
8
|
+
class PgReindexer < Thor
|
9
|
+
|
10
|
+
desc "dbs", "Show list of databases from database.yml (-n if no show sizes)"
|
11
|
+
method_option :no_size, :aliases => '-n', :type => :boolean
|
12
|
+
def dbs
|
13
|
+
list = db_list(!options[:no_size])
|
14
|
+
return unless list
|
15
|
+
|
16
|
+
render_db_list(list)
|
17
|
+
end
|
18
|
+
|
19
|
+
desc "tables ENV", "Show tables of database"
|
20
|
+
method_option :size, :aliases => '-s', :type => :string
|
21
|
+
def tables(env)
|
22
|
+
conn = get_connection(env)
|
23
|
+
relations = conn.get_struct_relations(options[:size])
|
24
|
+
render_tables(env, relations)
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "rebuild ENV (table|index)[,(table|index)]", "rebuild tables,indexes,pkeys"
|
28
|
+
method_option :write, :aliases => '--write', :type => :boolean
|
29
|
+
method_option :no_ask, :aliases => '--no-ask', :type => :boolean
|
30
|
+
def rebuild(env, to_reindex = '')
|
31
|
+
if !to_reindex || to_reindex.empty?
|
32
|
+
say "blank attributes to rebuild", :red
|
33
|
+
exit 1
|
34
|
+
end
|
35
|
+
|
36
|
+
conn = get_connection(env)
|
37
|
+
|
38
|
+
rebuild = conn.filter_relations(to_reindex)
|
39
|
+
|
40
|
+
rebuild.each do |row|
|
41
|
+
do_rebuild conn, row, options[:write], !options[:no_ask]
|
42
|
+
end
|
43
|
+
|
44
|
+
if rebuild.empty?
|
45
|
+
say "nothing to rebuild", :red
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
desc "install ENV", "Install need function to database, like function: swap_for_pkey"
|
50
|
+
def install(env)
|
51
|
+
conn = get_connection(env)
|
52
|
+
conn.install_swap_for_pkey
|
53
|
+
say "DONE!", :green
|
54
|
+
end
|
55
|
+
|
56
|
+
desc "queries ENV", "Show current queries on database ENV"
|
57
|
+
def queries(env)
|
58
|
+
conn = get_connection(env)
|
59
|
+
q = conn.queries
|
60
|
+
r = q.map do |query|
|
61
|
+
[query['procpid'], query['age'], query['current_query']]
|
62
|
+
end
|
63
|
+
|
64
|
+
print_table r
|
65
|
+
end
|
66
|
+
|
67
|
+
desc "cancel ENV PID", "Cancel pid query"
|
68
|
+
def cancel(env, pid)
|
69
|
+
conn = get_connection(env)
|
70
|
+
conn.cancel(pid)
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
def get_connection(env)
|
76
|
+
conn = connection(env)
|
77
|
+
unless conn
|
78
|
+
say "Not connected to #{env}, hm!!!", :red
|
79
|
+
exit 1
|
80
|
+
end
|
81
|
+
|
82
|
+
conn
|
83
|
+
end
|
84
|
+
|
85
|
+
def connection(env)
|
86
|
+
cn = connect(env, db_config[env])
|
87
|
+
end
|
88
|
+
|
89
|
+
def connect(env, conf)
|
90
|
+
@connects ||= {}
|
91
|
+
@connects[env] ||= PgReindex.new(conf) if conf
|
92
|
+
end
|
93
|
+
|
94
|
+
# load database.yml
|
95
|
+
def db_config
|
96
|
+
@db_config ||= begin
|
97
|
+
file = ENV['PGRE_CFG'] || `pwd`.to_s.chop + "/config/database.yml"
|
98
|
+
if File.exists?(file)
|
99
|
+
YAML.load_file(file)
|
100
|
+
else
|
101
|
+
say "Specify path to database.yml with env CFG, like: CFG=/../../database.yml", :red
|
102
|
+
exit 1
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def db_list(show_sizes = true)
|
108
|
+
cfg = db_config
|
109
|
+
return unless cfg
|
110
|
+
|
111
|
+
cfg = cfg.sort_by{|a| a[0]}
|
112
|
+
cfg.map do |env, config|
|
113
|
+
x = [env, config]
|
114
|
+
x << (connection(env).database_size(config['database']) rescue '-') if show_sizes
|
115
|
+
x
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def do_rebuild(conn, row, write = true, ask = true)
|
120
|
+
#ask = false unless write
|
121
|
+
|
122
|
+
say "=> rebuild #{row['index']} (#{row['i_size_p']}), write #{write}, ask #{ask}", :yellow
|
123
|
+
|
124
|
+
sqls = conn.index_sqls(row)
|
125
|
+
|
126
|
+
if ask || !write
|
127
|
+
sqls.each do |sql|
|
128
|
+
say " #{sql}"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
if ask
|
133
|
+
r = ask_until("Rebuild? (y/n)")
|
134
|
+
if r == '' || r == 'y' || r == 'Y'
|
135
|
+
else
|
136
|
+
say " ... skipped\n", :red
|
137
|
+
return
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
if row['index'].end_with?("_pkey")
|
142
|
+
unless conn.check_swap_for_pkey
|
143
|
+
say " ... You should install function swap_for_pkey for rebuild pkey, call: #{$0} install ENV", :red
|
144
|
+
return
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
started_at = Time.now
|
149
|
+
|
150
|
+
if write
|
151
|
+
sqls.each do |query|
|
152
|
+
say " => ", :bold, false
|
153
|
+
say "#{query}", nil, false
|
154
|
+
tm = Time.now
|
155
|
+
|
156
|
+
conn.exec(query)
|
157
|
+
|
158
|
+
say " (#{"%.3f" % (Time.now - tm)}s)"
|
159
|
+
end
|
160
|
+
|
161
|
+
new_size = conn.get_index_size(row['index'])['i_size_p'] rescue '-'
|
162
|
+
say "<= rebuilded #{row['index']} (#{new_size})", :yellow
|
163
|
+
say " DONE! (#{"%.3f" % (Time.now - started_at)}s)\n", :green
|
164
|
+
else
|
165
|
+
say " DRY-RUN DONE\n", :green
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def render_db_list(db_list)
|
170
|
+
say "Databases list:", :green
|
171
|
+
db_list.each do |row|
|
172
|
+
say "\t#{row[0].ljust(30)} ", :yellow, false
|
173
|
+
say "#{row[1]['database']}:#{row[1]['host']}".ljust(60)
|
174
|
+
say "#{row[2]}"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def render_tables(env, relations)
|
179
|
+
off1 = 65
|
180
|
+
off2 = 15
|
181
|
+
off3 = 15
|
182
|
+
|
183
|
+
say "tables for #{env.ljust(off1 - 9)}size#{''.ljust(off2 - 4)}indexes_size", :green
|
184
|
+
|
185
|
+
relations.each do |table, indexes|
|
186
|
+
say " #{table.ljust(off1)}", :yellow, false
|
187
|
+
f = indexes[0]
|
188
|
+
say "#{f['size_p'].ljust(off2)}", :yellow, false
|
189
|
+
say "#{f['total_i_size_p'].ljust(off3)}", :yellow, true
|
190
|
+
#say "#{f['total_size_p']}", :yellow
|
191
|
+
|
192
|
+
|
193
|
+
indexes.each do |rel|
|
194
|
+
say " #{(rel['index']+" ").ljust(off1 - 2 + off2, '.')}", nil, false
|
195
|
+
say "#{rel['i_size_p']}"
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def ask_until(str, answers = ['y', 'n', ''])
|
201
|
+
loop do
|
202
|
+
res = ask(str)
|
203
|
+
return res if answers.include?(res)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
end
|
208
|
+
|
209
|
+
PgReindexer.start
|
data/lib/pg-reindex.rb
ADDED
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
class PgReindex
|
4
|
+
|
5
|
+
MIN_SIZE = 50 # megabyte min, total table size
|
6
|
+
|
7
|
+
def initialize(conf)
|
8
|
+
cfg = {:host => conf['host'] || '127.0.0.1', :dbname => conf['database'],
|
9
|
+
:user => conf['username'] || `whoami`.chop, :password => conf['password'] || 'password', :port => conf['port'].to_s}
|
10
|
+
|
11
|
+
@conn = PGconn.new cfg
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_struct_relations(min_size = nil)
|
15
|
+
res = get_raw_relations
|
16
|
+
|
17
|
+
result = {}
|
18
|
+
res.each do |row|
|
19
|
+
next if row['total_size'].to_i < ((min_size || MIN_SIZE).to_f * 1024 * 1024).to_i
|
20
|
+
result[row['table']] ||= []
|
21
|
+
result[row['table']] << row
|
22
|
+
end
|
23
|
+
|
24
|
+
result = result.sort_by{|el| el[1][0]['total_size'].to_i}.reverse
|
25
|
+
|
26
|
+
result = result.map do |table, res|
|
27
|
+
[table, res]
|
28
|
+
end
|
29
|
+
|
30
|
+
result
|
31
|
+
end
|
32
|
+
|
33
|
+
def filter_relations(filter) # filter is a string with ,
|
34
|
+
return [] if !filter || filter.empty?
|
35
|
+
|
36
|
+
filter = filter.split(",")
|
37
|
+
|
38
|
+
get_raw_relations.select do |row|
|
39
|
+
filter.include?(row['index']) || filter.include?(row['table'])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_raw_relations
|
44
|
+
res = @conn.exec <<-SQL
|
45
|
+
SELECT C.relname AS "table",
|
46
|
+
i.relname "index",
|
47
|
+
i.oid "index_oid",
|
48
|
+
pg_relation_size(C.oid) AS "size",
|
49
|
+
pg_total_relation_size(C.oid) AS "total_size",
|
50
|
+
pg_size_pretty(pg_relation_size(C.oid)) AS "size_p",
|
51
|
+
pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size_p",
|
52
|
+
pg_size_pretty(pg_total_relation_size(C.oid) - pg_relation_size(C.oid)) AS "total_i_size_p",
|
53
|
+
pg_relation_size(i.oid) as "i_size",
|
54
|
+
pg_size_pretty(pg_relation_size(i.oid)) as "i_size_p"
|
55
|
+
FROM pg_class C, pg_class i, pg_index ix, pg_namespace N
|
56
|
+
WHERE nspname IN ('public') AND
|
57
|
+
C.oid = ix.indrelid and i.oid = ix.indexrelid
|
58
|
+
AND C.oid = ix.indrelid and i.oid = ix.indexrelid
|
59
|
+
AND C.relname not like 'pg_%'
|
60
|
+
AND N.oid = C.relnamespace
|
61
|
+
ORDER BY c.relname, i.relname
|
62
|
+
SQL
|
63
|
+
end
|
64
|
+
|
65
|
+
def get_index_size(relation_name)
|
66
|
+
res = @conn.exec "SELECT pg_relation_size(oid) as i_size, pg_size_pretty(pg_relation_size(oid)) as i_size_p
|
67
|
+
from pg_class WHERE relname = E'#{relation_name}'"
|
68
|
+
res[0]
|
69
|
+
rescue
|
70
|
+
{}
|
71
|
+
end
|
72
|
+
|
73
|
+
def install_swap_for_pkey
|
74
|
+
@conn.exec("create language plpgsql") rescue nil
|
75
|
+
@conn.exec(swap_for_pkey_sql)
|
76
|
+
end
|
77
|
+
|
78
|
+
def check_swap_for_pkey
|
79
|
+
res = @conn.exec <<-SQL
|
80
|
+
SELECT proname FROM pg_catalog.pg_namespace n JOIN pg_catalog.pg_proc p ON pronamespace = n.oid
|
81
|
+
WHERE nspname = 'public' and proname = 'swap_for_pkey'
|
82
|
+
SQL
|
83
|
+
|
84
|
+
res.values.size == 1
|
85
|
+
end
|
86
|
+
|
87
|
+
def swap_for_pkey_sql
|
88
|
+
<<-SQL
|
89
|
+
CREATE OR REPLACE FUNCTION swap_for_pkey(text,text,text) returns integer
|
90
|
+
AS
|
91
|
+
$$
|
92
|
+
DECLARE
|
93
|
+
cmd text;
|
94
|
+
oid1 integer;
|
95
|
+
oid2 integer;
|
96
|
+
filenode1 integer;
|
97
|
+
filenode2 integer;
|
98
|
+
relation text;
|
99
|
+
BEGIN
|
100
|
+
select oid::integer into oid1 from pg_class where relname=$2 and relnamespace = (select oid from pg_namespace where nspname=$1);
|
101
|
+
RAISE NOTICE 'PKEY OID: %',oid1;
|
102
|
+
select relfilenode::integer into filenode1 from pg_class where oid=oid1;
|
103
|
+
select oid::integer into oid2 from pg_class where relname=$3 and relnamespace = (select oid from pg_namespace where nspname=$1);
|
104
|
+
RAISE NOTICE 'PKEY OID: %',oid2;
|
105
|
+
select relfilenode::integer into filenode2 from pg_class where oid=oid2;
|
106
|
+
select (indrelid::regclass)::text into relation from pg_index where indexrelid=oid1;
|
107
|
+
RAISE NOTICE 'RELATION NAME: %',relation;
|
108
|
+
cmd:='LOCK '||relation||';';
|
109
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
110
|
+
Execute cmd;
|
111
|
+
cmd:='UPDATE pg_class SET relfilenode='||filenode2|| ' WHERE oid='||oid1||';';
|
112
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
113
|
+
Execute cmd;
|
114
|
+
cmd:='UPDATE pg_class SET relfilenode='||filenode1|| ' WHERE oid='||oid2||';';
|
115
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
116
|
+
Execute cmd;
|
117
|
+
cmd:='DROP INDEX '||$1||'.'||$3||';';
|
118
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
119
|
+
Execute cmd;
|
120
|
+
return 0;
|
121
|
+
END;
|
122
|
+
$$language plpgsql;
|
123
|
+
SQL
|
124
|
+
end
|
125
|
+
|
126
|
+
def database_size(database)
|
127
|
+
res = @conn.exec("SELECT pg_size_pretty(pg_database_size('#{database}')) as db_size")
|
128
|
+
res.first['db_size']
|
129
|
+
end
|
130
|
+
|
131
|
+
def index_sqls(index_row)
|
132
|
+
index_sql_array(index_row['table'], index_row['index_oid'], index_row['index'])
|
133
|
+
end
|
134
|
+
|
135
|
+
def index_sql_array(table, oid, name)
|
136
|
+
new_name = if name.size < 61
|
137
|
+
name + "_2"
|
138
|
+
else
|
139
|
+
name[0..-3] + "_2"
|
140
|
+
end
|
141
|
+
|
142
|
+
if name.end_with?('_pkey')
|
143
|
+
[
|
144
|
+
index_sql(table, oid, name, new_name),
|
145
|
+
"ANALYZE #{table}",
|
146
|
+
"SELECT swap_for_pkey('public', '#{name}', '#{new_name}')"
|
147
|
+
]
|
148
|
+
else
|
149
|
+
[
|
150
|
+
index_sql(table, oid, name, new_name),
|
151
|
+
"ANALYZE #{table}",
|
152
|
+
"DROP INDEX #{name}",
|
153
|
+
"ALTER INDEX #{new_name} RENAME TO #{name}"
|
154
|
+
]
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def index_sql(table, oid, name, new_name)
|
159
|
+
str = index_def(oid).gsub(name, new_name)
|
160
|
+
|
161
|
+
pos = str.index(new_name)
|
162
|
+
|
163
|
+
before = str[0..pos-2]
|
164
|
+
after = str[pos..-1]
|
165
|
+
|
166
|
+
"#{before} CONCURRENTLY #{after}"
|
167
|
+
end
|
168
|
+
|
169
|
+
def index_def(oid)
|
170
|
+
sql = "SELECT pg_get_indexdef(#{oid}) as q"
|
171
|
+
exec(sql)[0]['q']
|
172
|
+
end
|
173
|
+
|
174
|
+
def exec(sql)
|
175
|
+
@conn.exec sql
|
176
|
+
end
|
177
|
+
|
178
|
+
def queries
|
179
|
+
@conn.exec <<-SQL
|
180
|
+
SELECT procpid, now() - query_start as age, current_query
|
181
|
+
FROM
|
182
|
+
pg_stat_activity AS a JOIN
|
183
|
+
pg_locks AS l ON a.procpid = l.pid
|
184
|
+
WHERE
|
185
|
+
virtualxid IS NOT NULL order by age desc;
|
186
|
+
SQL
|
187
|
+
end
|
188
|
+
|
189
|
+
def cancel(procpid)
|
190
|
+
@conn.exec "select pg_cancel_backend(#{procpid.to_i})"
|
191
|
+
end
|
192
|
+
|
193
|
+
end
|
data/pg_reindex.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{pg_reindex}
|
5
|
+
s.version = "0.1.1"
|
6
|
+
|
7
|
+
s.authors = ["Makarchev Konstantin"]
|
8
|
+
|
9
|
+
s.description = %q{Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.}
|
10
|
+
s.summary = %q{Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.}
|
11
|
+
|
12
|
+
s.email = %q{kostya27@gmail.com}
|
13
|
+
s.homepage = %q{http://github.com/kostya/pg_reindex}
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.add_dependency 'thor'
|
21
|
+
s.add_dependency 'pg'
|
22
|
+
s.add_development_dependency "rspec"
|
23
|
+
s.add_development_dependency "rake"
|
24
|
+
s.add_development_dependency 'activerecord'
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe PgReindex do
|
4
|
+
before :each do
|
5
|
+
@pgre = PgReindex.new($cfg)
|
6
|
+
end
|
7
|
+
|
8
|
+
def row(name)
|
9
|
+
@pgre.filter_relations(name).first
|
10
|
+
end
|
11
|
+
|
12
|
+
it "get_raw_relations" do
|
13
|
+
res = @pgre.get_raw_relations
|
14
|
+
v = []
|
15
|
+
res.each{|h| v << h }
|
16
|
+
|
17
|
+
v.map{|el| el['table']}.uniq.should == ['tests']
|
18
|
+
v.map{|el| el['index']}.uniq.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
19
|
+
v.map{|el| el['index_oid']}.sum.to_i.should > 0
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "filter_relations" do
|
23
|
+
it "1" do
|
24
|
+
res = @pgre.filter_relations('a_b_c')
|
25
|
+
res.size.should == 1
|
26
|
+
res[0]['index'].should == 'a_b_c'
|
27
|
+
end
|
28
|
+
|
29
|
+
it "2" do
|
30
|
+
res = @pgre.filter_relations('a_b_c,index_tests_on_a,tests_pkey,xxx')
|
31
|
+
res.size.should == 3
|
32
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "tests_pkey"]
|
33
|
+
end
|
34
|
+
|
35
|
+
it "3" do
|
36
|
+
res = @pgre.filter_relations('tests,xxx')
|
37
|
+
res.size.should == 4
|
38
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
39
|
+
end
|
40
|
+
|
41
|
+
it "4" do
|
42
|
+
res = @pgre.filter_relations('xxx')
|
43
|
+
res.size.should == 0
|
44
|
+
end
|
45
|
+
|
46
|
+
it "5" do
|
47
|
+
res = @pgre.filter_relations('tests,a_b_c')
|
48
|
+
res.size.should == 4
|
49
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
it "database_size" do
|
54
|
+
s = @pgre.database_size('pgre_test')
|
55
|
+
s.to_i.should > 1
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "index sqls" do
|
59
|
+
it "index sqls for a" do
|
60
|
+
@pgre.index_sqls(row('index_tests_on_a')).should == ["CREATE INDEX CONCURRENTLY index_tests_on_a_2 ON tests USING btree (a)",
|
61
|
+
"ANALYZE tests", "DROP INDEX index_tests_on_a", "ALTER INDEX index_tests_on_a_2 RENAME TO index_tests_on_a"]
|
62
|
+
end
|
63
|
+
|
64
|
+
it "index sqls for b,c" do
|
65
|
+
@pgre.index_sqls(row('index_tests_on_b_and_c')).should == ["CREATE INDEX CONCURRENTLY index_tests_on_b_and_c_2 ON tests USING btree (b, c)",
|
66
|
+
"ANALYZE tests", "DROP INDEX index_tests_on_b_and_c", "ALTER INDEX index_tests_on_b_and_c_2 RENAME TO index_tests_on_b_and_c"]
|
67
|
+
end
|
68
|
+
|
69
|
+
it "index sqls for a,b,c" do
|
70
|
+
@pgre.index_sqls(row('a_b_c')).should == ["CREATE UNIQUE INDEX CONCURRENTLY a_b_c_2 ON tests USING btree (a, b, c) WHERE ((a > 0) AND (b < 0))",
|
71
|
+
"ANALYZE tests", "DROP INDEX a_b_c", "ALTER INDEX a_b_c_2 RENAME TO a_b_c"]
|
72
|
+
end
|
73
|
+
|
74
|
+
it "index sqls for pkey" do
|
75
|
+
@pgre.index_sqls(row('tests_pkey')).should == ["CREATE UNIQUE INDEX CONCURRENTLY tests_pkey_2 ON tests USING btree (id)",
|
76
|
+
"ANALYZE tests", "SELECT swap_for_pkey('public', 'tests_pkey', 'tests_pkey_2')"]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
it "index def" do
|
81
|
+
r = row('a_b_c')
|
82
|
+
@pgre.index_def(r['index_oid']).should == "CREATE UNIQUE INDEX a_b_c ON tests USING btree (a, b, c) WHERE ((a > 0) AND (b < 0))"
|
83
|
+
end
|
84
|
+
|
85
|
+
it "rebuilds index" do
|
86
|
+
r = row('a_b_c')
|
87
|
+
r['index_oid'].to_i.should > 0
|
88
|
+
def1 = @pgre.index_def(r['index_oid'])
|
89
|
+
|
90
|
+
sqls = @pgre.index_sqls(r)
|
91
|
+
sqls.each do |sql|
|
92
|
+
@pgre.exec sql
|
93
|
+
end
|
94
|
+
|
95
|
+
r2 = row('a_b_c')
|
96
|
+
r2['index_oid'].to_i.should > 0
|
97
|
+
|
98
|
+
r2['index_oid'].should_not == r['index_oid']
|
99
|
+
r2['index'].should == r['index']
|
100
|
+
|
101
|
+
def2 = @pgre.index_def(r2['index_oid'])
|
102
|
+
def1.should == def2
|
103
|
+
|
104
|
+
# index should be
|
105
|
+
res = @pgre.filter_relations('tests')
|
106
|
+
res.size.should == 4
|
107
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
108
|
+
end
|
109
|
+
|
110
|
+
it "rebuilds pkey" do
|
111
|
+
# install swap_for_pkey
|
112
|
+
@pgre.install_swap_for_pkey
|
113
|
+
|
114
|
+
r = row('tests_pkey')
|
115
|
+
r['index_oid'].to_i.should > 0
|
116
|
+
def1 = @pgre.index_def(r['index_oid'])
|
117
|
+
|
118
|
+
sqls = @pgre.index_sqls(r)
|
119
|
+
sqls.each do |sql|
|
120
|
+
@pgre.exec sql
|
121
|
+
end
|
122
|
+
|
123
|
+
r2 = row('tests_pkey')
|
124
|
+
r2['index_oid'].to_i.should > 0
|
125
|
+
|
126
|
+
r2['index_oid'].should == r['index_oid']
|
127
|
+
def1.should == @pgre.index_def(r2['index_oid'])
|
128
|
+
|
129
|
+
# index should be
|
130
|
+
res = @pgre.filter_relations('tests')
|
131
|
+
res.size.should == 4
|
132
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$cfg = {'adapter' => 'postgresql', 'database' => 'pgre_test', 'encoding' => 'utf8', 'username' => 'kostya', 'password' => 'password'}
|
2
|
+
ActiveRecord::Base.establish_connection $cfg
|
3
|
+
|
4
|
+
class Test < ActiveRecord::Base
|
5
|
+
self.table_name = 'tests'
|
6
|
+
end
|
7
|
+
|
8
|
+
def pg_create_schema
|
9
|
+
ActiveRecord::Migration.create_table :tests do |t|
|
10
|
+
t.integer :a
|
11
|
+
t.integer :b
|
12
|
+
t.integer :c
|
13
|
+
end
|
14
|
+
|
15
|
+
ActiveRecord::Migration.add_index :tests, :a
|
16
|
+
ActiveRecord::Migration.add_index :tests, [:b, :c]
|
17
|
+
ActiveRecord::Migration.execute "create unique index a_b_c on tests using btree(a,b,c) where a > 0 and b < 0"
|
18
|
+
end
|
19
|
+
|
20
|
+
def pg_drop_data
|
21
|
+
ActiveRecord::Migration.drop_table :tests
|
22
|
+
end
|
23
|
+
|
24
|
+
pg_drop_data rescue nil
|
25
|
+
pg_create_schema
|
metadata
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pg_reindex
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 25
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 1
|
10
|
+
version: 0.1.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Makarchev Konstantin
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-06-07 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: thor
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 3
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: pg
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 3
|
43
|
+
segments:
|
44
|
+
- 0
|
45
|
+
version: "0"
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: rspec
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
hash: 3
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
version: "0"
|
60
|
+
type: :development
|
61
|
+
version_requirements: *id003
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: rake
|
64
|
+
prerelease: false
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
hash: 3
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
type: :development
|
75
|
+
version_requirements: *id004
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: activerecord
|
78
|
+
prerelease: false
|
79
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
80
|
+
none: false
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
hash: 3
|
85
|
+
segments:
|
86
|
+
- 0
|
87
|
+
version: "0"
|
88
|
+
type: :development
|
89
|
+
version_requirements: *id005
|
90
|
+
description: Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
91
|
+
email: kostya27@gmail.com
|
92
|
+
executables:
|
93
|
+
- pgre
|
94
|
+
extensions: []
|
95
|
+
|
96
|
+
extra_rdoc_files: []
|
97
|
+
|
98
|
+
files:
|
99
|
+
- .gitignore
|
100
|
+
- Gemfile
|
101
|
+
- Gemfile.lock
|
102
|
+
- MIT-LICENSE
|
103
|
+
- README.md
|
104
|
+
- Rakefile
|
105
|
+
- bin/pgre
|
106
|
+
- lib/pg-reindex.rb
|
107
|
+
- pg_reindex.gemspec
|
108
|
+
- spec/reindex_spec.rb
|
109
|
+
- spec/spec_helper.rb
|
110
|
+
- spec/spec_support.rb
|
111
|
+
homepage: http://github.com/kostya/pg_reindex
|
112
|
+
licenses: []
|
113
|
+
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
|
117
|
+
require_paths:
|
118
|
+
- lib
|
119
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
120
|
+
none: false
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
hash: 3
|
125
|
+
segments:
|
126
|
+
- 0
|
127
|
+
version: "0"
|
128
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ">="
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
hash: 3
|
134
|
+
segments:
|
135
|
+
- 0
|
136
|
+
version: "0"
|
137
|
+
requirements: []
|
138
|
+
|
139
|
+
rubyforge_project:
|
140
|
+
rubygems_version: 1.8.24
|
141
|
+
signing_key:
|
142
|
+
specification_version: 3
|
143
|
+
summary: Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
144
|
+
test_files: []
|
145
|
+
|