pg_reindex 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +47 -0
- data/MIT-LICENSE +20 -0
- data/README.md +63 -0
- data/Rakefile +8 -0
- data/bin/pgre +209 -0
- data/lib/pg-reindex.rb +193 -0
- data/pg_reindex.gemspec +26 -0
- data/spec/reindex_spec.rb +135 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/spec_support.rb +25 -0
- metadata +145 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
pg_reindex (0.1.1)
|
5
|
+
pg
|
6
|
+
thor
|
7
|
+
|
8
|
+
GEM
|
9
|
+
remote: http://rubygems.org/
|
10
|
+
specs:
|
11
|
+
activemodel (3.2.5)
|
12
|
+
activesupport (= 3.2.5)
|
13
|
+
builder (~> 3.0.0)
|
14
|
+
activerecord (3.2.5)
|
15
|
+
activemodel (= 3.2.5)
|
16
|
+
activesupport (= 3.2.5)
|
17
|
+
arel (~> 3.0.2)
|
18
|
+
tzinfo (~> 0.3.29)
|
19
|
+
activesupport (3.2.5)
|
20
|
+
i18n (~> 0.6)
|
21
|
+
multi_json (~> 1.0)
|
22
|
+
arel (3.0.2)
|
23
|
+
builder (3.0.0)
|
24
|
+
diff-lcs (1.1.3)
|
25
|
+
i18n (0.6.0)
|
26
|
+
multi_json (1.3.6)
|
27
|
+
pg (0.13.2)
|
28
|
+
rake (0.9.2.2)
|
29
|
+
rspec (2.10.0)
|
30
|
+
rspec-core (~> 2.10.0)
|
31
|
+
rspec-expectations (~> 2.10.0)
|
32
|
+
rspec-mocks (~> 2.10.0)
|
33
|
+
rspec-core (2.10.1)
|
34
|
+
rspec-expectations (2.10.0)
|
35
|
+
diff-lcs (~> 1.1.3)
|
36
|
+
rspec-mocks (2.10.1)
|
37
|
+
thor (0.15.2)
|
38
|
+
tzinfo (0.3.33)
|
39
|
+
|
40
|
+
PLATFORMS
|
41
|
+
ruby
|
42
|
+
|
43
|
+
DEPENDENCIES
|
44
|
+
activerecord
|
45
|
+
pg_reindex!
|
46
|
+
rake
|
47
|
+
rspec
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Makarchev K
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
Pg Reindex
|
2
|
+
==========
|
3
|
+
|
4
|
+
Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
5
|
+
|
6
|
+
Install:
|
7
|
+
|
8
|
+
$ gem install pg_reindex
|
9
|
+
|
10
|
+
Using:
|
11
|
+
|
12
|
+
export PGRE_CFG=/some/path/database.yml
|
13
|
+
pgre --help
|
14
|
+
|
15
|
+
User in connection should be an owner of relations. And for rebuild pkey, should be superuser.
|
16
|
+
|
17
|
+
Tasks:
|
18
|
+
|
19
|
+
pgre dbs # Show list of databases from database.yml
|
20
|
+
pgre install ENV # Install function swap_for_pkey to database
|
21
|
+
pgre rebuild ENV (table|index)[,(table|index)] # rebuild tables,indexes,pkeys
|
22
|
+
pgre tables ENV # Show tables of database
|
23
|
+
|
24
|
+
Examples:
|
25
|
+
|
26
|
+
# Show tables with indexes, which full size more than 1Gb
|
27
|
+
pgre tables production -s 1000
|
28
|
+
|
29
|
+
# Show process without really rebuild
|
30
|
+
pgre rebuild production users,some_index1,some_index2
|
31
|
+
|
32
|
+
# Rebuild indexes
|
33
|
+
pgre rebuild production users --write
|
34
|
+
|
35
|
+
# Rebuild indexes and no ask confirmation (not recommended)
|
36
|
+
pgre rebuild production users --write --no-ask
|
37
|
+
|
38
|
+
|
39
|
+
Explanation/Warning:
|
40
|
+
--------------------
|
41
|
+
|
42
|
+
Rebuild index produces sqls:
|
43
|
+
|
44
|
+
1. CREATE INDEX CONCURRENTLY bla2 on some_table USING btree (some_field);
|
45
|
+
2. ANALYZE some_table;
|
46
|
+
3. DROP INDEX bla;
|
47
|
+
4. ALTER INDEX bla2 RENAME TO bla;
|
48
|
+
|
49
|
+
2 can be blocked by long running query(LRQ), or autovacuum (in this case just kill autovacuum or wait LRQ).
|
50
|
+
By careful, if between 2 and (3,4) started LRQ or autovacuum, (3,4) can blocks all queries on this table. If this happens, and (3,4) not quit after < 30s, should stop (3,4) by cancel query in PostgreSQL. And execute (3,4) manually.
|
51
|
+
|
52
|
+
|
53
|
+
Rebuild pkey produces sqls:
|
54
|
+
|
55
|
+
1. CREATE UNIQUE INDEX CONCURRENTLY some_table_pkey2 on some_table USING btree (ID);
|
56
|
+
2. ANALYZE some_table;
|
57
|
+
3. SELECT swap_for_pkey('public', 'some_table_pkey', 'some_table_pkey2');
|
58
|
+
|
59
|
+
Same issue with 2 and 3.
|
60
|
+
|
61
|
+
|
62
|
+
|
63
|
+
MIT-LICENCE
|
data/Rakefile
ADDED
data/bin/pgre
ADDED
@@ -0,0 +1,209 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'thor'
|
5
|
+
require 'yaml'
|
6
|
+
require File.dirname(__FILE__) + '/../lib/pg-reindex'
|
7
|
+
|
8
|
+
class PgReindexer < Thor
|
9
|
+
|
10
|
+
desc "dbs", "Show list of databases from database.yml (-n if no show sizes)"
|
11
|
+
method_option :no_size, :aliases => '-n', :type => :boolean
|
12
|
+
def dbs
|
13
|
+
list = db_list(!options[:no_size])
|
14
|
+
return unless list
|
15
|
+
|
16
|
+
render_db_list(list)
|
17
|
+
end
|
18
|
+
|
19
|
+
desc "tables ENV", "Show tables of database"
|
20
|
+
method_option :size, :aliases => '-s', :type => :string
|
21
|
+
def tables(env)
|
22
|
+
conn = get_connection(env)
|
23
|
+
relations = conn.get_struct_relations(options[:size])
|
24
|
+
render_tables(env, relations)
|
25
|
+
end
|
26
|
+
|
27
|
+
desc "rebuild ENV (table|index)[,(table|index)]", "rebuild tables,indexes,pkeys"
|
28
|
+
method_option :write, :aliases => '--write', :type => :boolean
|
29
|
+
method_option :no_ask, :aliases => '--no-ask', :type => :boolean
|
30
|
+
def rebuild(env, to_reindex = '')
|
31
|
+
if !to_reindex || to_reindex.empty?
|
32
|
+
say "blank attributes to rebuild", :red
|
33
|
+
exit 1
|
34
|
+
end
|
35
|
+
|
36
|
+
conn = get_connection(env)
|
37
|
+
|
38
|
+
rebuild = conn.filter_relations(to_reindex)
|
39
|
+
|
40
|
+
rebuild.each do |row|
|
41
|
+
do_rebuild conn, row, options[:write], !options[:no_ask]
|
42
|
+
end
|
43
|
+
|
44
|
+
if rebuild.empty?
|
45
|
+
say "nothing to rebuild", :red
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
desc "install ENV", "Install need function to database, like function: swap_for_pkey"
|
50
|
+
def install(env)
|
51
|
+
conn = get_connection(env)
|
52
|
+
conn.install_swap_for_pkey
|
53
|
+
say "DONE!", :green
|
54
|
+
end
|
55
|
+
|
56
|
+
desc "queries ENV", "Show current queries on database ENV"
|
57
|
+
def queries(env)
|
58
|
+
conn = get_connection(env)
|
59
|
+
q = conn.queries
|
60
|
+
r = q.map do |query|
|
61
|
+
[query['procpid'], query['age'], query['current_query']]
|
62
|
+
end
|
63
|
+
|
64
|
+
print_table r
|
65
|
+
end
|
66
|
+
|
67
|
+
desc "cancel ENV PID", "Cancel pid query"
|
68
|
+
def cancel(env, pid)
|
69
|
+
conn = get_connection(env)
|
70
|
+
conn.cancel(pid)
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
|
75
|
+
def get_connection(env)
|
76
|
+
conn = connection(env)
|
77
|
+
unless conn
|
78
|
+
say "Not connected to #{env}, hm!!!", :red
|
79
|
+
exit 1
|
80
|
+
end
|
81
|
+
|
82
|
+
conn
|
83
|
+
end
|
84
|
+
|
85
|
+
def connection(env)
|
86
|
+
cn = connect(env, db_config[env])
|
87
|
+
end
|
88
|
+
|
89
|
+
def connect(env, conf)
|
90
|
+
@connects ||= {}
|
91
|
+
@connects[env] ||= PgReindex.new(conf) if conf
|
92
|
+
end
|
93
|
+
|
94
|
+
# load database.yml
|
95
|
+
def db_config
|
96
|
+
@db_config ||= begin
|
97
|
+
file = ENV['PGRE_CFG'] || `pwd`.to_s.chop + "/config/database.yml"
|
98
|
+
if File.exists?(file)
|
99
|
+
YAML.load_file(file)
|
100
|
+
else
|
101
|
+
say "Specify path to database.yml with env CFG, like: CFG=/../../database.yml", :red
|
102
|
+
exit 1
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def db_list(show_sizes = true)
|
108
|
+
cfg = db_config
|
109
|
+
return unless cfg
|
110
|
+
|
111
|
+
cfg = cfg.sort_by{|a| a[0]}
|
112
|
+
cfg.map do |env, config|
|
113
|
+
x = [env, config]
|
114
|
+
x << (connection(env).database_size(config['database']) rescue '-') if show_sizes
|
115
|
+
x
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def do_rebuild(conn, row, write = true, ask = true)
|
120
|
+
#ask = false unless write
|
121
|
+
|
122
|
+
say "=> rebuild #{row['index']} (#{row['i_size_p']}), write #{write}, ask #{ask}", :yellow
|
123
|
+
|
124
|
+
sqls = conn.index_sqls(row)
|
125
|
+
|
126
|
+
if ask || !write
|
127
|
+
sqls.each do |sql|
|
128
|
+
say " #{sql}"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
if ask
|
133
|
+
r = ask_until("Rebuild? (y/n)")
|
134
|
+
if r == '' || r == 'y' || r == 'Y'
|
135
|
+
else
|
136
|
+
say " ... skipped\n", :red
|
137
|
+
return
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
if row['index'].end_with?("_pkey")
|
142
|
+
unless conn.check_swap_for_pkey
|
143
|
+
say " ... You should install function swap_for_pkey for rebuild pkey, call: #{$0} install ENV", :red
|
144
|
+
return
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
started_at = Time.now
|
149
|
+
|
150
|
+
if write
|
151
|
+
sqls.each do |query|
|
152
|
+
say " => ", :bold, false
|
153
|
+
say "#{query}", nil, false
|
154
|
+
tm = Time.now
|
155
|
+
|
156
|
+
conn.exec(query)
|
157
|
+
|
158
|
+
say " (#{"%.3f" % (Time.now - tm)}s)"
|
159
|
+
end
|
160
|
+
|
161
|
+
new_size = conn.get_index_size(row['index'])['i_size_p'] rescue '-'
|
162
|
+
say "<= rebuilded #{row['index']} (#{new_size})", :yellow
|
163
|
+
say " DONE! (#{"%.3f" % (Time.now - started_at)}s)\n", :green
|
164
|
+
else
|
165
|
+
say " DRY-RUN DONE\n", :green
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def render_db_list(db_list)
|
170
|
+
say "Databases list:", :green
|
171
|
+
db_list.each do |row|
|
172
|
+
say "\t#{row[0].ljust(30)} ", :yellow, false
|
173
|
+
say "#{row[1]['database']}:#{row[1]['host']}".ljust(60)
|
174
|
+
say "#{row[2]}"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def render_tables(env, relations)
|
179
|
+
off1 = 65
|
180
|
+
off2 = 15
|
181
|
+
off3 = 15
|
182
|
+
|
183
|
+
say "tables for #{env.ljust(off1 - 9)}size#{''.ljust(off2 - 4)}indexes_size", :green
|
184
|
+
|
185
|
+
relations.each do |table, indexes|
|
186
|
+
say " #{table.ljust(off1)}", :yellow, false
|
187
|
+
f = indexes[0]
|
188
|
+
say "#{f['size_p'].ljust(off2)}", :yellow, false
|
189
|
+
say "#{f['total_i_size_p'].ljust(off3)}", :yellow, true
|
190
|
+
#say "#{f['total_size_p']}", :yellow
|
191
|
+
|
192
|
+
|
193
|
+
indexes.each do |rel|
|
194
|
+
say " #{(rel['index']+" ").ljust(off1 - 2 + off2, '.')}", nil, false
|
195
|
+
say "#{rel['i_size_p']}"
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def ask_until(str, answers = ['y', 'n', ''])
|
201
|
+
loop do
|
202
|
+
res = ask(str)
|
203
|
+
return res if answers.include?(res)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
end
|
208
|
+
|
209
|
+
PgReindexer.start
|
data/lib/pg-reindex.rb
ADDED
@@ -0,0 +1,193 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
class PgReindex
|
4
|
+
|
5
|
+
MIN_SIZE = 50 # megabyte min, total table size
|
6
|
+
|
7
|
+
def initialize(conf)
|
8
|
+
cfg = {:host => conf['host'] || '127.0.0.1', :dbname => conf['database'],
|
9
|
+
:user => conf['username'] || `whoami`.chop, :password => conf['password'] || 'password', :port => conf['port'].to_s}
|
10
|
+
|
11
|
+
@conn = PGconn.new cfg
|
12
|
+
end
|
13
|
+
|
14
|
+
def get_struct_relations(min_size = nil)
|
15
|
+
res = get_raw_relations
|
16
|
+
|
17
|
+
result = {}
|
18
|
+
res.each do |row|
|
19
|
+
next if row['total_size'].to_i < ((min_size || MIN_SIZE).to_f * 1024 * 1024).to_i
|
20
|
+
result[row['table']] ||= []
|
21
|
+
result[row['table']] << row
|
22
|
+
end
|
23
|
+
|
24
|
+
result = result.sort_by{|el| el[1][0]['total_size'].to_i}.reverse
|
25
|
+
|
26
|
+
result = result.map do |table, res|
|
27
|
+
[table, res]
|
28
|
+
end
|
29
|
+
|
30
|
+
result
|
31
|
+
end
|
32
|
+
|
33
|
+
def filter_relations(filter) # filter is a string with ,
|
34
|
+
return [] if !filter || filter.empty?
|
35
|
+
|
36
|
+
filter = filter.split(",")
|
37
|
+
|
38
|
+
get_raw_relations.select do |row|
|
39
|
+
filter.include?(row['index']) || filter.include?(row['table'])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def get_raw_relations
|
44
|
+
res = @conn.exec <<-SQL
|
45
|
+
SELECT C.relname AS "table",
|
46
|
+
i.relname "index",
|
47
|
+
i.oid "index_oid",
|
48
|
+
pg_relation_size(C.oid) AS "size",
|
49
|
+
pg_total_relation_size(C.oid) AS "total_size",
|
50
|
+
pg_size_pretty(pg_relation_size(C.oid)) AS "size_p",
|
51
|
+
pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size_p",
|
52
|
+
pg_size_pretty(pg_total_relation_size(C.oid) - pg_relation_size(C.oid)) AS "total_i_size_p",
|
53
|
+
pg_relation_size(i.oid) as "i_size",
|
54
|
+
pg_size_pretty(pg_relation_size(i.oid)) as "i_size_p"
|
55
|
+
FROM pg_class C, pg_class i, pg_index ix, pg_namespace N
|
56
|
+
WHERE nspname IN ('public') AND
|
57
|
+
C.oid = ix.indrelid and i.oid = ix.indexrelid
|
58
|
+
AND C.oid = ix.indrelid and i.oid = ix.indexrelid
|
59
|
+
AND C.relname not like 'pg_%'
|
60
|
+
AND N.oid = C.relnamespace
|
61
|
+
ORDER BY c.relname, i.relname
|
62
|
+
SQL
|
63
|
+
end
|
64
|
+
|
65
|
+
def get_index_size(relation_name)
|
66
|
+
res = @conn.exec "SELECT pg_relation_size(oid) as i_size, pg_size_pretty(pg_relation_size(oid)) as i_size_p
|
67
|
+
from pg_class WHERE relname = E'#{relation_name}'"
|
68
|
+
res[0]
|
69
|
+
rescue
|
70
|
+
{}
|
71
|
+
end
|
72
|
+
|
73
|
+
def install_swap_for_pkey
|
74
|
+
@conn.exec("create language plpgsql") rescue nil
|
75
|
+
@conn.exec(swap_for_pkey_sql)
|
76
|
+
end
|
77
|
+
|
78
|
+
def check_swap_for_pkey
|
79
|
+
res = @conn.exec <<-SQL
|
80
|
+
SELECT proname FROM pg_catalog.pg_namespace n JOIN pg_catalog.pg_proc p ON pronamespace = n.oid
|
81
|
+
WHERE nspname = 'public' and proname = 'swap_for_pkey'
|
82
|
+
SQL
|
83
|
+
|
84
|
+
res.values.size == 1
|
85
|
+
end
|
86
|
+
|
87
|
+
def swap_for_pkey_sql
|
88
|
+
<<-SQL
|
89
|
+
CREATE OR REPLACE FUNCTION swap_for_pkey(text,text,text) returns integer
|
90
|
+
AS
|
91
|
+
$$
|
92
|
+
DECLARE
|
93
|
+
cmd text;
|
94
|
+
oid1 integer;
|
95
|
+
oid2 integer;
|
96
|
+
filenode1 integer;
|
97
|
+
filenode2 integer;
|
98
|
+
relation text;
|
99
|
+
BEGIN
|
100
|
+
select oid::integer into oid1 from pg_class where relname=$2 and relnamespace = (select oid from pg_namespace where nspname=$1);
|
101
|
+
RAISE NOTICE 'PKEY OID: %',oid1;
|
102
|
+
select relfilenode::integer into filenode1 from pg_class where oid=oid1;
|
103
|
+
select oid::integer into oid2 from pg_class where relname=$3 and relnamespace = (select oid from pg_namespace where nspname=$1);
|
104
|
+
RAISE NOTICE 'PKEY OID: %',oid2;
|
105
|
+
select relfilenode::integer into filenode2 from pg_class where oid=oid2;
|
106
|
+
select (indrelid::regclass)::text into relation from pg_index where indexrelid=oid1;
|
107
|
+
RAISE NOTICE 'RELATION NAME: %',relation;
|
108
|
+
cmd:='LOCK '||relation||';';
|
109
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
110
|
+
Execute cmd;
|
111
|
+
cmd:='UPDATE pg_class SET relfilenode='||filenode2|| ' WHERE oid='||oid1||';';
|
112
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
113
|
+
Execute cmd;
|
114
|
+
cmd:='UPDATE pg_class SET relfilenode='||filenode1|| ' WHERE oid='||oid2||';';
|
115
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
116
|
+
Execute cmd;
|
117
|
+
cmd:='DROP INDEX '||$1||'.'||$3||';';
|
118
|
+
RAISE NOTICE 'Executing :- %',cmd;
|
119
|
+
Execute cmd;
|
120
|
+
return 0;
|
121
|
+
END;
|
122
|
+
$$language plpgsql;
|
123
|
+
SQL
|
124
|
+
end
|
125
|
+
|
126
|
+
def database_size(database)
|
127
|
+
res = @conn.exec("SELECT pg_size_pretty(pg_database_size('#{database}')) as db_size")
|
128
|
+
res.first['db_size']
|
129
|
+
end
|
130
|
+
|
131
|
+
def index_sqls(index_row)
|
132
|
+
index_sql_array(index_row['table'], index_row['index_oid'], index_row['index'])
|
133
|
+
end
|
134
|
+
|
135
|
+
def index_sql_array(table, oid, name)
|
136
|
+
new_name = if name.size < 61
|
137
|
+
name + "_2"
|
138
|
+
else
|
139
|
+
name[0..-3] + "_2"
|
140
|
+
end
|
141
|
+
|
142
|
+
if name.end_with?('_pkey')
|
143
|
+
[
|
144
|
+
index_sql(table, oid, name, new_name),
|
145
|
+
"ANALYZE #{table}",
|
146
|
+
"SELECT swap_for_pkey('public', '#{name}', '#{new_name}')"
|
147
|
+
]
|
148
|
+
else
|
149
|
+
[
|
150
|
+
index_sql(table, oid, name, new_name),
|
151
|
+
"ANALYZE #{table}",
|
152
|
+
"DROP INDEX #{name}",
|
153
|
+
"ALTER INDEX #{new_name} RENAME TO #{name}"
|
154
|
+
]
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def index_sql(table, oid, name, new_name)
|
159
|
+
str = index_def(oid).gsub(name, new_name)
|
160
|
+
|
161
|
+
pos = str.index(new_name)
|
162
|
+
|
163
|
+
before = str[0..pos-2]
|
164
|
+
after = str[pos..-1]
|
165
|
+
|
166
|
+
"#{before} CONCURRENTLY #{after}"
|
167
|
+
end
|
168
|
+
|
169
|
+
def index_def(oid)
|
170
|
+
sql = "SELECT pg_get_indexdef(#{oid}) as q"
|
171
|
+
exec(sql)[0]['q']
|
172
|
+
end
|
173
|
+
|
174
|
+
def exec(sql)
|
175
|
+
@conn.exec sql
|
176
|
+
end
|
177
|
+
|
178
|
+
def queries
|
179
|
+
@conn.exec <<-SQL
|
180
|
+
SELECT procpid, now() - query_start as age, current_query
|
181
|
+
FROM
|
182
|
+
pg_stat_activity AS a JOIN
|
183
|
+
pg_locks AS l ON a.procpid = l.pid
|
184
|
+
WHERE
|
185
|
+
virtualxid IS NOT NULL order by age desc;
|
186
|
+
SQL
|
187
|
+
end
|
188
|
+
|
189
|
+
def cancel(procpid)
|
190
|
+
@conn.exec "select pg_cancel_backend(#{procpid.to_i})"
|
191
|
+
end
|
192
|
+
|
193
|
+
end
|
data/pg_reindex.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{pg_reindex}
|
5
|
+
s.version = "0.1.1"
|
6
|
+
|
7
|
+
s.authors = ["Makarchev Konstantin"]
|
8
|
+
|
9
|
+
s.description = %q{Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.}
|
10
|
+
s.summary = %q{Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.}
|
11
|
+
|
12
|
+
s.email = %q{kostya27@gmail.com}
|
13
|
+
s.homepage = %q{http://github.com/kostya/pg_reindex}
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.add_dependency 'thor'
|
21
|
+
s.add_dependency 'pg'
|
22
|
+
s.add_development_dependency "rspec"
|
23
|
+
s.add_development_dependency "rake"
|
24
|
+
s.add_development_dependency 'activerecord'
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe PgReindex do
|
4
|
+
before :each do
|
5
|
+
@pgre = PgReindex.new($cfg)
|
6
|
+
end
|
7
|
+
|
8
|
+
def row(name)
|
9
|
+
@pgre.filter_relations(name).first
|
10
|
+
end
|
11
|
+
|
12
|
+
it "get_raw_relations" do
|
13
|
+
res = @pgre.get_raw_relations
|
14
|
+
v = []
|
15
|
+
res.each{|h| v << h }
|
16
|
+
|
17
|
+
v.map{|el| el['table']}.uniq.should == ['tests']
|
18
|
+
v.map{|el| el['index']}.uniq.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
19
|
+
v.map{|el| el['index_oid']}.sum.to_i.should > 0
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "filter_relations" do
|
23
|
+
it "1" do
|
24
|
+
res = @pgre.filter_relations('a_b_c')
|
25
|
+
res.size.should == 1
|
26
|
+
res[0]['index'].should == 'a_b_c'
|
27
|
+
end
|
28
|
+
|
29
|
+
it "2" do
|
30
|
+
res = @pgre.filter_relations('a_b_c,index_tests_on_a,tests_pkey,xxx')
|
31
|
+
res.size.should == 3
|
32
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "tests_pkey"]
|
33
|
+
end
|
34
|
+
|
35
|
+
it "3" do
|
36
|
+
res = @pgre.filter_relations('tests,xxx')
|
37
|
+
res.size.should == 4
|
38
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
39
|
+
end
|
40
|
+
|
41
|
+
it "4" do
|
42
|
+
res = @pgre.filter_relations('xxx')
|
43
|
+
res.size.should == 0
|
44
|
+
end
|
45
|
+
|
46
|
+
it "5" do
|
47
|
+
res = @pgre.filter_relations('tests,a_b_c')
|
48
|
+
res.size.should == 4
|
49
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
it "database_size" do
|
54
|
+
s = @pgre.database_size('pgre_test')
|
55
|
+
s.to_i.should > 1
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "index sqls" do
|
59
|
+
it "index sqls for a" do
|
60
|
+
@pgre.index_sqls(row('index_tests_on_a')).should == ["CREATE INDEX CONCURRENTLY index_tests_on_a_2 ON tests USING btree (a)",
|
61
|
+
"ANALYZE tests", "DROP INDEX index_tests_on_a", "ALTER INDEX index_tests_on_a_2 RENAME TO index_tests_on_a"]
|
62
|
+
end
|
63
|
+
|
64
|
+
it "index sqls for b,c" do
|
65
|
+
@pgre.index_sqls(row('index_tests_on_b_and_c')).should == ["CREATE INDEX CONCURRENTLY index_tests_on_b_and_c_2 ON tests USING btree (b, c)",
|
66
|
+
"ANALYZE tests", "DROP INDEX index_tests_on_b_and_c", "ALTER INDEX index_tests_on_b_and_c_2 RENAME TO index_tests_on_b_and_c"]
|
67
|
+
end
|
68
|
+
|
69
|
+
it "index sqls for a,b,c" do
|
70
|
+
@pgre.index_sqls(row('a_b_c')).should == ["CREATE UNIQUE INDEX CONCURRENTLY a_b_c_2 ON tests USING btree (a, b, c) WHERE ((a > 0) AND (b < 0))",
|
71
|
+
"ANALYZE tests", "DROP INDEX a_b_c", "ALTER INDEX a_b_c_2 RENAME TO a_b_c"]
|
72
|
+
end
|
73
|
+
|
74
|
+
it "index sqls for pkey" do
|
75
|
+
@pgre.index_sqls(row('tests_pkey')).should == ["CREATE UNIQUE INDEX CONCURRENTLY tests_pkey_2 ON tests USING btree (id)",
|
76
|
+
"ANALYZE tests", "SELECT swap_for_pkey('public', 'tests_pkey', 'tests_pkey_2')"]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
it "index def" do
|
81
|
+
r = row('a_b_c')
|
82
|
+
@pgre.index_def(r['index_oid']).should == "CREATE UNIQUE INDEX a_b_c ON tests USING btree (a, b, c) WHERE ((a > 0) AND (b < 0))"
|
83
|
+
end
|
84
|
+
|
85
|
+
it "rebuilds index" do
|
86
|
+
r = row('a_b_c')
|
87
|
+
r['index_oid'].to_i.should > 0
|
88
|
+
def1 = @pgre.index_def(r['index_oid'])
|
89
|
+
|
90
|
+
sqls = @pgre.index_sqls(r)
|
91
|
+
sqls.each do |sql|
|
92
|
+
@pgre.exec sql
|
93
|
+
end
|
94
|
+
|
95
|
+
r2 = row('a_b_c')
|
96
|
+
r2['index_oid'].to_i.should > 0
|
97
|
+
|
98
|
+
r2['index_oid'].should_not == r['index_oid']
|
99
|
+
r2['index'].should == r['index']
|
100
|
+
|
101
|
+
def2 = @pgre.index_def(r2['index_oid'])
|
102
|
+
def1.should == def2
|
103
|
+
|
104
|
+
# index should be
|
105
|
+
res = @pgre.filter_relations('tests')
|
106
|
+
res.size.should == 4
|
107
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
108
|
+
end
|
109
|
+
|
110
|
+
it "rebuilds pkey" do
|
111
|
+
# install swap_for_pkey
|
112
|
+
@pgre.install_swap_for_pkey
|
113
|
+
|
114
|
+
r = row('tests_pkey')
|
115
|
+
r['index_oid'].to_i.should > 0
|
116
|
+
def1 = @pgre.index_def(r['index_oid'])
|
117
|
+
|
118
|
+
sqls = @pgre.index_sqls(r)
|
119
|
+
sqls.each do |sql|
|
120
|
+
@pgre.exec sql
|
121
|
+
end
|
122
|
+
|
123
|
+
r2 = row('tests_pkey')
|
124
|
+
r2['index_oid'].to_i.should > 0
|
125
|
+
|
126
|
+
r2['index_oid'].should == r['index_oid']
|
127
|
+
def1.should == @pgre.index_def(r2['index_oid'])
|
128
|
+
|
129
|
+
# index should be
|
130
|
+
res = @pgre.filter_relations('tests')
|
131
|
+
res.size.should == 4
|
132
|
+
res.map{|el| el['index']}.sort.should == ["a_b_c", "index_tests_on_a", "index_tests_on_b_and_c", "tests_pkey"]
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
$cfg = {'adapter' => 'postgresql', 'database' => 'pgre_test', 'encoding' => 'utf8', 'username' => 'kostya', 'password' => 'password'}
|
2
|
+
ActiveRecord::Base.establish_connection $cfg
|
3
|
+
|
4
|
+
class Test < ActiveRecord::Base
|
5
|
+
self.table_name = 'tests'
|
6
|
+
end
|
7
|
+
|
8
|
+
def pg_create_schema
|
9
|
+
ActiveRecord::Migration.create_table :tests do |t|
|
10
|
+
t.integer :a
|
11
|
+
t.integer :b
|
12
|
+
t.integer :c
|
13
|
+
end
|
14
|
+
|
15
|
+
ActiveRecord::Migration.add_index :tests, :a
|
16
|
+
ActiveRecord::Migration.add_index :tests, [:b, :c]
|
17
|
+
ActiveRecord::Migration.execute "create unique index a_b_c on tests using btree(a,b,c) where a > 0 and b < 0"
|
18
|
+
end
|
19
|
+
|
20
|
+
def pg_drop_data
|
21
|
+
ActiveRecord::Migration.drop_table :tests
|
22
|
+
end
|
23
|
+
|
24
|
+
pg_drop_data rescue nil
|
25
|
+
pg_create_schema
|
metadata
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pg_reindex
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 25
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
- 1
|
10
|
+
version: 0.1.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Makarchev Konstantin
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-06-07 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: thor
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 3
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: pg
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 3
|
43
|
+
segments:
|
44
|
+
- 0
|
45
|
+
version: "0"
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: rspec
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
hash: 3
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
version: "0"
|
60
|
+
type: :development
|
61
|
+
version_requirements: *id003
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: rake
|
64
|
+
prerelease: false
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
hash: 3
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
type: :development
|
75
|
+
version_requirements: *id004
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: activerecord
|
78
|
+
prerelease: false
|
79
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
80
|
+
none: false
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
hash: 3
|
85
|
+
segments:
|
86
|
+
- 0
|
87
|
+
version: "0"
|
88
|
+
type: :development
|
89
|
+
version_requirements: *id005
|
90
|
+
description: Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
91
|
+
email: kostya27@gmail.com
|
92
|
+
executables:
|
93
|
+
- pgre
|
94
|
+
extensions: []
|
95
|
+
|
96
|
+
extra_rdoc_files: []
|
97
|
+
|
98
|
+
files:
|
99
|
+
- .gitignore
|
100
|
+
- Gemfile
|
101
|
+
- Gemfile.lock
|
102
|
+
- MIT-LICENSE
|
103
|
+
- README.md
|
104
|
+
- Rakefile
|
105
|
+
- bin/pgre
|
106
|
+
- lib/pg-reindex.rb
|
107
|
+
- pg_reindex.gemspec
|
108
|
+
- spec/reindex_spec.rb
|
109
|
+
- spec/spec_helper.rb
|
110
|
+
- spec/spec_support.rb
|
111
|
+
homepage: http://github.com/kostya/pg_reindex
|
112
|
+
licenses: []
|
113
|
+
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
|
117
|
+
require_paths:
|
118
|
+
- lib
|
119
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
120
|
+
none: false
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
hash: 3
|
125
|
+
segments:
|
126
|
+
- 0
|
127
|
+
version: "0"
|
128
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ">="
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
hash: 3
|
134
|
+
segments:
|
135
|
+
- 0
|
136
|
+
version: "0"
|
137
|
+
requirements: []
|
138
|
+
|
139
|
+
rubyforge_project:
|
140
|
+
rubygems_version: 1.8.24
|
141
|
+
signing_key:
|
142
|
+
specification_version: 3
|
143
|
+
summary: Console utility for gracefully rebuild indexes/pkeys for PostgreSQL, with minimal locking in semi-auto mode.
|
144
|
+
test_files: []
|
145
|
+
|