debloater 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 33b62165f1a67989f3ba01274484d44adf38ec35
4
+ data.tar.gz: 841d4684a2f920552621faf5a17bdc043756e30a
5
+ SHA512:
6
+ metadata.gz: 0cc3f9c0af2e80101fe7fb8aeb3571ac707f11332adbbcd91f0669a6da21c8f777f2364ad7465193a5782cfcef529a4869f3319aa78446c95e7d5a2eb781f917
7
+ data.tar.gz: 3c6745026be290c0df30697fcc0c957fdbc616d285b92ac4d49837122e70a48718b2e1f32fca266daae0e233bf9a4971320b531f62c045b32e79b1fd7ac185fd
@@ -0,0 +1,12 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+
11
+ # rspec failure tracking
12
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
@@ -0,0 +1 @@
1
+ 2.4.1
@@ -0,0 +1,13 @@
1
+ dist: trusty
2
+ sudo: true
3
+ addons:
4
+ postgresql: "9.6"
5
+ language: ruby
6
+ rvm:
7
+ - 2.4.1
8
+ before_install:
9
+ - gem install bundler
10
+ before_script:
11
+ - psql --version
12
+ - psql -c 'create database debloater_test;' -U postgres
13
+
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in debloater.gemspec
4
+ gemspec
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 Julien Letessier
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,76 @@
1
+ # debloater
2
+
3
+ Safely rebuilds PostgreSQL indices on a live database.
4
+
5
+
6
+ ## Installation
7
+
8
+ With a recent version of Ruby installed, run:
9
+
10
+ gem install debloater
11
+
12
+
13
+ ## Usage
14
+
15
+ If you run `debloater` without arguments, you should read:
16
+
17
+ ```
18
+ Usage: debloater [options] database
19
+ -h HOST Database host to connect to [localhost]
20
+ -p PORT Port to connect to [5432]
21
+ -U USER Username to connect with [postgres]
22
+ -W Prompt for password (default)
23
+ -w No prompt for password
24
+ --auto Do not ask for confirmation before debloating
25
+ --min-mb [SIZE] Do not debloat if the bloat size is lower than SIZE megabytes [50]
26
+ --max-density [FRACTION] Do not debloat if the index density is higher than FRACTION [0.75]
27
+ --help Prints this help
28
+ ```
29
+
30
+ ### Caveats
31
+
32
+ The `pgstattuple` extension is required; install it if asked with:
33
+
34
+ ```sql
35
+ CREATE EXTENSION pgstattuple;
36
+ ```
37
+
38
+ Within this extension, permission to run `pgstatindex()` is required; this may
39
+ be problematic on some platforms, e.g. Amazon RDS. `debloater` will fall back to
40
+ a function called `get_pgstatindex()` with the same profile, which you can
41
+ create with the following script (run as an administrator):
42
+
43
+ ```sql
44
+ CREATE OR REPLACE FUNCTION get_pgstatindex(
45
+ relname regclass,
46
+ OUT index_size bigint,
47
+ OUT avg_leaf_density float8,
48
+ OUT leaf_fragmentation float8
49
+ )
50
+ AS $$
51
+ BEGIN
52
+ SELECT i.index_size, i.avg_leaf_density, i.leaf_fragmentation
53
+ FROM pgstatindex(relname) i
54
+ INTO index_size, avg_leaf_density, leaf_fragmentation;
55
+ END;
56
+ $$ LANGUAGE plpgsql
57
+ VOLATILE
58
+ SECURITY DEFINER;
59
+ ```
60
+
61
+
62
+ ## Development
63
+
64
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
65
+
66
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
67
+
68
+ ## Contributing
69
+
70
+ Bug reports and pull requests are welcome on GitHub at https://github.com/deliveroo/debloater.
71
+
72
+
73
+ ## License
74
+
75
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
76
+
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "debloater"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,30 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'debloater/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'debloater'
8
+ spec.version = Debloater::VERSION
9
+ spec.authors = ['Julien Letessier']
10
+ spec.email = ['julien.letessier@gmail.com']
11
+
12
+ spec.summary = %q{Safely rebuilds bloated PostgreSQL indices}
13
+ spec.description = spec.summary
14
+ spec.homepage = 'https://github.com/deliveroo/debloater'
15
+ spec.license = 'MIT'
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ end
20
+ spec.bindir = 'exe'
21
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
+ spec.require_paths = ['lib']
23
+
24
+ spec.add_development_dependency 'bundler', '~> 1.14'
25
+ spec.add_development_dependency 'rake', '~> 10.0'
26
+ spec.add_development_dependency 'rspec', '~> 3.0'
27
+ spec.add_development_dependency 'pry'
28
+
29
+ spec.add_runtime_dependency 'pg'
30
+ end
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'debloater/cli'
4
+
5
+ Debloater::CLI.new(ARGV).run
@@ -0,0 +1,5 @@
1
+ require "debloater/version"
2
+
3
+ module Debloater
4
+ # Your code goes here...
5
+ end
@@ -0,0 +1,95 @@
1
+ require 'io/console'
2
+ require 'optionparser'
3
+ require 'debloater/connection'
4
+ require 'debloater/engine'
5
+
6
+ module Debloater
7
+ class CLI
8
+ DEFAULTS = {
9
+ connection: {
10
+ host: 'localhost',
11
+ port: 5432,
12
+ user: 'postgres',
13
+ password: nil,
14
+ dbname: nil,
15
+ },
16
+ engine: {
17
+ confirm: true,
18
+ min_mb: 50,
19
+ max_density: 0.75,
20
+ },
21
+ prompt_password: true,
22
+ }
23
+
24
+ def initialize(argv)
25
+ @options = _parse(argv.dup)
26
+ end
27
+
28
+ def run
29
+ conn = Connection.new(@options[:connection])
30
+ Engine.new(conn, @options[:engine]).run
31
+ end
32
+
33
+ private
34
+
35
+ def _parse(argv)
36
+ options = DEFAULTS.dup
37
+ parser = OptionParser.new do |opts|
38
+ opts.banner = "Usage: debloater [options] database"
39
+
40
+ opts.on('-h HOST', 'Database host to connect to [localhost]') do |v|
41
+ options[:connection][:host] = v
42
+ end
43
+
44
+ opts.on('-p PORT', 'Port to connect to [5432]') do |v|
45
+ options[:connection][:port] = v
46
+ end
47
+
48
+ opts.on('-U USER', 'Username to connect with [postgres]') do |v|
49
+ options[:connection][:user] = v
50
+ end
51
+
52
+ opts.on('-W', 'Prompt for password (default)') do
53
+ options[:prompt_password] = true
54
+ end
55
+
56
+ opts.on('-w', 'No prompt for password') do
57
+ options[:prompt_password] = false
58
+ end
59
+
60
+ opts.on('--auto', 'Do not ask for confirmation before debloating') do |v|
61
+ options[:engine][:confirm] = false
62
+ end
63
+
64
+ opts.on('--min-mb [SIZE]', 'Do not debloat if the bloat size is lower than SIZE megabytes [50]') do |v|
65
+ options[:engine][:min_mb] = v.to_f
66
+ end
67
+
68
+ opts.on('--max-density [FRACTION]', 'Do not debloat if the index density is higher than FRACTION [0.75]') do |v|
69
+ options[:engine][:min_mb] = v.to_f
70
+ end
71
+
72
+ opts.on('--help', 'Prints this help') do
73
+ puts opts
74
+ exit
75
+ end
76
+ end
77
+
78
+ parser.parse!(argv)
79
+
80
+ case argv.length
81
+ when 1 then
82
+ options[:connection][:dbname] = argv.pop
83
+ else
84
+ puts parser
85
+ exit 1
86
+ end
87
+
88
+ if options[:prompt_password]
89
+ options[:connection][:password] = IO.console.getpass('Enter password (no echo):')
90
+ end
91
+
92
+ options
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,61 @@
1
+ require 'pg'
2
+ require 'forwardable'
3
+ require 'debloater/helpers'
4
+
5
+ module Debloater
6
+ class Connection
7
+ include Helpers
8
+ extend Forwardable
9
+
10
+ def initialize(options)
11
+ @pg = PG::Connection.open(options)
12
+
13
+ exec %{
14
+ SET statement_timeout = 0;
15
+ }
16
+ end
17
+
18
+ def_delegators :@pg, :exec, :exec_params
19
+
20
+ def statindex_method
21
+ @_guessed_index_method ||=
22
+ if _check_pgstatindex
23
+ method = :pgstatindex
24
+ elsif _check_get_pgstatindex
25
+ method = :get_pgstatindex
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def _check_pgstatindex
32
+ exec %{
33
+ select * from pgstatindex(0);
34
+ }
35
+ rescue PG::UndefinedFunction => e
36
+ _fatal e, msg: 'The `pgstattuple` extension is missing. Consult the README.'
37
+ rescue PG::InternalError
38
+ # expected, as there is no index with OID zero
39
+ true
40
+ rescue PG::InsufficientPrivilege
41
+ _log "`pgstatindex` not permitted, falling back to get_pgstatindex"
42
+ false
43
+ rescue PG::Error => e
44
+ _fatal e
45
+ end
46
+
47
+ def _check_get_pgstatindex
48
+ exec %{
49
+ select * from get_pgstatindex(0);
50
+ }
51
+ rescue PG::UndefinedFunction => e
52
+ _fatal e, msg: 'The `get_pgstatindex` function is not found. Consult the README.'
53
+ rescue PG::InternalError
54
+ # expected, as there is no index with OID zero
55
+ true
56
+ rescue PG::Error => e
57
+ _fatal e
58
+ end
59
+
60
+ end
61
+ end
@@ -0,0 +1,53 @@
1
+ require 'pry'
2
+ require 'debloater/helpers'
3
+ require 'debloater/index'
4
+
5
+ module Debloater
6
+ class Engine
7
+ include Helpers
8
+
9
+ def initialize(conn, confirm: true, min_mb: 50, max_density: 0.75)
10
+ @conn = conn
11
+ @confirm = confirm
12
+ @min_mb = min_mb
13
+ @max_density = max_density
14
+ end
15
+
16
+
17
+ def run
18
+ debloated_indices = []
19
+ Index.each(@conn) do |index|
20
+ if index.pkey?
21
+ _log "Skipping primary key index '#{index.name}'"
22
+ next
23
+ end
24
+
25
+ unless index.valid?
26
+ _log "Skipping invalid (non-Btree) index '#{index.name}'"
27
+ next
28
+ end
29
+
30
+ puts format('%<name>-65s %<size>10d %<frag>3d pct %<bloat>6d MB' % {
31
+ name: index.name,
32
+ size: index.size,
33
+ frag: (100 - index.density * 100).to_i,
34
+ bloat: index.bloat,
35
+ })
36
+
37
+ if index.bloat < @min_mb || index.density > @max_density
38
+ _log "Skipping non-bloated index '#{index.name}'"
39
+ next
40
+ end
41
+
42
+ if @confirm
43
+ $stderr.write "debloat this index? [y/N] "
44
+ next unless $stdin.gets.strip =~ /^y$/i
45
+ end
46
+
47
+ index.debloat!
48
+ debloated_indices << index
49
+ end
50
+ debloated_indices
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,16 @@
1
+ module Debloater
2
+ module Helpers
3
+ def _log(line)
4
+ $stderr.write("#{line}\n")
5
+ end
6
+
7
+ def _fatal(e, msg: nil)
8
+ _log "Fatal error:"
9
+ _log msg if msg
10
+
11
+ _log "Exception: #{e.class.name}"
12
+ _log "Details: #{e.message}"
13
+ exit 1
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,105 @@
1
+ require 'debloater/helpers'
2
+
3
+ module Debloater
4
+ class Index
5
+ include Helpers
6
+
7
+ # conn: connection object
8
+ # method: the function used to obtain index information
9
+ # name: the index name
10
+ def initialize(conn, name:, sql:)
11
+ @conn = conn
12
+ @name = name
13
+ @sql = sql
14
+ end
15
+
16
+ attr_reader :name
17
+
18
+ def self.each(conn)
19
+ data = conn.exec_params %{
20
+ select * from pg_indexes where schemaname = $1
21
+ }, ['public']
22
+ data.sort_by { |d|
23
+ d['indexname']
24
+ }.each { |d|
25
+ next if d['indexname'] =~ /idx_debloat/
26
+ yield new(conn, name: d['indexname'], sql: d['indexdef'])
27
+ }
28
+ rescue PG::Error => e
29
+ _fatal e, msg: 'Could not list indexes'
30
+ end
31
+
32
+
33
+ def pkey?
34
+ @name =~ /pkey/
35
+ end
36
+
37
+
38
+ def valid?
39
+ !_metadata.nil?
40
+ end
41
+
42
+
43
+ def density
44
+ @density ||= 0.01 * _metadata['avg_leaf_density'].to_f
45
+ end
46
+
47
+
48
+ def bloat
49
+ @bloat ||= size * (1 - density)
50
+ end
51
+
52
+
53
+ def size
54
+ @size ||= _metadata['index_size'].to_i / 1024**2
55
+ end
56
+
57
+
58
+ def debloat!
59
+ [
60
+ "DROP INDEX IF EXISTS idx_debloat_new",
61
+ @sql.
62
+ sub(/CONCURRENTLY/i, '').
63
+ sub(/CREATE (UNIQUE)? INDEX/i, 'CREATE \1 INDEX CONCURRENTLY').
64
+ sub(@name, 'idx_debloat_new'),
65
+ 'BEGIN',
66
+ "ALTER INDEX #{@name} RENAME TO idx_debloat_old",
67
+ "ALTER INDEX idx_debloat_new RENAME TO #{@name}",
68
+ 'DROP INDEX idx_debloat_old',
69
+ 'COMMIT',
70
+ ].each do |sql|
71
+ _log "\t#{sql}"
72
+ begin
73
+ @conn.exec sql
74
+ rescue PG::DependentObjectsStillExist => e
75
+ _log "Could not debloat '#{@name}', skipping. Details:"
76
+ _log e.message
77
+ rescue PG::Error => e
78
+ _fatal e, msg: "Failure during index debloating, you may want to delete the temporary index 'idx_debloat_new' manually."
79
+ end
80
+ end
81
+ end
82
+
83
+
84
+ private
85
+
86
+
87
+ def _metadata
88
+ return @_metadata if defined?(@_metadata)
89
+
90
+ @_metadata = @conn.exec(%{
91
+ select * from #{@conn.statindex_method}('#{@name}');
92
+ }).to_a.first
93
+ rescue PG::InvalidName
94
+ _log "Invalid named index '#{@name}'"
95
+ @_metadata = nil
96
+ rescue PG::Error => e
97
+ if e.message =~ /is not a btree index/
98
+ _log "Invalid non-Btree index '#{@name}'"
99
+ @_metadata = nil
100
+ else
101
+ _fatal e, msg: "Could not fetch metadata for index '#{@name}'"
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,3 @@
1
+ module Debloater
2
+ VERSION = "0.1.0"
3
+ end
metadata ADDED
@@ -0,0 +1,134 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: debloater
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Julien Letessier
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2017-06-21 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.14'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.14'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: pg
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description: Safely rebuilds bloated PostgreSQL indices
84
+ email:
85
+ - julien.letessier@gmail.com
86
+ executables:
87
+ - debloater
88
+ extensions: []
89
+ extra_rdoc_files: []
90
+ files:
91
+ - ".gitignore"
92
+ - ".rspec"
93
+ - ".ruby-version"
94
+ - ".travis.yml"
95
+ - Gemfile
96
+ - LICENSE.txt
97
+ - README.md
98
+ - Rakefile
99
+ - bin/console
100
+ - bin/setup
101
+ - debloater.gemspec
102
+ - exe/debloater
103
+ - lib/debloater.rb
104
+ - lib/debloater/cli.rb
105
+ - lib/debloater/connection.rb
106
+ - lib/debloater/engine.rb
107
+ - lib/debloater/helpers.rb
108
+ - lib/debloater/index.rb
109
+ - lib/debloater/version.rb
110
+ homepage: https://github.com/deliveroo/debloater
111
+ licenses:
112
+ - MIT
113
+ metadata: {}
114
+ post_install_message:
115
+ rdoc_options: []
116
+ require_paths:
117
+ - lib
118
+ required_ruby_version: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: '0'
123
+ required_rubygems_version: !ruby/object:Gem::Requirement
124
+ requirements:
125
+ - - ">="
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ requirements: []
129
+ rubyforge_project:
130
+ rubygems_version: 2.6.11
131
+ signing_key:
132
+ specification_version: 4
133
+ summary: Safely rebuilds bloated PostgreSQL indices
134
+ test_files: []