prodder 1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/.travis.yml +52 -0
- data/Gemfile +16 -0
- data/Gemfile.lock +64 -0
- data/LICENSE.txt +21 -0
- data/README.md +272 -0
- data/Rakefile +20 -0
- data/bin/prodder +4 -0
- data/features/commit.feature +69 -0
- data/features/dump.feature +187 -0
- data/features/init.feature +24 -0
- data/features/lint.feature +46 -0
- data/features/prodder.feature +76 -0
- data/features/push.feature +25 -0
- data/features/step_definitions/git_steps.rb +65 -0
- data/features/step_definitions/prodder_steps.rb +150 -0
- data/features/support/blog.git.tgz +0 -0
- data/features/support/env.rb +116 -0
- data/features/support/prodder__blog_prod.sql +153 -0
- data/lib/prodder.rb +5 -0
- data/lib/prodder/cli.rb +135 -0
- data/lib/prodder/config.rb +95 -0
- data/lib/prodder/git.rb +97 -0
- data/lib/prodder/pg.rb +486 -0
- data/lib/prodder/prodder.rake +390 -0
- data/lib/prodder/project.rb +150 -0
- data/lib/prodder/railtie.rb +7 -0
- data/lib/prodder/version.rb +3 -0
- data/prodder.gemspec +25 -0
- data/spec/config_spec.rb +64 -0
- data/spec/spec_helper.rb +3 -0
- metadata +91 -0
data/lib/prodder.rb
ADDED
data/lib/prodder/cli.rb
ADDED
@@ -0,0 +1,135 @@
|
|
1
|
+
require 'prodder'
|
2
|
+
require 'thor'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
require 'pp' # TODO rm
|
6
|
+
|
7
|
+
class Prodder::CLI < Thor
|
8
|
+
include Thor::Actions
|
9
|
+
|
10
|
+
method_option :config, type: :string, aliases: '-c'
|
11
|
+
method_option :workspace, type: :string, aliases: '-w', default: File.join(Dir.pwd, 'prodder-workspace')
|
12
|
+
|
13
|
+
def initialize(*args)
|
14
|
+
super
|
15
|
+
|
16
|
+
# Help isn't printed when we don't provide --config, which is friggin absurd.
|
17
|
+
if options[:config].nil?
|
18
|
+
help
|
19
|
+
raise Thor::RequiredArgumentMissingError, "No value provided for required option '--config'"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
desc "init [*PROJECTS]", "Initialize the named projects"
|
24
|
+
def init(*projects)
|
25
|
+
select_projects(projects).each { |project| project.init }
|
26
|
+
|
27
|
+
rescue Prodder::Git::GitError => ex
|
28
|
+
puts "Failed to run '#{ex.command}':"
|
29
|
+
puts ex.error
|
30
|
+
exit 1
|
31
|
+
end
|
32
|
+
|
33
|
+
desc "dump [*PROJECTS]", "Dump production data into the named projects"
|
34
|
+
def dump(*projects)
|
35
|
+
select_projects(projects).each do |project|
|
36
|
+
project.dump
|
37
|
+
if project.git.dirty?
|
38
|
+
puts "#{project.name}: updates introduced."
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
rescue Prodder::Project::SeedConfigFileMissing => ex
|
43
|
+
puts "No such file: #{ex.filename}"
|
44
|
+
exit 1
|
45
|
+
|
46
|
+
rescue Prodder::PG::PGDumpError => ex
|
47
|
+
puts ex.message
|
48
|
+
exit 1
|
49
|
+
end
|
50
|
+
|
51
|
+
desc "commit [*PROJECTS]", "Commit prodder data changes to the named projects"
|
52
|
+
def commit(*projects)
|
53
|
+
select_projects(projects).each do |project|
|
54
|
+
if project.git.dirty?
|
55
|
+
puts "#{project.name}: committing changes."
|
56
|
+
project.commit
|
57
|
+
else
|
58
|
+
puts "#{project.name}: no changes to commit."
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
rescue Prodder::Git::GitError => ex
|
63
|
+
puts "Failed to run '#{ex.command}':"
|
64
|
+
puts ex.error
|
65
|
+
exit 1
|
66
|
+
end
|
67
|
+
|
68
|
+
desc "push [*PROJECTS]", "Push new commits to the remote repositories of the named projects"
|
69
|
+
def push(*projects)
|
70
|
+
select_projects(projects).each do |project|
|
71
|
+
if project.nothing_to_push?
|
72
|
+
puts "#{project.name}: nothing to push."
|
73
|
+
else
|
74
|
+
puts "#{project.name}: pushing new commit."
|
75
|
+
project.push
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
rescue Prodder::Git::NotFastForward => ex
|
80
|
+
puts "Refusing to push to remote #{ex.remote}: origin/master is not a fast forward from master."
|
81
|
+
exit 1
|
82
|
+
|
83
|
+
rescue Prodder::Git::GitError => ex
|
84
|
+
puts "Failed to run '#{ex.command}':"
|
85
|
+
puts ex.error
|
86
|
+
exit 1
|
87
|
+
end
|
88
|
+
|
89
|
+
desc "ls", "List all known projects"
|
90
|
+
def ls(*projects)
|
91
|
+
config.projects.each { |project| puts project.name }
|
92
|
+
end
|
93
|
+
|
94
|
+
private
|
95
|
+
|
96
|
+
def select_projects(projects)
|
97
|
+
config.select_projects(projects) do |undefined|
|
98
|
+
puts "Project not defined: #{undefined.join(', ')}"
|
99
|
+
exit 1
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def config
|
104
|
+
return @config if @config
|
105
|
+
|
106
|
+
contents = File.read options[:config]
|
107
|
+
projects = YAML.load contents
|
108
|
+
|
109
|
+
@config = Prodder::Config.new(projects).tap do |config|
|
110
|
+
config.workspace = options[:workspace]
|
111
|
+
config.lint!
|
112
|
+
end
|
113
|
+
|
114
|
+
rescue Errno::ENOENT
|
115
|
+
puts "Config file not found: #{options[:config]}"
|
116
|
+
exit 1
|
117
|
+
rescue Psych::SyntaxError
|
118
|
+
puts "Invalid YAML in config file #{options[:config]}. Current file contents:\n\n#{contents}"
|
119
|
+
exit 1
|
120
|
+
|
121
|
+
rescue Prodder::Config::PathError => ex
|
122
|
+
puts "`#{ex.message}` could not be found on your $PATH."
|
123
|
+
puts
|
124
|
+
puts "Current PATH:\n#{ENV['PATH']}"
|
125
|
+
exit 1
|
126
|
+
|
127
|
+
rescue Prodder::Config::LintError => ex
|
128
|
+
puts ex.errors.join("\n")
|
129
|
+
puts
|
130
|
+
puts "Example configuration:"
|
131
|
+
puts Prodder::Config.example_contents
|
132
|
+
exit 1
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'prodder/project'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Prodder
|
5
|
+
class Config
|
6
|
+
PathError = Class.new(StandardError)
|
7
|
+
|
8
|
+
LintError = Class.new(StandardError) do
|
9
|
+
attr_reader :errors
|
10
|
+
|
11
|
+
def initialize(errors)
|
12
|
+
@errors = errors
|
13
|
+
super errors.join "\n"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_accessor :workspace
|
18
|
+
|
19
|
+
def initialize(project_definitions)
|
20
|
+
@config = project_definitions
|
21
|
+
end
|
22
|
+
|
23
|
+
def assert_in_path(*cmds)
|
24
|
+
path = ENV['PATH'].split(File::PATH_SEPARATOR)
|
25
|
+
cmds.each do |cmd|
|
26
|
+
raise PathError.new(cmd) unless path.find { |dir| File.exist? File.join(dir, cmd) }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def lint
|
31
|
+
assert_in_path 'pg_dump'
|
32
|
+
assert_in_path 'git'
|
33
|
+
|
34
|
+
required = {
|
35
|
+
'structure_file' => [],
|
36
|
+
'seed_file' => [],
|
37
|
+
'db' => %w[name host user tables],
|
38
|
+
'git' => %w[origin author]
|
39
|
+
}
|
40
|
+
|
41
|
+
@config.each_with_object([]) do |(project, defn), errors|
|
42
|
+
required.each do |top, inner|
|
43
|
+
if !defn.key?(top)
|
44
|
+
errors << "Missing required configuration key: #{project}/#{top}"
|
45
|
+
else
|
46
|
+
errors.concat inner.reject { |key| defn[top].key?(key) }.map { |key|
|
47
|
+
"Missing required configuration key: #{project}/#{top}/#{key}"
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def lint!
|
55
|
+
lint.tap { |errors| raise LintError.new(errors) if errors.any? }
|
56
|
+
end
|
57
|
+
|
58
|
+
def projects
|
59
|
+
@projects ||= @config.map { |name, defn| Project.new(name, File.join(workspace, name), defn) }
|
60
|
+
end
|
61
|
+
|
62
|
+
def select_projects(names)
|
63
|
+
return projects if names.empty?
|
64
|
+
|
65
|
+
matches = projects.select { |project| names.include?(project.name) }
|
66
|
+
|
67
|
+
if matches.size != names.size
|
68
|
+
unmatched = names - matches.map(&:name)
|
69
|
+
yield unmatched if block_given?
|
70
|
+
end
|
71
|
+
|
72
|
+
matches
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.example_contents
|
76
|
+
<<-EOF.gsub(/^ /, '')
|
77
|
+
blog:
|
78
|
+
structure_file: db/structure.sql
|
79
|
+
seed_file: db/seeds.sql
|
80
|
+
quality_check_file: db/quality_checks.sql
|
81
|
+
git:
|
82
|
+
origin: git@github.com:your/repo.git
|
83
|
+
author: prodder <prodder@example.com>
|
84
|
+
db:
|
85
|
+
name: database_name
|
86
|
+
host: database.server.example.com
|
87
|
+
user: username
|
88
|
+
password: password
|
89
|
+
tables:
|
90
|
+
- posts
|
91
|
+
- authors
|
92
|
+
EOF
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
data/lib/prodder/git.rb
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
module Prodder
|
2
|
+
class Git
|
3
|
+
GitError = Class.new(StandardError) do
|
4
|
+
attr_reader :command, :error
|
5
|
+
def initialize(command, error); @command, @error = command, error; end
|
6
|
+
end
|
7
|
+
|
8
|
+
NotFoundError = Class.new(StandardError)
|
9
|
+
NotFastForward = Class.new(StandardError) do
|
10
|
+
attr_reader :remote
|
11
|
+
def initialize(remote); @remote = remote; end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(local, remote)
|
15
|
+
@local = local
|
16
|
+
@remote = remote
|
17
|
+
end
|
18
|
+
|
19
|
+
def clone_or_remote_update
|
20
|
+
if File.directory? File.join(@local, '.git')
|
21
|
+
remote_update
|
22
|
+
checkout 'master'
|
23
|
+
reset 'origin/master', true
|
24
|
+
else
|
25
|
+
clone
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def dirty?
|
30
|
+
inside_repo { git('status', '--porcelain') != '' }
|
31
|
+
end
|
32
|
+
|
33
|
+
def tracked?(file)
|
34
|
+
inside_repo { git('ls-files', file) != '' }
|
35
|
+
end
|
36
|
+
|
37
|
+
def no_new_commits?
|
38
|
+
inside_repo do
|
39
|
+
git('show-ref', '--hash', 'origin/master') == git('show-ref', '--hash', 'refs/heads/master')
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def fast_forward?
|
44
|
+
inside_repo do
|
45
|
+
git('merge-base', 'master', 'origin/master') == git('show-ref', '--hash', 'origin/master')
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def clone
|
50
|
+
git 'clone', @remote, @local
|
51
|
+
end
|
52
|
+
|
53
|
+
def remote_update
|
54
|
+
inside_repo { git 'remote', 'update' }
|
55
|
+
end
|
56
|
+
|
57
|
+
def checkout(branch)
|
58
|
+
inside_repo { git 'checkout', branch }
|
59
|
+
end
|
60
|
+
|
61
|
+
def reset(sharef, hard = false)
|
62
|
+
inside_repo { git 'reset', hard ? '--hard' : '', sharef }
|
63
|
+
end
|
64
|
+
|
65
|
+
def add(file)
|
66
|
+
inside_repo { git 'add', file }
|
67
|
+
end
|
68
|
+
|
69
|
+
def commit(message, author)
|
70
|
+
inside_repo { git 'commit', "--author='#{author}'", "-m", message }
|
71
|
+
end
|
72
|
+
|
73
|
+
def push
|
74
|
+
inside_repo { git 'push', 'origin', 'master:master' }
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
def inside_repo(&block)
|
80
|
+
Dir.chdir @local, &block
|
81
|
+
end
|
82
|
+
|
83
|
+
def git(*cmd, &block)
|
84
|
+
cmd = ['git', *cmd]
|
85
|
+
|
86
|
+
Open3.popen3(*cmd) do |stdin, out, err, thr|
|
87
|
+
out = out.read
|
88
|
+
err = err.read
|
89
|
+
raise GitError.new(cmd.join(' '), err) if !thr.value.success?
|
90
|
+
block.call(out, err, thr) if block
|
91
|
+
out
|
92
|
+
end
|
93
|
+
rescue Errno::ENOENT
|
94
|
+
raise NotFoundError
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/prodder/pg.rb
ADDED
@@ -0,0 +1,486 @@
|
|
1
|
+
require 'open3'
|
2
|
+
require 'pg'
|
3
|
+
|
4
|
+
module Prodder
|
5
|
+
class PG
|
6
|
+
PGDumpError = Class.new(StandardError)
|
7
|
+
|
8
|
+
attr_reader :credentials
|
9
|
+
|
10
|
+
def initialize(credentials = {})
|
11
|
+
@credentials = credentials
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_role(role, opts = [])
|
15
|
+
arguments = [
|
16
|
+
'createuser',
|
17
|
+
'--no-password',
|
18
|
+
'--no-superuser',
|
19
|
+
'--no-createrole',
|
20
|
+
'--no-createdb'
|
21
|
+
]
|
22
|
+
|
23
|
+
arguments.push *opts, role
|
24
|
+
run arguments
|
25
|
+
end
|
26
|
+
|
27
|
+
def drop_role(role)
|
28
|
+
run ['dropuser', role]
|
29
|
+
end
|
30
|
+
|
31
|
+
def create_db(db_name, sql = nil)
|
32
|
+
run ['createdb', db_name]
|
33
|
+
psql db_name, sql if sql
|
34
|
+
end
|
35
|
+
|
36
|
+
def drop_db(db_name)
|
37
|
+
run ['dropdb', db_name]
|
38
|
+
end
|
39
|
+
|
40
|
+
def psql(db_name, sql)
|
41
|
+
run ['psql', db_name], sql
|
42
|
+
end
|
43
|
+
|
44
|
+
def dump_settings(db_name, filename)
|
45
|
+
sql = <<-SQL
|
46
|
+
select unnest(setconfig)
|
47
|
+
from pg_catalog.pg_db_role_setting
|
48
|
+
join pg_database on pg_database.oid = setdatabase
|
49
|
+
-- 0 = default, for all users
|
50
|
+
where setrole = 0
|
51
|
+
and datname = '#{db_name}'
|
52
|
+
SQL
|
53
|
+
|
54
|
+
arguments = [
|
55
|
+
'-t',
|
56
|
+
'-c', sql
|
57
|
+
]
|
58
|
+
|
59
|
+
run ['psql', *arguments.push(db_name)] do |out, err, success|
|
60
|
+
raise PGDumpError.new(err) if !success
|
61
|
+
File.open(filename, 'w') do |f|
|
62
|
+
out.each_line do |setting|
|
63
|
+
f.write "ALTER DATABASE #{db_name} SET #{setting}" unless setting.gsub(/\s+/, '').empty?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def dump_structure(db_name, filename, options = {})
|
70
|
+
arguments = [
|
71
|
+
'--schema-only',
|
72
|
+
'--no-privileges',
|
73
|
+
'--no-owner',
|
74
|
+
'--host', credentials['host'],
|
75
|
+
'--username', credentials['user']
|
76
|
+
]
|
77
|
+
|
78
|
+
if options[:exclude_schemas].respond_to? :map
|
79
|
+
arguments.concat options[:exclude_schemas].map { |schema| ['--exclude-schema', schema] }.flatten
|
80
|
+
end
|
81
|
+
|
82
|
+
if options[:exclude_tables].respond_to? :map
|
83
|
+
arguments.concat options[:exclude_tables].map { |table| ['--exclude-table', table] }.flatten
|
84
|
+
end
|
85
|
+
|
86
|
+
pg_dump filename, arguments.push(db_name)
|
87
|
+
end
|
88
|
+
|
89
|
+
def dump_tables(db_name, tables, filename)
|
90
|
+
pg_dump filename, [
|
91
|
+
'--data-only',
|
92
|
+
'--no-privileges',
|
93
|
+
'--no-owner',
|
94
|
+
'--disable-triggers',
|
95
|
+
'--host', credentials['host'],
|
96
|
+
'--username', credentials['user'],
|
97
|
+
*tables.map { |table| ['--table', table] }.flatten,
|
98
|
+
db_name
|
99
|
+
]
|
100
|
+
end
|
101
|
+
|
102
|
+
def dump_permissions(db_name, filename, options = {})
|
103
|
+
perm_out_sql = ""
|
104
|
+
user_list = []
|
105
|
+
|
106
|
+
perm_out_sql << dump_db_access_control(db_name, user_list, options)
|
107
|
+
perm_out_sql.prepend pg_dumpall db_name, user_list, options
|
108
|
+
|
109
|
+
perm_out_sql.prepend(alter_role_function)
|
110
|
+
perm_out_sql.prepend(create_role_function)
|
111
|
+
perm_out_sql.prepend(granted_by_function)
|
112
|
+
perm_out_sql.prepend("SET client_min_messages TO WARNING;\n")
|
113
|
+
perm_out_sql << drop_role_create_function
|
114
|
+
perm_out_sql << drop_role_alter_function
|
115
|
+
perm_out_sql << drop_granted_by_function
|
116
|
+
|
117
|
+
File.open(filename, 'w') { |f| f.write perm_out_sql }
|
118
|
+
end
|
119
|
+
|
120
|
+
#From pg_dump
|
121
|
+
ACL_GRANT = /^GRANT /
|
122
|
+
ACL_REVOKE = /^REVOKE /
|
123
|
+
DEFAULT_PRIVILEGES = /^ALTER DEFAULT PRIVILEGES /
|
124
|
+
SET_OBJECT_OWNERSHIP = /.* OWNER TO /
|
125
|
+
SEARCH_PATH = /SET search_path = .*/
|
126
|
+
|
127
|
+
def dump_db_access_control(db_name, user_list, options)
|
128
|
+
perm_out_sql = ""
|
129
|
+
arguments = [
|
130
|
+
'--schema-only',
|
131
|
+
'--host', credentials['host'],
|
132
|
+
'--username', credentials['user']
|
133
|
+
]
|
134
|
+
|
135
|
+
if options[:exclude_schemas].respond_to? :map
|
136
|
+
arguments.concat options[:exclude_schemas].map { |schema| ['--exclude-schema', schema] }.flatten
|
137
|
+
end
|
138
|
+
|
139
|
+
if options[:exclude_tables].respond_to? :map
|
140
|
+
arguments.concat options[:exclude_tables].map { |table| ['--exclude-table', table] }.flatten
|
141
|
+
end
|
142
|
+
|
143
|
+
arguments.push db_name
|
144
|
+
|
145
|
+
white_list = options[:included_users] || []
|
146
|
+
irrelevant_login_roles = irrelevant_login_roles(db_name, white_list).map { |user| user['rolname'] }
|
147
|
+
|
148
|
+
run ['pg_dump', *arguments] do |out, err, success|
|
149
|
+
out.each_line do |line|
|
150
|
+
if line.match(ACL_GRANT) ||
|
151
|
+
line.match(ACL_REVOKE) ||
|
152
|
+
line.match(DEFAULT_PRIVILEGES) ||
|
153
|
+
line.match(SET_OBJECT_OWNERSHIP)||
|
154
|
+
line.match(SEARCH_PATH)
|
155
|
+
|
156
|
+
unless irrelevant_login_roles.include?(line.match(/ (\S*);$/)[1])
|
157
|
+
perm_out_sql << line
|
158
|
+
user_list << (line.match(/ (\S*);$/)[1]).gsub(/"/, '')
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
raise PGDumpError.new(err) if !success
|
163
|
+
end
|
164
|
+
|
165
|
+
user_list.uniq!
|
166
|
+
perm_out_sql
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def pg_conn(db_name)
|
172
|
+
conn = ::PG.connect(
|
173
|
+
dbname: db_name,
|
174
|
+
host: credentials['host'],
|
175
|
+
user: credentials['user'],
|
176
|
+
password: credentials['password']
|
177
|
+
)
|
178
|
+
|
179
|
+
res = yield(conn)
|
180
|
+
conn.close
|
181
|
+
res
|
182
|
+
end
|
183
|
+
|
184
|
+
def irrelevant_login_roles(db_name, white_list)
|
185
|
+
white_list ||= []
|
186
|
+
login_role_list = pg_conn(db_name) do |conn|
|
187
|
+
conn.exec('SELECT oid, rolname FROM pg_roles WHERE rolcanlogin AND NOT rolsuper').map do |role|
|
188
|
+
{'oid' => role['oid'], 'rolname' => role['rolname'] }
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
login_role_list.reject! { |user| white_list.include?(user['rolname']) }
|
193
|
+
login_role_list
|
194
|
+
end
|
195
|
+
|
196
|
+
def run(cmd, stdin_data = nil, &block)
|
197
|
+
# TODO use a tmp .pgpass file instead of $PGPASSWORD
|
198
|
+
env = { 'PGPASSWORD' => credentials['password'] }
|
199
|
+
Open3.popen3(env, *cmd) do |stdin, out, err, thr|
|
200
|
+
if stdin_data
|
201
|
+
stdin.write stdin_data
|
202
|
+
stdin.close
|
203
|
+
end
|
204
|
+
|
205
|
+
out, err = out.read, err.read
|
206
|
+
puts err if err
|
207
|
+
block.call(out, err, thr.value.success?) if block
|
208
|
+
out
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def pg_dump(filename, cmd)
|
213
|
+
run ['pg_dump', *cmd] do |out, err, success|
|
214
|
+
raise PGDumpError.new(err) if !success
|
215
|
+
File.open(filename, 'w') { |f| f.write out }
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
def pg_dumpall(db_name, user_list, options)
|
220
|
+
white_list = options[:included_users] || []
|
221
|
+
irrelevant_login_roles = irrelevant_login_roles(db_name, white_list).map { |user| user['oid'] }
|
222
|
+
|
223
|
+
roles_and_memberships = pg_conn(db_name) { |conn| conn.exec smart_pgdumpall(user_list, irrelevant_login_roles) }
|
224
|
+
|
225
|
+
rolcreate_sql, rolalter_sql, rolgrant_sql = "", "", ""
|
226
|
+
created_roles = Set.new
|
227
|
+
|
228
|
+
roles_and_memberships.each do |role|
|
229
|
+
unless created_roles.member? role['rolname']
|
230
|
+
created_roles << role['rolname']
|
231
|
+
tmp_sql = ""
|
232
|
+
rolcreate_sql << "SELECT * FROM create_role_if_not_exists('#{role['rolname']}');\n"
|
233
|
+
tmp_sql << "ALTER ROLE \"#{role['rolname']}\" WITH"
|
234
|
+
|
235
|
+
[
|
236
|
+
['rolsuper', 'SUPERUSER'],
|
237
|
+
['rolinherit', 'INHERIT'],
|
238
|
+
['rolcreaterole', 'CREATEROLE'],
|
239
|
+
['rolcreatedb', 'CREATEDB'],
|
240
|
+
['rolcanlogin', 'LOGIN'],
|
241
|
+
['rolreplication', 'REPLICATION']
|
242
|
+
].each do |key, modifier|
|
243
|
+
tmp_sql << if role[key].eql? 't'
|
244
|
+
" #{modifier}"
|
245
|
+
else
|
246
|
+
modifier.prepend ' NO'
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
tmp_sql << " CONNECTION LIMIT #{role['rolconnlimit']}" unless role['rolconnlimit'].eql?("-1")
|
251
|
+
tmp_sql << " VALID UNTIL '#{role['rolvaliduntil']}'" unless role['rolvaliduntil'].nil?
|
252
|
+
tmp_sql << ";\n"
|
253
|
+
tmp_sql << "COMMENT ON ROLE \"#{role['rolname']}\" IS '#{role['rolcomment']}';\n" unless role['rolcomment'].nil?
|
254
|
+
rolalter_sql << "SELECT * FROM alter_role('#{role['rolname']}', $$#{tmp_sql}$$);\n"
|
255
|
+
end
|
256
|
+
|
257
|
+
unless role['member'].nil?
|
258
|
+
tmp_sql = ""
|
259
|
+
rolgrant_sql << "GRANT \"#{role['roleid']}\" TO \"#{role['member']}\""
|
260
|
+
rolgrant_sql << " WITH ADMIN OPTION" if role['admin_option'].eql?('t')
|
261
|
+
rolgrant_sql << ";\n"
|
262
|
+
tmp_sql << "GRANT \"#{role['roleid']}\" TO \"#{role['member']}\""
|
263
|
+
tmp_sql << " WITH ADMIN OPTION" if role['admin_option'].eql?('t')
|
264
|
+
tmp_sql << " GRANTED BY #{role['grantor']}" if role['grantor'].eql?('t')
|
265
|
+
tmp_sql << ";"
|
266
|
+
rolgrant_sql << "SELECT * FROM granted_by('#{role['grantor']}', $$#{tmp_sql}$$);\n"
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
rolcreate_sql << rolalter_sql << rolgrant_sql
|
271
|
+
end
|
272
|
+
|
273
|
+
def smart_pgdumpall(user_list, irrelevant_login_roles)
|
274
|
+
irrelevant_login_roles << -1 if irrelevant_login_roles.respond_to?(:empty?) && irrelevant_login_roles.empty?
|
275
|
+
replace_bind_variables(<<-SQL, [user_list, irrelevant_login_roles])
|
276
|
+
WITH RECURSIVE memberships(roleid, member, admin_option, grantor) AS (
|
277
|
+
SELECT ur.oid AS roleid,
|
278
|
+
NULL::oid AS member,
|
279
|
+
NULL::boolean AS admin_option,
|
280
|
+
NULL::oid AS grantor
|
281
|
+
FROM pg_roles ur
|
282
|
+
WHERE ur.rolname IN (?)
|
283
|
+
UNION
|
284
|
+
SELECT COALESCE(a.roleid, r.oid) AS roleid,
|
285
|
+
a.member AS member,
|
286
|
+
a.admin_option AS admin_option,
|
287
|
+
a.grantor AS grantor
|
288
|
+
FROM pg_auth_members a
|
289
|
+
FULL OUTER JOIN pg_roles r ON FALSE
|
290
|
+
JOIN memberships
|
291
|
+
ON (memberships.roleid = a.member)
|
292
|
+
OR (memberships.roleid = r.oid OR memberships.member = r.oid)
|
293
|
+
OR (memberships.roleid = a.roleid AND COALESCE(memberships.member, 0::oid) <> a.member AND a.member NOT IN(?))
|
294
|
+
)
|
295
|
+
SELECT DISTINCT ON(ur.rolname, um.rolname)
|
296
|
+
ur.rolname AS roleid,
|
297
|
+
um.rolname AS member,
|
298
|
+
memberships.admin_option,
|
299
|
+
ug.rolname AS grantor,
|
300
|
+
ur.rolname, ur.rolsuper, ur.rolinherit,
|
301
|
+
ur.rolcreaterole, ur.rolcreatedb,
|
302
|
+
ur.rolcanlogin, ur.rolconnlimit,
|
303
|
+
ur.rolvaliduntil, ur.rolreplication,
|
304
|
+
pg_catalog.shobj_description(memberships.roleid, 'pg_authid') as rolcomment
|
305
|
+
FROM memberships
|
306
|
+
LEFT JOIN pg_roles ur on ur.oid = memberships.roleid
|
307
|
+
LEFT JOIN pg_roles um on um.oid = memberships.member
|
308
|
+
LEFT JOIN pg_roles ug on ug.oid = memberships.grantor
|
309
|
+
ORDER BY 1,2 NULLS LAST;
|
310
|
+
SQL
|
311
|
+
end
|
312
|
+
|
313
|
+
def replace_bind_variable(value)
|
314
|
+
if value.respond_to?(:map)
|
315
|
+
if value.respond_to?(:empty?) && value.empty?
|
316
|
+
quote(nil)
|
317
|
+
else
|
318
|
+
value.map { |v| quote(v) }.join(',')
|
319
|
+
end
|
320
|
+
else
|
321
|
+
quote(value)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def quote_string(s)
|
326
|
+
s.gsub(/\\/, '\&\&').gsub(/'/, "''")
|
327
|
+
end
|
328
|
+
|
329
|
+
def quote(value)
|
330
|
+
"'#{quote_string(value.to_s)}'"
|
331
|
+
end
|
332
|
+
|
333
|
+
def replace_bind_variables(statement, values)
|
334
|
+
values.each do |value|
|
335
|
+
statement.sub!(/\?/) do
|
336
|
+
replace_bind_variable(value)
|
337
|
+
end
|
338
|
+
end
|
339
|
+
statement
|
340
|
+
end
|
341
|
+
|
342
|
+
def alter_role_function
|
343
|
+
<<-SQL
|
344
|
+
CREATE OR REPLACE FUNCTION public.alter_role(rolename VARCHAR, sql TEXT)
|
345
|
+
RETURNS TEXT
|
346
|
+
AS
|
347
|
+
$alter_role$
|
348
|
+
DECLARE
|
349
|
+
r RECORD;
|
350
|
+
compensating_sql TEXT := '**!!**ALTER ROLE ';
|
351
|
+
BEGIN
|
352
|
+
EXECUTE 'SELECT rolsuper, rolinherit,
|
353
|
+
rolcreaterole, rolcreatedb,
|
354
|
+
rolcanlogin, rolconnlimit,
|
355
|
+
rolvaliduntil, rolreplication,
|
356
|
+
pg_catalog.shobj_description(oid, $1) as rolcomment
|
357
|
+
FROM pg_roles
|
358
|
+
WHERE rolname = $2'
|
359
|
+
INTO r
|
360
|
+
USING 'pg_authid', rolename;
|
361
|
+
compensating_sql := compensating_sql || '"' || rolename || '" WITH';
|
362
|
+
IF r.rolsuper THEN
|
363
|
+
compensating_sql := compensating_sql || ' SUPERUSER';
|
364
|
+
ELSE
|
365
|
+
compensating_sql := compensating_sql || ' NOSUPERUSER';
|
366
|
+
END IF;
|
367
|
+
IF r.rolinherit THEN
|
368
|
+
compensating_sql := compensating_sql || ' INHERIT';
|
369
|
+
ELSE
|
370
|
+
compensating_sql := compensating_sql || ' NOINHERIT';
|
371
|
+
END IF;
|
372
|
+
IF r.rolcreaterole THEN
|
373
|
+
compensating_sql := compensating_sql || ' CREATEROLE';
|
374
|
+
ELSE
|
375
|
+
compensating_sql := compensating_sql || ' NOCREATEROLE';
|
376
|
+
END IF;
|
377
|
+
IF r.rolcreatedb THEN
|
378
|
+
compensating_sql := compensating_sql || ' CREATEDB';
|
379
|
+
ELSE
|
380
|
+
compensating_sql := compensating_sql || ' NOCREATEDB';
|
381
|
+
END IF;
|
382
|
+
IF r.rolcanlogin THEN
|
383
|
+
compensating_sql := compensating_sql || ' LOGIN';
|
384
|
+
ELSE
|
385
|
+
compensating_sql := compensating_sql || ' NOLOGIN';
|
386
|
+
END IF;
|
387
|
+
IF r.rolreplication THEN
|
388
|
+
compensating_sql := compensating_sql || ' REPLICATION';
|
389
|
+
ELSE
|
390
|
+
compensating_sql := compensating_sql || ' NOREPLICATION';
|
391
|
+
END IF;
|
392
|
+
IF r.rolconnlimit <> -1 THEN
|
393
|
+
compensating_sql := compensating_sql || ' CONNECTION LIMIT ' || r.rolconnlimit;
|
394
|
+
END IF;
|
395
|
+
IF r.rolvaliduntil IS NOT NULL THEN
|
396
|
+
compensating_sql := compensating_sql || ' VALID UNTIL ' || r.rolvaliduntil;
|
397
|
+
END IF;
|
398
|
+
compensating_sql := compensating_sql || ';\n';
|
399
|
+
IF r.rolcomment IS NOT NULL THEN
|
400
|
+
compensating_sql := compensating_sql || 'COMMENT ON ROLE "' || rolename || '" IS ' || r.rolcomment || ';\n';
|
401
|
+
END IF;
|
402
|
+
compensating_sql := compensating_sql || '**!!**';
|
403
|
+
EXECUTE sql;
|
404
|
+
RETURN compensating_sql;
|
405
|
+
END;
|
406
|
+
$alter_role$
|
407
|
+
LANGUAGE PLPGSQL;
|
408
|
+
SQL
|
409
|
+
end
|
410
|
+
|
411
|
+
def drop_role_alter_function
|
412
|
+
<<-SQL
|
413
|
+
\n
|
414
|
+
DROP FUNCTION public.alter_role(VARCHAR, TEXT);
|
415
|
+
\n
|
416
|
+
SQL
|
417
|
+
end
|
418
|
+
|
419
|
+
def granted_by_function
|
420
|
+
<<-SQL
|
421
|
+
\n
|
422
|
+
CREATE OR REPLACE FUNCTION public.granted_by(rolename VARCHAR, sql TEXT)
|
423
|
+
RETURNS BOOLEAN
|
424
|
+
AS
|
425
|
+
$role_exists$
|
426
|
+
DECLARE
|
427
|
+
BEGIN
|
428
|
+
IF EXISTS (
|
429
|
+
SELECT *
|
430
|
+
FROM pg_catalog.pg_roles
|
431
|
+
WHERE rolname = rolename) THEN
|
432
|
+
EXECUTE sql;
|
433
|
+
RETURN TRUE;
|
434
|
+
ELSE
|
435
|
+
RAISE NOTICE 'Rolename % does not exist, cannot set granted by', rolename;
|
436
|
+
RETURN FALSE;
|
437
|
+
END IF;
|
438
|
+
END;
|
439
|
+
$role_exists$
|
440
|
+
LANGUAGE PLPGSQL;
|
441
|
+
\n
|
442
|
+
SQL
|
443
|
+
end
|
444
|
+
|
445
|
+
def drop_granted_by_function
|
446
|
+
<<-SQL
|
447
|
+
\n
|
448
|
+
DROP FUNCTION public.granted_by(VARCHAR, TEXT);
|
449
|
+
\n
|
450
|
+
SQL
|
451
|
+
end
|
452
|
+
|
453
|
+
def create_role_function
|
454
|
+
<<-SQL
|
455
|
+
\n
|
456
|
+
CREATE OR REPLACE FUNCTION public.create_role_if_not_exists(rolename VARCHAR)
|
457
|
+
RETURNS TEXT
|
458
|
+
AS
|
459
|
+
$create_role_if_not_exists$
|
460
|
+
DECLARE
|
461
|
+
BEGIN
|
462
|
+
IF NOT EXISTS (
|
463
|
+
SELECT *
|
464
|
+
FROM pg_catalog.pg_roles
|
465
|
+
WHERE rolname = rolename) THEN
|
466
|
+
EXECUTE 'CREATE ROLE ' || quote_ident(rolename) || ' ;';
|
467
|
+
RETURN '**!!**DROP ROLE ''' || rolename || ''';**!!**';
|
468
|
+
ELSE
|
469
|
+
RETURN FALSE;
|
470
|
+
END IF;
|
471
|
+
END;
|
472
|
+
$create_role_if_not_exists$
|
473
|
+
LANGUAGE PLPGSQL;
|
474
|
+
\n
|
475
|
+
SQL
|
476
|
+
end
|
477
|
+
|
478
|
+
def drop_role_create_function
|
479
|
+
<<-SQL
|
480
|
+
\n
|
481
|
+
DROP FUNCTION public.create_role_if_not_exists(VARCHAR);
|
482
|
+
\n
|
483
|
+
SQL
|
484
|
+
end
|
485
|
+
end
|
486
|
+
end
|