forty 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/forty/acl.rb +27 -0
- data/lib/forty/configuration.rb +38 -0
- data/lib/forty/database.rb +25 -0
- data/lib/forty/dbms.rb +1 -0
- data/lib/forty/redshift/privilege.rb +49 -0
- data/lib/forty/sync.rb +724 -0
- data/lib/forty.rb +1 -0
- metadata +50 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 97efc5c8cb7262d1dfb61406f68497e31d49d90d
|
4
|
+
data.tar.gz: 393e67307c31c8273709d61bff8b6428186925ce
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: de46378f836e770d1d02f9e7319691ffc08dd59165255fd09440b86cba41f23e786674d2b84f18d2aa007ccdafaada6410b867bcb4bb7e41f3aeb011b29529cd
|
7
|
+
data.tar.gz: f866784cdcbce84f15f36943e0d188eaf731bdea969f0043ea2839040431c39fa518781399a24ca493bb18f4330af4f326d55101243e974f6c0f565813f9a489
|
data/lib/forty/acl.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Forty
|
4
|
+
class ACL
|
5
|
+
def initialize(path_to_acl_file)
|
6
|
+
raise('no path to ACL file provided') if path_to_acl_file.nil? or path_to_acl_file.empty?
|
7
|
+
|
8
|
+
if File.exist?(path_to_acl_file)
|
9
|
+
begin
|
10
|
+
@acl = JSON.parse(File.read(path_to_acl_file))
|
11
|
+
rescue StandardError
|
12
|
+
raise "ACL file #{path_to_acl_file} could not be parsed"
|
13
|
+
end
|
14
|
+
else
|
15
|
+
raise("ACL file not found at: #{path_to_acl_file}")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def [](key)
|
20
|
+
@acl[key]
|
21
|
+
end
|
22
|
+
|
23
|
+
def []=(key, value)
|
24
|
+
@acl[key] = value
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Forty
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :logger
|
4
|
+
attr_accessor :master_username
|
5
|
+
attr_accessor :schemas
|
6
|
+
attr_accessor :acl_file
|
7
|
+
attr_accessor :generate_passwords
|
8
|
+
end
|
9
|
+
|
10
|
+
class Database
|
11
|
+
attr_accessor :host
|
12
|
+
attr_accessor :port
|
13
|
+
attr_accessor :user
|
14
|
+
attr_accessor :password
|
15
|
+
attr_accessor :database
|
16
|
+
end
|
17
|
+
|
18
|
+
class << self
|
19
|
+
attr_writer :configuration
|
20
|
+
attr_writer :database_configuration
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.configuration
|
24
|
+
@configuration ||= Forty::Configuration.new
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.configure
|
28
|
+
yield(configuration)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.database_configuration
|
32
|
+
@database ||= Forty::Database.new
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.database
|
36
|
+
yield(database_configuration)
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
module Forty
|
4
|
+
class Database
|
5
|
+
attr_accessor :host
|
6
|
+
attr_accessor :port
|
7
|
+
attr_accessor :user
|
8
|
+
attr_accessor :password
|
9
|
+
attr_accessor :database
|
10
|
+
|
11
|
+
def execute(statement)
|
12
|
+
@db ||= PG.connect(
|
13
|
+
host: self.host,
|
14
|
+
port: self.port,
|
15
|
+
user: self.user,
|
16
|
+
password: self.password,
|
17
|
+
dbname: self.database
|
18
|
+
)
|
19
|
+
|
20
|
+
@db.exec(statement)
|
21
|
+
end
|
22
|
+
|
23
|
+
alias_method :dwh, :execute
|
24
|
+
end
|
25
|
+
end
|
data/lib/forty/dbms.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require_relative 'redshift/privilege'
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Forty
|
2
|
+
module Redshift
|
3
|
+
module Privilege
|
4
|
+
class Base
|
5
|
+
PRIVILEGES = self.constants.map { |const| self.const_get(const) }
|
6
|
+
|
7
|
+
def self.get_privilege_name_by_acronym(acronym)
|
8
|
+
privilege = self.constants.select do |constant|
|
9
|
+
self.const_get(constant).eql?(acronym)
|
10
|
+
end[0]
|
11
|
+
privilege.nil? ? nil : privilege.to_s.downcase
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.parse_privileges_from_string(privileges_string)
|
15
|
+
privileges = []
|
16
|
+
self.constants.each do |constant|
|
17
|
+
acronym = self.const_get(constant)
|
18
|
+
unless privileges_string.slice!(acronym).nil?
|
19
|
+
privileges << self.get_privilege_name_by_acronym(acronym)
|
20
|
+
end
|
21
|
+
break if privileges_string.empty?
|
22
|
+
end
|
23
|
+
privileges
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class Table < Base
|
28
|
+
ALL = 'arwdRxt'
|
29
|
+
SELECT = 'r'
|
30
|
+
UPDATE = 'w'
|
31
|
+
INSERT = 'a'
|
32
|
+
DELETE = 'd'
|
33
|
+
REFERENCES = 'x'
|
34
|
+
end
|
35
|
+
|
36
|
+
class Schema < Base
|
37
|
+
ALL = 'UC'
|
38
|
+
USAGE = 'U'
|
39
|
+
CREATE = 'C'
|
40
|
+
end
|
41
|
+
|
42
|
+
class Database < Base
|
43
|
+
ALL = 'CT'
|
44
|
+
CREATE = 'C'
|
45
|
+
TEMPORARY = 'T'
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/forty/sync.rb
ADDED
@@ -0,0 +1,724 @@
|
|
1
|
+
# require_relative 'configuration'
|
2
|
+
|
3
|
+
module Forty
|
4
|
+
def self.sync
|
5
|
+
Forty::Sync.new(
|
6
|
+
Forty.configuration.logger,
|
7
|
+
Forty.configuration.master_username,
|
8
|
+
Forty.configuration.schemas,
|
9
|
+
Forty::ACL.new(Forty.configuration.acl_file),
|
10
|
+
Forty.instance_variable_get(:@database),
|
11
|
+
false
|
12
|
+
).run
|
13
|
+
end
|
14
|
+
|
15
|
+
class Sync
|
16
|
+
class Error < StandardError; end
|
17
|
+
|
18
|
+
def initialize(logger, master_username, production_schemas, acl_config, executor, dry_run=true)
|
19
|
+
@logger = logger or raise Error, 'No logger provided'
|
20
|
+
@master_username = master_username or raise Error, 'No master username provided'
|
21
|
+
@production_schemas = production_schemas or raise Error, 'No production schemas provided'
|
22
|
+
@acl_config = acl_config or raise Error, 'No acl config provided'
|
23
|
+
@acl_config['users'] ||= {}
|
24
|
+
@acl_config['groups'] ||= {}
|
25
|
+
|
26
|
+
@executor = executor or raise Error, 'No dwh executor provided'
|
27
|
+
@dry_run = dry_run
|
28
|
+
|
29
|
+
@logger.warn('Dry mode disabled, executing on production') unless @dry_run
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
sync_users()
|
34
|
+
sync_groups()
|
35
|
+
sync_user_groups()
|
36
|
+
sync_user_roles()
|
37
|
+
sync_acl()
|
38
|
+
end
|
39
|
+
|
40
|
+
def sync_users
|
41
|
+
current_users = _get_current_dwh_users.keys
|
42
|
+
defined_users = @acl_config['users'].keys
|
43
|
+
|
44
|
+
undefined_users = (current_users - defined_users).uniq.compact
|
45
|
+
missing_users = (defined_users - current_users).uniq.compact
|
46
|
+
|
47
|
+
undefined_users.each { |user| _delete_user(user) }
|
48
|
+
|
49
|
+
missing_users.each do |user|
|
50
|
+
roles = @acl_config['users'][user]['roles'] || []
|
51
|
+
password = @acl_config['users'][user]['password']
|
52
|
+
search_path = @production_schemas.join(',')
|
53
|
+
|
54
|
+
_create_user(user, password, roles, search_path)
|
55
|
+
end
|
56
|
+
|
57
|
+
@logger.info('All users are in sync') if (undefined_users.count + missing_users.count) == 0
|
58
|
+
end
|
59
|
+
|
60
|
+
def sync_groups
|
61
|
+
current_groups = _get_current_dwh_groups().keys
|
62
|
+
defined_groups = @acl_config['groups'].keys
|
63
|
+
|
64
|
+
undefined_groups = (current_groups - defined_groups).uniq.compact
|
65
|
+
missing_groups = (defined_groups - current_groups).uniq.compact
|
66
|
+
|
67
|
+
undefined_groups.each { |group| _delete_group(group) }
|
68
|
+
missing_groups.each { |group| _create_group(group) }
|
69
|
+
|
70
|
+
@logger.info('All groups are in sync') if (undefined_groups.count + missing_groups.count) == 0
|
71
|
+
end
|
72
|
+
|
73
|
+
def sync_user_groups
|
74
|
+
current_user_groups = _get_current_user_groups()
|
75
|
+
defined_user_groups = _get_defined_user_groups()
|
76
|
+
_check_group_unknown(current_user_groups.keys, defined_user_groups.keys)
|
77
|
+
|
78
|
+
current_users = _get_current_dwh_users().keys
|
79
|
+
defined_users = _get_defined_users()
|
80
|
+
_check_user_unknown(current_users, defined_users)
|
81
|
+
|
82
|
+
diverged = 0
|
83
|
+
|
84
|
+
current_user_groups.each do |group, list|
|
85
|
+
current_list = list
|
86
|
+
defined_list = defined_user_groups[group] || []
|
87
|
+
|
88
|
+
undefined_assignments = (current_list - defined_list).uniq.compact
|
89
|
+
missing_assignments = (defined_list - current_list).uniq.compact
|
90
|
+
|
91
|
+
undefined_assignments.each { |user| _remove_user_from_group(user, group) }
|
92
|
+
missing_assignments.each { |user| _add_user_to_group(user, group) }
|
93
|
+
|
94
|
+
current_group_diverged = (undefined_assignments.count + missing_assignments.count)
|
95
|
+
diverged += current_group_diverged
|
96
|
+
|
97
|
+
@logger.debug("Users of group #{group} are in sync") if current_group_diverged == 0
|
98
|
+
end
|
99
|
+
|
100
|
+
@logger.info('All user groups are in sync') if diverged == 0
|
101
|
+
end
|
102
|
+
|
103
|
+
def sync_personal_schemas
|
104
|
+
users = @acl_config['users'].keys
|
105
|
+
users.each do |user|
|
106
|
+
next if user.eql?(@master_username)
|
107
|
+
schemas_owned_by_user = _get_currently_owned_schemas(user).uniq - @production_schemas
|
108
|
+
unless schemas_owned_by_user.empty?
|
109
|
+
tables_owned_by_user = _get_currently_owned_tables(user)
|
110
|
+
schemas_owned_by_user.each do |schema|
|
111
|
+
@executor.dwh("set search_path=#{schema}")
|
112
|
+
tables = @executor.dwh("select tablename from pg_tables where schemaname='#{schema}'").map { |row| "#{schema}.#{row['tablename']}" }
|
113
|
+
nonowned_tables_by_user = tables.uniq - tables_owned_by_user
|
114
|
+
nonowned_tables_by_user.each { |table| _execute_statement("alter table #{table} owner to #{user};") }
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def sync_user_roles
|
121
|
+
defined_user_roles = _get_defined_user_roles()
|
122
|
+
current_user_roles = _get_current_user_roles()
|
123
|
+
|
124
|
+
users = ((defined_user_roles.keys).concat(current_user_roles.keys)).uniq.compact
|
125
|
+
|
126
|
+
diverged = 0
|
127
|
+
|
128
|
+
users.each do |user|
|
129
|
+
next if user.eql?(@master_username)
|
130
|
+
|
131
|
+
raise Error, "Users are not in sync #{user}" if current_user_roles[user].nil? or defined_user_roles[user].nil?
|
132
|
+
|
133
|
+
undefined_roles = (current_user_roles[user] - defined_user_roles[user]).uniq.compact
|
134
|
+
missing_roles = (defined_user_roles[user] - current_user_roles[user]).uniq.compact
|
135
|
+
|
136
|
+
current_roles_diverged = (undefined_roles.count + missing_roles.count)
|
137
|
+
diverged += current_roles_diverged
|
138
|
+
|
139
|
+
undefined_roles.each { |role| _execute_statement("alter user #{user} no#{role};") }
|
140
|
+
missing_roles.each { |role| _execute_statement("alter user #{user} #{role};") }
|
141
|
+
|
142
|
+
@logger.debug("Roles of #{user} are in sync") if current_roles_diverged == 0
|
143
|
+
end
|
144
|
+
|
145
|
+
@logger.info('All user roles are in sync') if diverged == 0
|
146
|
+
end
|
147
|
+
|
148
|
+
def sync_acl
|
149
|
+
sync_database_acl()
|
150
|
+
sync_schema_acl()
|
151
|
+
sync_table_acl()
|
152
|
+
end
|
153
|
+
|
154
|
+
def sync_database_acl
|
155
|
+
current_database_acl = _get_current_database_acl()
|
156
|
+
defined_database_acl = _get_defined_database_acl()
|
157
|
+
|
158
|
+
diverged = _sync_typed_acl('database', current_database_acl, defined_database_acl)
|
159
|
+
@logger.info('All database privileges are in sync') if diverged == 0
|
160
|
+
end
|
161
|
+
|
162
|
+
def sync_schema_acl
|
163
|
+
current_schema_acl = _get_current_schema_acl()
|
164
|
+
defined_schema_acl = _get_defined_schema_acl()
|
165
|
+
|
166
|
+
diverged = _sync_typed_acl('schema', current_schema_acl, defined_schema_acl)
|
167
|
+
@logger.info('All schema privileges are in sync') if diverged == 0
|
168
|
+
end
|
169
|
+
|
170
|
+
def sync_table_acl
|
171
|
+
current_table_acl = _get_current_table_acl()
|
172
|
+
defined_table_acl = _get_defined_table_acl()
|
173
|
+
|
174
|
+
diverged = _sync_typed_acl('table', current_table_acl, defined_table_acl)
|
175
|
+
@logger.info('All table privileges are in sync') if diverged == 0
|
176
|
+
end
|
177
|
+
|
178
|
+
private
|
179
|
+
|
180
|
+
def _get_defined_user_groups
|
181
|
+
Hash[@acl_config['groups'].map do |group, _|
|
182
|
+
[group, @acl_config['users'].select do |_, data|
|
183
|
+
groups = data['groups'] || []
|
184
|
+
groups.include?(group)
|
185
|
+
end.keys]
|
186
|
+
end]
|
187
|
+
end
|
188
|
+
|
189
|
+
def _get_current_user_groups
|
190
|
+
current_groups = _get_current_dwh_groups()
|
191
|
+
current_users = _get_current_dwh_users().invert
|
192
|
+
|
193
|
+
current_user_groups = Hash[current_groups.map { |group, list| [group, list.map { |id| current_users[id] }]}]
|
194
|
+
current_user_groups
|
195
|
+
end
|
196
|
+
|
197
|
+
def _get_defined_user_roles
|
198
|
+
Hash[@acl_config['users'].map do |user, config|
|
199
|
+
user_groups = @acl_config['groups'].select { |group| (config['groups'] || []).include?(group) }
|
200
|
+
user_roles = config['roles'] || []
|
201
|
+
user_groups.each do |_, group_config|
|
202
|
+
user_roles.concat(group_config['roles']) if group_config['roles'].is_a?(Array)
|
203
|
+
end
|
204
|
+
[user, user_roles.uniq]
|
205
|
+
end]
|
206
|
+
end
|
207
|
+
|
208
|
+
def _get_current_user_roles
|
209
|
+
Hash[@executor.dwh(<<-SQL).map { |row| [row['usename'], row['user_roles'].split(',').select { |e| not e.empty? }.compact] }]
|
210
|
+
select
|
211
|
+
usename
|
212
|
+
, case when usecreatedb is true then 'createdb' else '' end
|
213
|
+
|| ',' ||
|
214
|
+
case when usesuper is true then 'createuser' else '' end
|
215
|
+
as user_roles
|
216
|
+
from pg_user
|
217
|
+
where usename != 'rdsdb'
|
218
|
+
order by usename
|
219
|
+
;
|
220
|
+
SQL
|
221
|
+
end
|
222
|
+
|
223
|
+
def _check_group_unknown(current_groups, defined_groups)
|
224
|
+
raise Error, 'Groups are out of sync!' if _mismatch?(current_groups, defined_groups)
|
225
|
+
end
|
226
|
+
|
227
|
+
def _check_user_unknown(current_users, defined_users)
|
228
|
+
raise Error, 'Users are out of sync!' if _mismatch?(current_users, defined_users)
|
229
|
+
end
|
230
|
+
|
231
|
+
def _mismatch?(current, defined)
|
232
|
+
mismatch_count = 0
|
233
|
+
mismatch_count += (current - defined).count
|
234
|
+
mismatch_count += (defined - current).count
|
235
|
+
mismatch_count > 0 ? true : false
|
236
|
+
end
|
237
|
+
|
238
|
+
def _execute_statement(statement)
|
239
|
+
attempts = 0
|
240
|
+
@logger.info(statement.sub(/(password\s+')(?:[^\s]+)(')/, '\1\2'))
|
241
|
+
if @dry_run === false
|
242
|
+
begin
|
243
|
+
@logger.info("Retrying to execute statement in #{attempts*10} seconds...") if attempts > 0
|
244
|
+
sleep (attempts*10)
|
245
|
+
attempts += 1
|
246
|
+
@executor.dwh(statement)
|
247
|
+
rescue PG::UndefinedTable => e
|
248
|
+
@logger.error("#{e.class}: #{e.message}" )
|
249
|
+
retry unless attempts > 3
|
250
|
+
raise Error, 'Maximum number of attempts exceeded, giving up'
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
def _create_group(group)
|
256
|
+
_execute_statement("create group #{group};")
|
257
|
+
end
|
258
|
+
|
259
|
+
def _delete_group(group)
|
260
|
+
full_group_name = "group #{group}"
|
261
|
+
|
262
|
+
acl = {
|
263
|
+
'table' => _get_current_table_acl()[full_group_name],
|
264
|
+
'schema' => _get_current_schema_acl()[full_group_name],
|
265
|
+
'database' => _get_current_database_acl()[full_group_name]
|
266
|
+
}
|
267
|
+
|
268
|
+
acl.each do |type, acl|
|
269
|
+
unless acl.nil? or acl.empty?
|
270
|
+
acl.each do |identifier, permissions|
|
271
|
+
_revoke_privileges(full_group_name, type, identifier, permissions)
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
_execute_statement("drop group #{group};")
|
277
|
+
end
|
278
|
+
|
279
|
+
def _create_user(user, password, roles=[], search_path=nil)
|
280
|
+
_execute_statement("create user #{user} with password '#{password}' #{roles.join(' ')};")
|
281
|
+
|
282
|
+
unless search_path.nil? or search_path.empty?
|
283
|
+
_execute_statement("alter user #{user} set search_path to #{search_path};")
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def _generate_password
|
288
|
+
begin
|
289
|
+
password = SecureRandom.base64.gsub(/[^a-zA-Z0-9]/, '')
|
290
|
+
raise 'Not valid' unless password.match(/[a-z]/) && password.match(/[A-Z]/) && password.match(/[0-9]/)
|
291
|
+
rescue
|
292
|
+
retry
|
293
|
+
else
|
294
|
+
password
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
def _delete_user(user)
|
299
|
+
raise Error, 'Please define the master user in the ACL file!' if user.eql?(@master_username)
|
300
|
+
|
301
|
+
schemas_owned_by_user = _get_currently_owned_schemas(user)
|
302
|
+
tables_owned_by_user = _get_currently_owned_tables(user)
|
303
|
+
|
304
|
+
_resolve_object_ownership_upon_user_deletion(schemas_owned_by_user, tables_owned_by_user)
|
305
|
+
_revoke_all_privileges(user)
|
306
|
+
|
307
|
+
_execute_statement("drop user #{user};")
|
308
|
+
end
|
309
|
+
|
310
|
+
def _resolve_object_ownership_upon_user_deletion(schemas, tables)
|
311
|
+
non_production_tables = tables.select { |table| !@production_schemas.include?(table.split('.')[0]) }
|
312
|
+
production_tables = tables.select { |table| @production_schemas.include?(table.split('.')[0]) }
|
313
|
+
|
314
|
+
non_production_tables.each { |table| _execute_statement("drop table #{table};") }
|
315
|
+
production_tables.each { |table| _execute_statement("alter table #{table} owner to #{@master_username};") }
|
316
|
+
|
317
|
+
non_production_schemas = (schemas - @production_schemas)
|
318
|
+
production_schemas = schemas.select { |schema| @production_schemas.include?(schema) }
|
319
|
+
|
320
|
+
non_production_schemas.each { |schema| _execute_statement("drop schema #{schema} cascade;") }
|
321
|
+
production_schemas.each { |schema| _execute_statement("alter schema #{schema} owner to #{@master_username};") }
|
322
|
+
end
|
323
|
+
|
324
|
+
def _revoke_all_privileges(grantee)
|
325
|
+
(_get_current_table_acl[grantee] || {}).each do |name, privileges|
|
326
|
+
_revoke_privileges(grantee, 'table', name, privileges)
|
327
|
+
end
|
328
|
+
|
329
|
+
(_get_current_schema_acl[grantee] || {}).each do |name, privileges|
|
330
|
+
_revoke_privileges(grantee, 'schema', name, privileges)
|
331
|
+
end
|
332
|
+
|
333
|
+
(_get_current_database_acl[grantee] || {}).each do |name, privileges|
|
334
|
+
_revoke_privileges(grantee, 'database', name, privileges)
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
def _add_user_to_group(user, group)
|
339
|
+
_execute_statement("alter group #{group} add user #{user};")
|
340
|
+
end
|
341
|
+
|
342
|
+
def _remove_user_from_group(user, group)
|
343
|
+
_execute_statement("alter group #{group} drop user #{user};")
|
344
|
+
end
|
345
|
+
|
346
|
+
def _get_current_dwh_users
|
347
|
+
query = <<-SQL
|
348
|
+
select distinct
|
349
|
+
usename as name
|
350
|
+
, usesysid as id
|
351
|
+
from pg_user
|
352
|
+
where usename != 'rdsdb'
|
353
|
+
;
|
354
|
+
SQL
|
355
|
+
|
356
|
+
raw_dwh_users = @executor.dwh(query)
|
357
|
+
|
358
|
+
Hash[raw_dwh_users.map do |row|
|
359
|
+
name = row['name']
|
360
|
+
id = row['id'].to_i
|
361
|
+
|
362
|
+
[name, id]
|
363
|
+
end]
|
364
|
+
end
|
365
|
+
|
366
|
+
def _get_defined_users
|
367
|
+
@acl_config['users'].keys
|
368
|
+
end
|
369
|
+
|
370
|
+
def _get_current_dwh_groups
|
371
|
+
query = <<-SQL
|
372
|
+
select distinct
|
373
|
+
groname as name
|
374
|
+
, array_to_string(grolist, ',') as user_list
|
375
|
+
from pg_group
|
376
|
+
;
|
377
|
+
SQL
|
378
|
+
raw_dwh_groups = @executor.dwh(query)
|
379
|
+
|
380
|
+
Hash[raw_dwh_groups.map do |row|
|
381
|
+
name = row['name']
|
382
|
+
user_ids = row['user_list'].to_s.split(',').map { |id| id.to_i }
|
383
|
+
|
384
|
+
[name, user_ids]
|
385
|
+
end]
|
386
|
+
end
|
387
|
+
|
388
|
+
def _get_current_schema_acl
|
389
|
+
query = <<-SQL
|
390
|
+
select
|
391
|
+
nspname as name
|
392
|
+
, array_to_string(nspacl, ',') as acls
|
393
|
+
from
|
394
|
+
pg_namespace
|
395
|
+
where
|
396
|
+
nspacl is not null
|
397
|
+
and nspowner != 1
|
398
|
+
;
|
399
|
+
SQL
|
400
|
+
|
401
|
+
raw_schema_acl = @executor.dwh(query)
|
402
|
+
_parse_current_acl('schema', raw_schema_acl)
|
403
|
+
end
|
404
|
+
|
405
|
+
def _get_current_database_acl
|
406
|
+
query = <<-SQL
|
407
|
+
select
|
408
|
+
datname as name
|
409
|
+
, array_to_string(datacl, ',') as acls
|
410
|
+
from pg_database
|
411
|
+
where
|
412
|
+
datacl is not null
|
413
|
+
and datdba != 1
|
414
|
+
;
|
415
|
+
SQL
|
416
|
+
|
417
|
+
raw_database_acl = @executor.dwh(query)
|
418
|
+
_parse_current_acl('database', raw_database_acl)
|
419
|
+
end
|
420
|
+
|
421
|
+
def _get_current_table_acl
|
422
|
+
query = <<-SQL
|
423
|
+
select
|
424
|
+
pg_namespace.nspname || '.' || pg_class.relname as name
|
425
|
+
, array_to_string(pg_class.relacl, ',') as acls
|
426
|
+
from pg_class
|
427
|
+
left join pg_namespace on pg_class.relnamespace = pg_namespace.oid
|
428
|
+
where
|
429
|
+
pg_class.relacl is not null
|
430
|
+
and pg_namespace.nspname not in (
|
431
|
+
'pg_catalog'
|
432
|
+
, 'pg_toast'
|
433
|
+
, 'information_schema'
|
434
|
+
)
|
435
|
+
order by
|
436
|
+
pg_namespace.nspname || '.' || pg_class.relname
|
437
|
+
;
|
438
|
+
SQL
|
439
|
+
|
440
|
+
raw_table_acl = @executor.dwh(query)
|
441
|
+
_parse_current_acl('table', raw_table_acl)
|
442
|
+
end
|
443
|
+
|
444
|
+
def _sync_typed_acl(identifier_type, current_acl, defined_acl)
|
445
|
+
diverged = 0
|
446
|
+
current_acl ||= {}
|
447
|
+
defined_acl ||= {}
|
448
|
+
|
449
|
+
grantees = []
|
450
|
+
.concat(current_acl.keys)
|
451
|
+
.concat(defined_acl.keys)
|
452
|
+
.uniq
|
453
|
+
.compact
|
454
|
+
|
455
|
+
known_grantees = []
|
456
|
+
.concat(_get_current_dwh_users().keys)
|
457
|
+
.concat(_get_current_dwh_groups().keys.map { |group| "group #{group}" })
|
458
|
+
.uniq
|
459
|
+
.compact
|
460
|
+
|
461
|
+
if grantees.any? { |grantee| !known_grantees.include?(grantee) }
|
462
|
+
raise Error, 'Users or groups not in sync!'
|
463
|
+
end
|
464
|
+
|
465
|
+
grantees.each do |grantee|
|
466
|
+
current_grantee_acl = current_acl[grantee] || {}
|
467
|
+
defined_grantee_acl = defined_acl[grantee] || {}
|
468
|
+
|
469
|
+
unsynced_privileges_count = _sync_privileges(grantee, identifier_type, current_grantee_acl, defined_grantee_acl)
|
470
|
+
diverged += unsynced_privileges_count
|
471
|
+
end
|
472
|
+
|
473
|
+
diverged
|
474
|
+
end
|
475
|
+
|
476
|
+
def _sync_privileges(grantee, identifier_type, current_acl, defined_acl)
|
477
|
+
current_acl ||= {}
|
478
|
+
defined_acl ||= {}
|
479
|
+
|
480
|
+
identifiers = []
|
481
|
+
.concat(current_acl.keys)
|
482
|
+
.concat(defined_acl.keys)
|
483
|
+
.uniq
|
484
|
+
.compact
|
485
|
+
|
486
|
+
unsynced_privileges = 0
|
487
|
+
|
488
|
+
identifiers.each do |identifier_name|
|
489
|
+
if _is_in_unmanaged_schema?(identifier_type, identifier_name)
|
490
|
+
@logger.debug("SKIPPED #{identifier_type} '#{identifier_name}'. Cannot sync privileges for object outside production schemas!")
|
491
|
+
else
|
492
|
+
current_privileges = current_acl[identifier_name] || []
|
493
|
+
defined_privileges = defined_acl[identifier_name] || []
|
494
|
+
|
495
|
+
undefined_privileges = (current_privileges - defined_privileges).uniq.compact
|
496
|
+
missing_privileges = (defined_privileges - current_privileges).uniq.compact
|
497
|
+
|
498
|
+
current_privileges_diverged = (undefined_privileges.count + missing_privileges.count)
|
499
|
+
unsynced_privileges += current_privileges_diverged
|
500
|
+
|
501
|
+
_revoke_privileges(grantee, identifier_type, identifier_name, undefined_privileges)
|
502
|
+
_grant_privileges(grantee, identifier_type, identifier_name, missing_privileges)
|
503
|
+
end
|
504
|
+
end
|
505
|
+
|
506
|
+
@logger.debug("#{identifier_type.capitalize} privileges for #{grantee} are in sync") if unsynced_privileges == 0
|
507
|
+
|
508
|
+
unsynced_privileges
|
509
|
+
end
|
510
|
+
|
511
|
+
def _is_in_unmanaged_schema?(identifier_type, identifier_name)
|
512
|
+
managed = true
|
513
|
+
case identifier_type
|
514
|
+
when 'schema'
|
515
|
+
managed = @production_schemas.include?(identifier_name)
|
516
|
+
when 'table'
|
517
|
+
managed = @production_schemas.any? { |p| identifier_name.start_with?("#{p}.") }
|
518
|
+
end
|
519
|
+
!managed
|
520
|
+
end
|
521
|
+
|
522
|
+
def _grant_privileges(grantee, identifier_type, identifier_name, privileges)
|
523
|
+
privileges ||= []
|
524
|
+
unless privileges.empty?
|
525
|
+
_execute_statement("grant #{privileges.join(',')} on #{identifier_type} #{identifier_name} to #{grantee};")
|
526
|
+
end
|
527
|
+
end
|
528
|
+
|
529
|
+
def _revoke_privileges(grantee, identifier_type, identifier_name, privileges)
|
530
|
+
privileges ||= []
|
531
|
+
unless privileges.empty?
|
532
|
+
_execute_statement("revoke #{privileges.join(',')} on #{identifier_type} #{identifier_name} from #{grantee};")
|
533
|
+
end
|
534
|
+
end
|
535
|
+
|
536
|
+
def _get_defined_acl(identifier_type)
|
537
|
+
defined_acl = {}
|
538
|
+
|
539
|
+
groups = Hash[@acl_config['groups'].map { |name, config| ["group #{name}", config] }] || {}
|
540
|
+
users = @acl_config['users'] || {}
|
541
|
+
|
542
|
+
grantees = {}
|
543
|
+
.merge(groups)
|
544
|
+
.merge(users)
|
545
|
+
|
546
|
+
grantees.each do |grantee, config|
|
547
|
+
permissions = config['permissions'] || []
|
548
|
+
parsed_permissions = _parse_defined_permissions(identifier_type, permissions)
|
549
|
+
defined_acl[grantee] = parsed_permissions unless parsed_permissions.empty?
|
550
|
+
end
|
551
|
+
|
552
|
+
defined_acl #.select { |_, permissions| !permissions.empty? }
|
553
|
+
end
|
554
|
+
|
555
|
+
def _get_defined_database_acl
|
556
|
+
_get_defined_acl('database')
|
557
|
+
end
|
558
|
+
|
559
|
+
def _get_defined_schema_acl
|
560
|
+
_get_defined_acl('schema')
|
561
|
+
end
|
562
|
+
|
563
|
+
def _get_defined_table_acl
|
564
|
+
_get_defined_acl('table')
|
565
|
+
end
|
566
|
+
|
567
|
+
def _parse_defined_permissions(identifier_type, raw_permissions)
|
568
|
+
defined_acl = {}
|
569
|
+
|
570
|
+
# Implicitly grant usage on schemas for which we grant table privileges
|
571
|
+
if identifier_type.eql?('schema')
|
572
|
+
table_permissions = raw_permissions.select { |permission| permission['type'].eql?('table') } || []
|
573
|
+
table_permissions.each do |table_permission|
|
574
|
+
table_permission['identifiers'].each do |schema_and_table|
|
575
|
+
schema, _ = schema_and_table.split('.')
|
576
|
+
schemas_to_grant_usage_on = []
|
577
|
+
|
578
|
+
if schema.eql?('*')
|
579
|
+
schemas_to_grant_usage_on = @production_schemas
|
580
|
+
else
|
581
|
+
schemas_to_grant_usage_on << schema
|
582
|
+
end
|
583
|
+
|
584
|
+
schemas_to_grant_usage_on.each do |schema_name|
|
585
|
+
defined_acl[schema_name] ||= []
|
586
|
+
defined_acl[schema_name].concat(['usage'])
|
587
|
+
end
|
588
|
+
end
|
589
|
+
end
|
590
|
+
end
|
591
|
+
|
592
|
+
permissions = raw_permissions.select { |permission| permission['type'].eql?(identifier_type) } || []
|
593
|
+
|
594
|
+
permissions.each do |permission|
|
595
|
+
permission['identifiers'].each do |identifier|
|
596
|
+
privileges = permission['privileges']
|
597
|
+
|
598
|
+
if identifier.match /\*/
|
599
|
+
case identifier_type
|
600
|
+
when 'database'
|
601
|
+
raise Error, 'Don\'t know how to resolve database identifiers with wildcard'
|
602
|
+
when 'schema'
|
603
|
+
@production_schemas.each do |schema|
|
604
|
+
defined_acl[schema] ||= []
|
605
|
+
defined_acl[schema].concat(privileges)
|
606
|
+
end
|
607
|
+
when 'table'
|
608
|
+
schema, table = identifier.split('.')
|
609
|
+
|
610
|
+
raise Error, 'Cannot resolve wildcard schema for specific table names' if schema.eql?('*') and !table.eql?('*')
|
611
|
+
|
612
|
+
tables_to_grant_privileges_on = []
|
613
|
+
|
614
|
+
if schema.eql?('*')
|
615
|
+
@production_schemas.each do |prod_schema|
|
616
|
+
tables = @executor.dwh(<<-SQL).map { |row| "#{prod_schema}.#{row['tablename']}" }
|
617
|
+
select tablename from pg_tables where schemaname='#{prod_schema}'
|
618
|
+
SQL
|
619
|
+
|
620
|
+
tables_to_grant_privileges_on.concat(tables)
|
621
|
+
end
|
622
|
+
else
|
623
|
+
tables_to_grant_privileges_on = @executor.dwh(<<-SQL).map { |row| "#{schema}.#{row['tablename']}" }
|
624
|
+
select tablename from pg_tables where schemaname='#{schema}'
|
625
|
+
SQL
|
626
|
+
end
|
627
|
+
|
628
|
+
tables_to_grant_privileges_on = tables_to_grant_privileges_on.uniq.compact
|
629
|
+
|
630
|
+
tables_to_grant_privileges_on.each do |table|
|
631
|
+
defined_acl[table] ||= []
|
632
|
+
defined_acl[table].concat(privileges)
|
633
|
+
end
|
634
|
+
end
|
635
|
+
else
|
636
|
+
defined_acl[identifier] ||= []
|
637
|
+
defined_acl[identifier].concat(privileges)
|
638
|
+
end
|
639
|
+
end
|
640
|
+
end
|
641
|
+
|
642
|
+
uniquely_defined_acl = Hash[defined_acl.map do |identifier, privileges|
|
643
|
+
unique_privileges = privileges.include?('all') ? ['all'] : privileges.uniq.compact
|
644
|
+
[identifier, unique_privileges]
|
645
|
+
end]
|
646
|
+
|
647
|
+
uniquely_defined_acl
|
648
|
+
end
|
649
|
+
|
650
|
+
def _parse_current_acl(identifier_type, raw_acl)
|
651
|
+
parsed_acls = {}
|
652
|
+
raw_acl.each do |row|
|
653
|
+
name = row['name']
|
654
|
+
parsed_acl = _parse_current_permissions(identifier_type, row['acls'])
|
655
|
+
parsed_acl.each do |grantee, privileges|
|
656
|
+
unless grantee.empty?
|
657
|
+
parsed_acls[grantee] ||= {}
|
658
|
+
parsed_acls[grantee][name] ||= []
|
659
|
+
parsed_acls[grantee][name].concat(privileges)
|
660
|
+
end
|
661
|
+
end
|
662
|
+
end
|
663
|
+
parsed_acls
|
664
|
+
end
|
665
|
+
|
666
|
+
def _parse_current_permissions(identifier_type, raw_permissions)
|
667
|
+
# http://www.postgresql.org/docs/8.1/static/sql-grant.html
|
668
|
+
# Typical ACL string:
|
669
|
+
#
|
670
|
+
# admin=arwdRxt/admin,jimdo=r/admin,"group selfservice=r/admin"
|
671
|
+
#
|
672
|
+
# =xxxx -- privileges granted to PUBLIC
|
673
|
+
# uname=xxxx -- privileges granted to a user
|
674
|
+
# group gname=xxxx -- privileges granted to a group
|
675
|
+
# /yyyy -- user who granted this privilege
|
676
|
+
|
677
|
+
privilege_type = case identifier_type
|
678
|
+
when 'database'
|
679
|
+
Privilege::Database
|
680
|
+
when 'schema'
|
681
|
+
Privilege::Schema
|
682
|
+
when 'table'
|
683
|
+
Privilege::Table
|
684
|
+
else
|
685
|
+
raise Error, 'wtf'
|
686
|
+
end
|
687
|
+
|
688
|
+
parsed_permissions = {}
|
689
|
+
permissions = raw_permissions.split(',')
|
690
|
+
permissions.map! { |entry| entry.delete('"') }
|
691
|
+
permissions.each do |permission|
|
692
|
+
grantee, permission_string = permission.split('=')
|
693
|
+
privileges_string = permission_string.split('/')[0]
|
694
|
+
|
695
|
+
next if grantee.eql?(@master_username) # superuser has access to everything anyway
|
696
|
+
|
697
|
+
parsed_permissions[grantee] ||= privilege_type.parse_privileges_from_string(privileges_string)
|
698
|
+
end
|
699
|
+
|
700
|
+
parsed_permissions
|
701
|
+
end
|
702
|
+
|
703
|
+
def _get_currently_owned_schemas(user)
|
704
|
+
query = <<-SQL
|
705
|
+
select pg_namespace.nspname as schemaname
|
706
|
+
from pg_namespace
|
707
|
+
left join pg_user on pg_namespace.nspowner = pg_user.usesysid
|
708
|
+
where pg_user.usename = '#{user}'
|
709
|
+
;
|
710
|
+
SQL
|
711
|
+
@executor.dwh(query).map { |row| row['schemaname'] }
|
712
|
+
end
|
713
|
+
|
714
|
+
def _get_currently_owned_tables(user)
|
715
|
+
query = <<-SQL
|
716
|
+
select (schemaname || '.' || tablename) as tablename
|
717
|
+
from pg_tables
|
718
|
+
where tableowner = '#{user}'
|
719
|
+
;
|
720
|
+
SQL
|
721
|
+
@executor.dwh(query).map { |row| row['tablename'] }
|
722
|
+
end
|
723
|
+
end
|
724
|
+
end
|
data/lib/forty.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Dir[File.dirname(__FILE__) + '/forty/*.rb'].each { |file| require file }
|
metadata
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: forty
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Stefanie Grunwald
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-07-25 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description:
|
14
|
+
email: steffi@physics.org
|
15
|
+
executables: []
|
16
|
+
extensions: []
|
17
|
+
extra_rdoc_files: []
|
18
|
+
files:
|
19
|
+
- lib/forty.rb
|
20
|
+
- lib/forty/acl.rb
|
21
|
+
- lib/forty/configuration.rb
|
22
|
+
- lib/forty/database.rb
|
23
|
+
- lib/forty/dbms.rb
|
24
|
+
- lib/forty/redshift/privilege.rb
|
25
|
+
- lib/forty/sync.rb
|
26
|
+
homepage: https://github.com/moertel/forty
|
27
|
+
licenses:
|
28
|
+
- MIT
|
29
|
+
metadata: {}
|
30
|
+
post_install_message:
|
31
|
+
rdoc_options: []
|
32
|
+
require_paths:
|
33
|
+
- lib
|
34
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '2.0'
|
39
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
requirements: []
|
45
|
+
rubyforge_project:
|
46
|
+
rubygems_version: 2.4.5
|
47
|
+
signing_key:
|
48
|
+
specification_version: 3
|
49
|
+
summary: Manage users, groups and ACL (access control lists) for AWS Redshift databases
|
50
|
+
test_files: []
|