hammer_cli_import 0.10.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +674 -0
- data/README.md +115 -0
- data/channel_data_pretty.json +10316 -0
- data/config/import/config_macros.yml +16 -0
- data/config/import/interview_answers.yml +13 -0
- data/config/import/role_map.yml +10 -0
- data/config/import.yml +2 -0
- data/lib/hammer_cli_import/activationkey.rb +156 -0
- data/lib/hammer_cli_import/all.rb +253 -0
- data/lib/hammer_cli_import/asynctasksreactor.rb +187 -0
- data/lib/hammer_cli_import/autoload.rb +27 -0
- data/lib/hammer_cli_import/base.rb +585 -0
- data/lib/hammer_cli_import/configfile.rb +392 -0
- data/lib/hammer_cli_import/contenthost.rb +243 -0
- data/lib/hammer_cli_import/contentview.rb +198 -0
- data/lib/hammer_cli_import/csvhelper.rb +68 -0
- data/lib/hammer_cli_import/deltahash.rb +86 -0
- data/lib/hammer_cli_import/fixtime.rb +27 -0
- data/lib/hammer_cli_import/hostcollection.rb +52 -0
- data/lib/hammer_cli_import/import.rb +31 -0
- data/lib/hammer_cli_import/importtools.rb +351 -0
- data/lib/hammer_cli_import/organization.rb +110 -0
- data/lib/hammer_cli_import/persistentmap.rb +225 -0
- data/lib/hammer_cli_import/repository.rb +91 -0
- data/lib/hammer_cli_import/repositoryenable.rb +250 -0
- data/lib/hammer_cli_import/templatesnippet.rb +67 -0
- data/lib/hammer_cli_import/user.rb +155 -0
- data/lib/hammer_cli_import/version.rb +25 -0
- data/lib/hammer_cli_import.rb +53 -0
- metadata +117 -0
@@ -0,0 +1,351 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2014 Red Hat Inc.
|
3
|
+
#
|
4
|
+
# This file is part of hammer-cli-import.
|
5
|
+
#
|
6
|
+
# hammer-cli-import is free software: you can redistribute it and/or modify
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
9
|
+
# (at your option) any later version.
|
10
|
+
#
|
11
|
+
# hammer-cli-import is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU General Public License for more details.
|
15
|
+
#
|
16
|
+
# You should have received a copy of the GNU General Public License
|
17
|
+
# along with hammer-cli-import. If not, see <http://www.gnu.org/licenses/>.
|
18
|
+
#
|
19
|
+
|
20
|
+
# Modules to help with imports. To be used as Extend/Import on classes that inherit
|
21
|
+
# from module HammerCLIImport::BaseCommand.
|
22
|
+
|
23
|
+
require 'logger'
|
24
|
+
|
25
|
+
module ImportTools
|
26
|
+
module Repository
|
27
|
+
module Extend
|
28
|
+
def add_repo_options
|
29
|
+
option ['--synchronize'], :flag, 'Synchronize imported repositories', :default => false
|
30
|
+
option ['--wait'], :flag, 'Wait for repository synchronization to finish', :default => false
|
31
|
+
|
32
|
+
add_async_tasks_reactor_options
|
33
|
+
|
34
|
+
validate_options do
|
35
|
+
option(:option_synchronize).required if option(:option_wait).exist?
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
module Include
|
41
|
+
def repo_synced?(repo)
|
42
|
+
raise ArgumentError, 'nil is not a valid repository' if repo.nil?
|
43
|
+
|
44
|
+
info = lookup_entity(:repositories, repo['id'], true)
|
45
|
+
last_sync_result = info['last_sync']['result'] if info['last_sync']
|
46
|
+
return false unless (info['sync_state'] == 'finished' || last_sync_result == 'success')
|
47
|
+
|
48
|
+
## (Temporary) workaround for 1131954
|
49
|
+
## updated_at is updated after sync for some reason...
|
50
|
+
# begin
|
51
|
+
# Time.parse(info['last_sync']) > Time.parse(info['updated_at'])
|
52
|
+
# rescue
|
53
|
+
# false
|
54
|
+
# end
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
58
|
+
def sync_repo(repo)
|
59
|
+
return unless option_synchronize?
|
60
|
+
task = api_call(:repositories, :sync, {:id => repo['id']})
|
61
|
+
debug "Sync of repo #{repo['id']} started!"
|
62
|
+
return unless option_wait?
|
63
|
+
wait_for_task task['id']
|
64
|
+
end
|
65
|
+
|
66
|
+
def sync_repo2(repo)
|
67
|
+
task = api_call(:repositories, :sync, {:id => repo['id']})
|
68
|
+
debug "Sync of repo #{repo['id']} started!"
|
69
|
+
task['id']
|
70
|
+
rescue
|
71
|
+
uuid = workaround_1116063 repo['id']
|
72
|
+
info 'Sync already running!'
|
73
|
+
uuid
|
74
|
+
end
|
75
|
+
|
76
|
+
def with_synced_repo(repo, &block)
|
77
|
+
# So we can not give empty block
|
78
|
+
if block_given?
|
79
|
+
action = block
|
80
|
+
else
|
81
|
+
action = proc {}
|
82
|
+
end
|
83
|
+
|
84
|
+
# Already synchronized? Do the Thing
|
85
|
+
# ElsIf asked to sync, sync-and-then-wait to Do the Thing
|
86
|
+
# Otherwise, shrug and Skip The Thing
|
87
|
+
if repo_synced?(repo)
|
88
|
+
action.call
|
89
|
+
elsif option_synchronize?
|
90
|
+
uuid = sync_repo2 repo
|
91
|
+
postpone_till([uuid], &action) if option_wait?
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
# When BZ 1116063 get fixed, this might
|
98
|
+
# be simplified using either
|
99
|
+
# > api_call(:repositories, :show, {'id' => 1})
|
100
|
+
# or maybe with
|
101
|
+
# > api_call(:sync, :index, {:repository_id => 1})
|
102
|
+
def workaround_1116063(repo_id)
|
103
|
+
res = api_call :foreman_tasks, :bulk_search, \
|
104
|
+
:searches => [{:type => :resource, :resource_type => 'Katello::Repository', :resource_id => repo_id}]
|
105
|
+
|
106
|
+
res.first['results'] \
|
107
|
+
.select { |x| x['result'] != 'error' } \
|
108
|
+
.max_by { |x| Time.parse(x['started_at']) }['id']
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
module LifecycleEnvironment
|
114
|
+
module Include
|
115
|
+
def get_env(org_id, name = 'Library')
|
116
|
+
@lc_environments ||= {}
|
117
|
+
@lc_environments[org_id] ||= {}
|
118
|
+
unless @lc_environments[org_id][name]
|
119
|
+
res = api_call :lifecycle_environments, :index, {:organization_id => org_id, :name => name}
|
120
|
+
@lc_environments[org_id][name] = res['results'].find { |x| x['name'] == name }
|
121
|
+
end
|
122
|
+
@lc_environments[org_id][name]
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
module Task
|
128
|
+
module Include
|
129
|
+
# [uuid] -> {uuid => {:finished => bool, :progress => Float}}
|
130
|
+
def annotate_tasks(uuids)
|
131
|
+
ret = {}
|
132
|
+
get_tasks_statuses(uuids).each do |uuid, stat|
|
133
|
+
ret[uuid] = { :finished => stat['state'] == 'stopped',
|
134
|
+
:progress => stat['progress']}
|
135
|
+
end
|
136
|
+
ret
|
137
|
+
end
|
138
|
+
|
139
|
+
private
|
140
|
+
|
141
|
+
# [uuid] -> {uuid: task_status}
|
142
|
+
def get_tasks_statuses(uuids)
|
143
|
+
searches = uuids.collect { |uuid| {:type => :task, :task_id => uuid} }
|
144
|
+
ret = api_call :foreman_tasks, :bulk_search, {:searches => searches}
|
145
|
+
statuses = {}
|
146
|
+
ret.each do |status_result|
|
147
|
+
status_result['results'].each do |task_info|
|
148
|
+
statuses[task_info['id']] = task_info
|
149
|
+
end
|
150
|
+
end
|
151
|
+
statuses
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
module ContentView
|
157
|
+
module Include
|
158
|
+
def publish_content_view(id, entity_type = :content_views)
|
159
|
+
mapped_api_call entity_type, :publish, {:id => id}
|
160
|
+
rescue => e
|
161
|
+
warn "Publishing of #{to_singular(entity_type)} [#{id}] failed with #{e.class}: #{e.message}"
|
162
|
+
end
|
163
|
+
|
164
|
+
def create_composite_content_view(entity_type, org_id, cv_label, cv_description, cvs)
|
165
|
+
return nil if cvs.empty?
|
166
|
+
if cvs.size == 1
|
167
|
+
return cvs.to_a[0]
|
168
|
+
else
|
169
|
+
# create composite content view
|
170
|
+
cv_versions = []
|
171
|
+
cvs.each do |cv_id|
|
172
|
+
cvvs = list_server_entities(:content_view_versions, {:content_view_id => cv_id})
|
173
|
+
cv_versions << cvvs.collect { |cv| cv['id'] }.max
|
174
|
+
end
|
175
|
+
cv = lookup_entity_in_cache(entity_type, 'label' => cv_label)
|
176
|
+
if cv
|
177
|
+
info " Content view #{cv_label} already created, reusing."
|
178
|
+
else
|
179
|
+
# create composite content view
|
180
|
+
# for activation key purposes
|
181
|
+
cv = create_entity(
|
182
|
+
entity_type,
|
183
|
+
{
|
184
|
+
:organization_id => org_id,
|
185
|
+
:name => cv_label,
|
186
|
+
:label => cv_label,
|
187
|
+
:composite => true,
|
188
|
+
:description => cv_description,
|
189
|
+
:component_ids => cv_versions
|
190
|
+
},
|
191
|
+
cv_label)
|
192
|
+
# publish the content view
|
193
|
+
info " Publishing content view: #{cv['id']}"
|
194
|
+
publish_content_view(cv['id'], entity_type)
|
195
|
+
end
|
196
|
+
return cv['id']
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
# use entity_type as parameter to be able to re-use the method for
|
201
|
+
# :content_views, :ak_content_views, :redhat_content_views, ...
|
202
|
+
def delete_content_view(cv_id, entity_type = :content_views)
|
203
|
+
raise "delete_content_view with #{entity_type}" unless map_target_entity[entity_type] == :content_views
|
204
|
+
|
205
|
+
content_view = get_cache(entity_type)[cv_id]
|
206
|
+
# first delete the content view from associated environments
|
207
|
+
cves = content_view['environments'].collect { |e| e['id'] }
|
208
|
+
cves.each do |cve|
|
209
|
+
begin
|
210
|
+
task = mapped_api_call(
|
211
|
+
entity_type,
|
212
|
+
:remove_from_environment,
|
213
|
+
{
|
214
|
+
:id => content_view['id'],
|
215
|
+
:environment_id => cve
|
216
|
+
})
|
217
|
+
rescue => e
|
218
|
+
warn "Failed to remove content view [#{cv_id}] from environment #{cve} with #{e.class}: #{e.message}"
|
219
|
+
end
|
220
|
+
wait_for_task(task['id'], 1, 0)
|
221
|
+
end
|
222
|
+
|
223
|
+
if content_view['versions'] && !content_view['versions'].empty?
|
224
|
+
cv_version_ids = content_view['versions'].collect { |v| v['id'] }
|
225
|
+
|
226
|
+
begin
|
227
|
+
task = mapped_api_call(
|
228
|
+
entity_type,
|
229
|
+
:remove,
|
230
|
+
{
|
231
|
+
:id => content_view['id'],
|
232
|
+
:content_view_version_ids => cv_version_ids
|
233
|
+
})
|
234
|
+
|
235
|
+
wait_for_task(task['id'], 1, 0)
|
236
|
+
rescue => e
|
237
|
+
warn "Failed to remove versions of content view [#{cv_id}] with #{e.class}: #{e.message}"
|
238
|
+
end
|
239
|
+
else
|
240
|
+
debug "No versions found for #{to_singular(entity_type)} #{cv_id}"
|
241
|
+
end
|
242
|
+
|
243
|
+
delete_entity_by_import_id(entity_type, content_view['id'])
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
module ImportLogging
|
249
|
+
module Extend
|
250
|
+
def add_logging_options
|
251
|
+
# Logging options
|
252
|
+
# quiet = go to logfile only
|
253
|
+
# verbose = all output goes to STDOUT as well as log
|
254
|
+
# debug = enable debug-output
|
255
|
+
# default = no debug, only PROGRESS-and-above to STDOUT
|
256
|
+
option ['--quiet'], :flag, 'Be silent - no output to STDOUT', :default => false
|
257
|
+
option ['--debug'], :flag, 'Turn on debugging-information', :default => false
|
258
|
+
option ['--verbose'],
|
259
|
+
:flag,
|
260
|
+
'Be noisy - everything goes to STDOUT and to a logfile',
|
261
|
+
:default => false
|
262
|
+
option ['--logfile'],
|
263
|
+
'LOGFILE',
|
264
|
+
'Where output is logged to',
|
265
|
+
:default => File.expand_path('~/import.log')
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
module Include
|
270
|
+
def setup_logging
|
271
|
+
@curr_lvl = Logger::INFO
|
272
|
+
@curr_lvl = Logger::DEBUG if option_debug?
|
273
|
+
|
274
|
+
@logger = Logger.new(File.new(option_logfile, 'a'))
|
275
|
+
@logger.level = @curr_lvl
|
276
|
+
end
|
277
|
+
|
278
|
+
def debug(s)
|
279
|
+
log(Logger::DEBUG, s)
|
280
|
+
end
|
281
|
+
|
282
|
+
def info(s)
|
283
|
+
log(Logger::INFO, s)
|
284
|
+
end
|
285
|
+
|
286
|
+
def progress(s)
|
287
|
+
log(Logger::INFO, s, true)
|
288
|
+
end
|
289
|
+
|
290
|
+
def warn(s)
|
291
|
+
log(Logger::WARN, s)
|
292
|
+
end
|
293
|
+
|
294
|
+
def error(s)
|
295
|
+
log(Logger::ERROR, s)
|
296
|
+
end
|
297
|
+
|
298
|
+
def fatal(s)
|
299
|
+
log(Logger::FATAL, s)
|
300
|
+
end
|
301
|
+
|
302
|
+
def logtrace(e)
|
303
|
+
@logger.log(Logger::ERROR, (e.backtrace.join "\n"))
|
304
|
+
end
|
305
|
+
|
306
|
+
def log(lvl, s, always = false)
|
307
|
+
@logger.log(lvl, s)
|
308
|
+
return if option_quiet?
|
309
|
+
|
310
|
+
if always
|
311
|
+
puts s
|
312
|
+
elsif option_verbose?
|
313
|
+
puts s if lvl >= @curr_lvl
|
314
|
+
else
|
315
|
+
puts s if lvl > @curr_lvl
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
module Exceptional
|
322
|
+
module Include
|
323
|
+
def handle_missing_and_supress(what, &block)
|
324
|
+
block.call
|
325
|
+
rescue HammerCLIImport::MissingObjectError => moe
|
326
|
+
error moe.message
|
327
|
+
rescue => e
|
328
|
+
error "Caught #{e.class}:#{e.message} while #{what}"
|
329
|
+
logtrace e
|
330
|
+
end
|
331
|
+
|
332
|
+
# this method catches everything sent to stdout and stderr
|
333
|
+
# and disallows any summary changes
|
334
|
+
#
|
335
|
+
# this is a bad hack, but we need it, as sat6 cannot tell,
|
336
|
+
# whether there're still content hosts associated with a content view
|
337
|
+
# so we try to delete system content views silently
|
338
|
+
def silently(&block)
|
339
|
+
summary_backup = @summary.clone
|
340
|
+
$stdout = StringIO.new
|
341
|
+
$stderr = StringIO.new
|
342
|
+
block.call
|
343
|
+
ensure
|
344
|
+
$stdout = STDOUT
|
345
|
+
$stderr = STDERR
|
346
|
+
@summary = summary_backup
|
347
|
+
end
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
|
@@ -0,0 +1,110 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2014 Red Hat Inc.
|
3
|
+
#
|
4
|
+
# This file is part of hammer-cli-import.
|
5
|
+
#
|
6
|
+
# hammer-cli-import is free software: you can redistribute it and/or modify
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
9
|
+
# (at your option) any later version.
|
10
|
+
#
|
11
|
+
# hammer-cli-import is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU General Public License for more details.
|
15
|
+
#
|
16
|
+
# You should have received a copy of the GNU General Public License
|
17
|
+
# along with hammer-cli-import. If not, see <http://www.gnu.org/licenses/>.
|
18
|
+
#
|
19
|
+
|
20
|
+
require 'hammer_cli'
|
21
|
+
|
22
|
+
module HammerCLIImport
|
23
|
+
class ImportCommand
|
24
|
+
class OrganizationImportCommand < BaseCommand
|
25
|
+
command_name 'organization'
|
26
|
+
reportname = 'users'
|
27
|
+
desc "Import Organizations (from spacewalk-report #{reportname})."
|
28
|
+
|
29
|
+
option ['--into-org-id'], 'ORG_ID', 'Import all organizations into one specified by id' do |x|
|
30
|
+
Integer(x)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Where do we expect to find manifest-files?
|
34
|
+
# NOTE: we won't upload manifests if we're doing into-org-id - the expectation is that
|
35
|
+
# you have already set up your org
|
36
|
+
option ['--upload-manifests-from'],
|
37
|
+
'MANIFEST_DIR',
|
38
|
+
'Upload manifests found at MANIFEST_DIR. Assumes manifest for "ORG NAME" will be of the form ORG_NAME.zip'
|
39
|
+
|
40
|
+
csv_columns 'organization_id', 'organization'
|
41
|
+
|
42
|
+
persistent_maps :organizations
|
43
|
+
|
44
|
+
def mk_org_hash(data)
|
45
|
+
{
|
46
|
+
:id => data['organization_id'].to_i,
|
47
|
+
:name => data['organization'],
|
48
|
+
:description => "Imported '#{data['organization']}' organization from Red Hat Satellite 5"
|
49
|
+
}
|
50
|
+
end
|
51
|
+
|
52
|
+
# :subscriptions :upload {:org_id => id, :content => File.new(filename, 'rb')}
|
53
|
+
def upload_manifest_for(label, id)
|
54
|
+
# Remember labels we've already processed in this run
|
55
|
+
@manifests ||= []
|
56
|
+
return if @manifests.include? label
|
57
|
+
|
58
|
+
@manifests << label
|
59
|
+
filename = option_upload_manifests_from + '/' + label + '.zip'
|
60
|
+
unless File.exist? filename
|
61
|
+
error "No manifest #{filename} available."
|
62
|
+
return
|
63
|
+
end
|
64
|
+
|
65
|
+
info "Uploading manifest #{filename} to org-id #{id}"
|
66
|
+
manifest_file = File.new(filename, 'rb')
|
67
|
+
request_headers = {:content_type => 'multipart/form-data', :multipart => true}
|
68
|
+
|
69
|
+
rc = api_call :subscriptions, :upload, {:organization_id => id, :content => manifest_file}, request_headers
|
70
|
+
wait_for_task(rc['id'])
|
71
|
+
report_summary :uploaded, :manifests
|
72
|
+
end
|
73
|
+
|
74
|
+
def import_single_row(data)
|
75
|
+
if option_into_org_id
|
76
|
+
unless lookup_entity_in_cache(:organizations, {'id' => option_into_org_id})
|
77
|
+
warn "Organization [#{option_into_org_id}] not found. Skipping."
|
78
|
+
return
|
79
|
+
end
|
80
|
+
map_entity(:organizations, data['organization_id'].to_i, option_into_org_id)
|
81
|
+
return
|
82
|
+
end
|
83
|
+
org = mk_org_hash data
|
84
|
+
new_org = create_entity(:organizations, org, data['organization_id'].to_i)
|
85
|
+
upload_manifest_for(new_org['label'], new_org['id']) unless option_upload_manifests_from.nil?
|
86
|
+
end
|
87
|
+
|
88
|
+
def delete_single_row(data)
|
89
|
+
org_id = data['organization_id'].to_i
|
90
|
+
unless @pm[:organizations][org_id]
|
91
|
+
warn "#{to_singular(:organizations).capitalize} with id #{org_id} wasn't imported. Skipping deletion."
|
92
|
+
return
|
93
|
+
end
|
94
|
+
target_org_id = get_translated_id(:organizations, org_id)
|
95
|
+
if last_in_cache?(:organizations, target_org_id)
|
96
|
+
warn "Won't delete last organization [#{target_org_id}]. Unmapping only."
|
97
|
+
unmap_entity(:organizations, target_org_id)
|
98
|
+
return
|
99
|
+
end
|
100
|
+
if target_org_id == 1
|
101
|
+
warn "Won't delete organization with id [#{target_org_id}]. Unmapping only."
|
102
|
+
unmap_entity(:organizations, target_org_id)
|
103
|
+
return
|
104
|
+
end
|
105
|
+
delete_entity(:organizations, org_id)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
|
@@ -0,0 +1,225 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2014 Red Hat Inc.
|
3
|
+
#
|
4
|
+
# This file is part of hammer-cli-import.
|
5
|
+
#
|
6
|
+
# hammer-cli-import is free software: you can redistribute it and/or modify
|
7
|
+
# it under the terms of the GNU General Public License as published by
|
8
|
+
# the Free Software Foundation, either version 3 of the License, or
|
9
|
+
# (at your option) any later version.
|
10
|
+
#
|
11
|
+
# hammer-cli-import is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU General Public License for more details.
|
15
|
+
#
|
16
|
+
# You should have received a copy of the GNU General Public License
|
17
|
+
# along with hammer-cli-import. If not, see <http://www.gnu.org/licenses/>.
|
18
|
+
#
|
19
|
+
|
20
|
+
require 'csv'
|
21
|
+
require 'set'
|
22
|
+
|
23
|
+
module PersistentMap
|
24
|
+
class PersistentMapError < RuntimeError
|
25
|
+
end
|
26
|
+
|
27
|
+
class << Fixnum
|
28
|
+
def from_s(x)
|
29
|
+
Integer(x) rescue 0
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class << String
|
34
|
+
def from_s(x)
|
35
|
+
x
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class << self
|
40
|
+
def definitions
|
41
|
+
return @definitions if @definitions
|
42
|
+
@definitions = {}
|
43
|
+
|
44
|
+
[:content_views, :host_collections, :organizations, :repositories, :users].each do |symbol|
|
45
|
+
@definitions[symbol] = ['sat5' => Fixnum], ['sat6' => Fixnum], symbol
|
46
|
+
end
|
47
|
+
|
48
|
+
@definitions[:activation_keys] = ['org_id' => String], ['sat6' => Fixnum], :activation_keys
|
49
|
+
@definitions[:ak_content_views] = ['ak_id' => String], ['sat6' => Fixnum], :content_views
|
50
|
+
@definitions[:system_content_views] = ['ch_seq' => String], ['sat6' => Fixnum], :content_views
|
51
|
+
@definitions[:local_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum], :repositories
|
52
|
+
@definitions[:products] = [{'org_id' => Fixnum}, {'label' => String}], ['sat6' => Fixnum], :products
|
53
|
+
@definitions[:puppet_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}],
|
54
|
+
['sat6' => Fixnum], :repositories
|
55
|
+
@definitions[:redhat_content_views] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum],
|
56
|
+
:content_views
|
57
|
+
@definitions[:redhat_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum],
|
58
|
+
:repositories
|
59
|
+
@definitions[:systems] = ['sat5' => Fixnum], ['sat6' => String], :systems
|
60
|
+
@definitions[:template_snippets] = ['id' => Fixnum], ['sat6' => Fixnum], :config_templates
|
61
|
+
|
62
|
+
@definitions.freeze
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
module Extend
|
67
|
+
attr_reader :maps, :map_description, :map_target_entity
|
68
|
+
|
69
|
+
def persistent_map(symbol)
|
70
|
+
defs = PersistentMap.definitions
|
71
|
+
|
72
|
+
raise PersistentMapError, "Unknown persistent map: #{symbol}" unless defs.key? symbol
|
73
|
+
|
74
|
+
# Names of persistent maps
|
75
|
+
@maps ||= []
|
76
|
+
@maps.push symbol
|
77
|
+
|
78
|
+
key_spec, val_spec, target_entity = defs[symbol]
|
79
|
+
|
80
|
+
# Which entities they are mapped to?
|
81
|
+
# Usually they are mapped to the same entities on Sat6 (speaking of api)
|
82
|
+
# But sometimes you need to create same type of Sat6 entities based on
|
83
|
+
# different Sat5 entities, and then it is time for this extra option.
|
84
|
+
@map_target_entity ||= {}
|
85
|
+
@map_target_entity[symbol] = target_entity
|
86
|
+
|
87
|
+
# How keys and values looks like (so they can be nicely stored)
|
88
|
+
@map_description ||= {}
|
89
|
+
@map_description[symbol] = [key_spec, val_spec]
|
90
|
+
end
|
91
|
+
|
92
|
+
def persistent_maps(*list)
|
93
|
+
raise PersistentMapError, 'Persistent maps should be declared only once' if @maps
|
94
|
+
list.each do |map_sym|
|
95
|
+
persistent_map map_sym
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
module Include
|
101
|
+
def maps
|
102
|
+
self.class.maps
|
103
|
+
end
|
104
|
+
|
105
|
+
def map_target_entity
|
106
|
+
self.class.map_target_entity
|
107
|
+
end
|
108
|
+
|
109
|
+
def load_persistent_maps
|
110
|
+
@pm = {}
|
111
|
+
maps.each do |map_sym|
|
112
|
+
hash = {}
|
113
|
+
Dir[File.join data_dir, "#{map_sym}-*.csv"].sort.each do |filename|
|
114
|
+
reader = CSV.open(filename, 'r')
|
115
|
+
header = reader.shift
|
116
|
+
raise PersistentMapError, "Importing :#{map_sym} from file #{filename}" unless header == (pm_csv_headers map_sym)
|
117
|
+
reader.each do |row|
|
118
|
+
key, value = pm_decode_row map_sym, row
|
119
|
+
delkey = row[-1] == '-'
|
120
|
+
if delkey
|
121
|
+
hash.delete key
|
122
|
+
else
|
123
|
+
hash[key] = value
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
@pm[map_sym] = add_checks(DeltaHash[hash], self.class.map_description[map_sym], map_sym)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def save_persistent_maps
|
132
|
+
maps.each do |map_sym|
|
133
|
+
next unless @pm[map_sym].changed?
|
134
|
+
CSV.open((File.join data_dir, "#{map_sym}-#{Time.now.utc.iso8601}.csv"), 'wb') do |csv|
|
135
|
+
csv << (pm_csv_headers map_sym)
|
136
|
+
@pm[map_sym].new.each do |key, value|
|
137
|
+
key = [key] unless key.is_a? Array
|
138
|
+
value = [value] unless value.is_a? Array
|
139
|
+
csv << key + value + [nil]
|
140
|
+
end
|
141
|
+
delval = [nil] * (val_arity map_sym)
|
142
|
+
@pm[map_sym].del.each do |key|
|
143
|
+
key = [key] unless key.is_a? Array
|
144
|
+
csv << key + delval + ['-']
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# Consider entities deleted if they are not present in cache
|
151
|
+
def prune_persistent_maps(cache)
|
152
|
+
maps.each do |map_sym|
|
153
|
+
entity_ids = cache[map_target_entity[map_sym]].keys
|
154
|
+
pm_hash = @pm[map_sym].to_hash
|
155
|
+
extra = pm_hash.values.to_set - entity_ids.to_set
|
156
|
+
|
157
|
+
next if extra.empty?
|
158
|
+
|
159
|
+
debug "Removing #{map_sym} from persistent map: #{extra.to_a.join(' ')}"
|
160
|
+
pm_hash.each do |key, value|
|
161
|
+
@pm[map_sym].delete key if extra.include? value
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
private
|
167
|
+
|
168
|
+
# Protective black magic.
|
169
|
+
# Checks whether given values and keys matches description
|
170
|
+
# at the moment of insertion...
|
171
|
+
def add_checks(hash, kv_desc, map_sym)
|
172
|
+
hash.instance_eval do
|
173
|
+
ks, vs = kv_desc
|
174
|
+
@key_desc = ks.collect(&:values) .flatten
|
175
|
+
@val_desc = vs.collect(&:values) .flatten
|
176
|
+
@map_sym = map_sym
|
177
|
+
end
|
178
|
+
class << hash
|
179
|
+
def []=(ks, vs)
|
180
|
+
key = ks
|
181
|
+
val = vs
|
182
|
+
key = [key] unless key.is_a? Array
|
183
|
+
val = [val] unless val.is_a? Array
|
184
|
+
raise "Bad key for persistent map #{@map_sym}: (#{key.inspect} - #{@key_desc.inspect})" \
|
185
|
+
unless key.size == @key_desc.size && key.zip(@key_desc).all? { |k, d| k.is_a? d }
|
186
|
+
raise "Bad value for persistent map #{@map_sym}: (#{val.inspect} - #{@val_desc.inspect}" \
|
187
|
+
unless val.size == @val_desc.size && val.zip(@val_desc).all? { |v, d| v.is_a? d }
|
188
|
+
super
|
189
|
+
end
|
190
|
+
end
|
191
|
+
hash
|
192
|
+
end
|
193
|
+
|
194
|
+
def pm_decode_row(map_sym, row)
|
195
|
+
key_spec, val_spec = self.class.map_description[map_sym]
|
196
|
+
key = []
|
197
|
+
value = []
|
198
|
+
|
199
|
+
key_spec.each do |spec|
|
200
|
+
x = row.shift
|
201
|
+
key.push(spec.values.first.from_s x)
|
202
|
+
end
|
203
|
+
|
204
|
+
val_spec.each do |spec|
|
205
|
+
x = row.shift
|
206
|
+
value.push(spec.values.first.from_s x)
|
207
|
+
end
|
208
|
+
|
209
|
+
key = key[0] if key.size == 1
|
210
|
+
value = value[0] if value.size == 1
|
211
|
+
[key, value]
|
212
|
+
end
|
213
|
+
|
214
|
+
def pm_csv_headers(symbol)
|
215
|
+
key_spec, val_spec = self.class.map_description[symbol]
|
216
|
+
(key_spec + val_spec).collect { |x| x.keys[0] } + ['delete']
|
217
|
+
end
|
218
|
+
|
219
|
+
def val_arity(symbol)
|
220
|
+
_key_spec, val_spec = self.class.map_description[symbol]
|
221
|
+
val_spec.size
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby
|