simp-rake-helpers 1.0.15 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- N2ZmNDFjZDQ0NWFjNTNkNWQ3MTFlYjgwYmJlNWUxOWFlZmE5YjQ3ZQ==
4
+ ZmM4NTg3YTQxOTJiN2M2OWQzYmUyYzNkYjJiNzUzZDNjYWNmOWNmMA==
5
5
  data.tar.gz: !binary |-
6
- NTFhZTA4YTQ2ZGM5N2NhYTRjMGRmY2Y1ZjI4ODJjOWMzNDYxODU2Ng==
6
+ NDVkODU1ODAyNmI3ZmFjMGRmOTFiN2NkNzZiYjdkNTlkNjY3Yjk5Yg==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- NGQ1MDMyOTUyOTQyYzQ0NTBmMmVjMzhmNTBkZDA3NDgxMWQ0ZGQ2Mjk5YzAy
10
- YTFkOWVhZTdiMWM3YjM1YmU4YTU3OTk3YzdiOTYwODBmMmI2OTg3MmMwNmY1
11
- OTBmZjc4YzY1NGI4YjVlN2NiNjA3NDA1ZmQ5OTQ0NDUyOGY2MGU=
9
+ MmJlOWM0NWU2ZTllZGE0Mjc4NTc3MmVjNzZiMDEyMDg0ZjQ1MzdlYjY0N2Zl
10
+ YWZkZTc2Y2MxMDFjYzMzZTY2MjhlMTk3ZWYyOWU3OTA5YTJkNTYwMTJkNjFh
11
+ N2JjZWJmZjgwY2VjMThhZDRiOGRiM2Q3NWVkZjk0Zjc2ZTI0Mjg=
12
12
  data.tar.gz: !binary |-
13
- NWQxNTRmNzI1NmM5ZGRjZjNhYjIyNTE3MzMxN2YxMjBlNTIzMjFlMzlhMDMy
14
- NWE3MGI0MGFmZTFhMTEwYWEyZjc3NzdmYjJmMmExZTg2OTMxZDQ1YWQ4MGEy
15
- Y2RjM2RiNzY3NzFhOTlhMTRiYTlkMzcyYzU4MGZhNjgyNWIzNDY=
13
+ Y2EwYzRkOTkzNmIxNmMwMjk0MTg5ODE1MWY1MjRlYzUxMjZkYTlhYTBlZjJl
14
+ NjBiYjMzNWRkYTA2MDU0NmUxMDhlZjMwNmNjYjFlNDU0MzhhMGE2YWU0NWNh
15
+ NzQyMTM0MmJiMDBmMjRlYTY4ODc4NjIwZjVkNTIzNjkzYWEzZjY=
@@ -0,0 +1,363 @@
1
+ #!/usr/bin/rake -T
2
+
3
+ require 'simp/rake'
4
+ require 'json'
5
+ include Simp::Rake
6
+
7
+ class SIMPBuildException < Exception
8
+ end
9
+
10
+ require 'simp/build/release_mapper'
11
+ module Simp; end
12
+ module Simp::Rake; end
13
+ module Simp::Rake::Build
14
+ class Auto < ::Rake::TaskLib
15
+ def initialize( run_dir )
16
+ @base_dir = run_dir
17
+ define
18
+ end
19
+
20
+ # define rake tasks
21
+ def define
22
+ namespace :build do
23
+ desc <<-EOM
24
+ Automatically detect and build a SIMP ISO for a target SIMP release.
25
+
26
+ This task runs all other build tasks
27
+
28
+ Arguments:
29
+ * :release => SIMP release to build (e.g., '5.1.X')
30
+ - :iso_paths => path to source ISO(s) (colon-delimited list of files/directories) [Default: '.']
31
+ - :tarball => SIMP build tarball file; if given, skips tar build. [Default: 'false']
32
+ - :output_dir => path to write SIMP ISO. [Default: './SIMP_ISO']
33
+ - :do_checksum => Use sha256sum checksum to compare each ISO. [Default: 'false']
34
+ - :key_name => Key name to sign packages [Default: 'dev']
35
+ - :packer_vars => Write a packer vars.json to go with this ISO [Default: 'true']
36
+ - :verbose => Enable verbose reporting. [Default: 'false']
37
+
38
+ ENV vars:
39
+ - SIMP_BUILD_staging_dir => Path to stage big build assets [Default: './SIMP_ISO_STAGING']
40
+ - SIMP_BUILD_rm_staging_dir => 'yes' forcibly removes the staging dir before starting
41
+ - SIMP_BUILD_force_dirty => 'yes' tries to checks out subrepos even if dirty
42
+ - SIMP_BUILD_docs => 'yes' builds & includes documentation
43
+ - SIMP_BUILD_checkout => 'no' will skip the git repo checkouts
44
+ - SIMP_BUILD_bundle => 'no' skips running bundle in each subrepo
45
+ - SIMP_BUILD_unpack => 'no' skips the unpack section
46
+ - SIMP_BUILD_unpack_merge => 'no' prevents auto-merging the unpacked DVD
47
+ - SIMP_BUILD_prune => 'no' passes :prune=>false to iso:build
48
+ - SIMP_BUILD_iso_name => Renames the output ISO filename [Default: false]
49
+ - SIMP_BUILD_iso_tag => Appended to the output ISO's filename [Default: false]
50
+
51
+ Notes:
52
+ - To skip `tar:build` (including `pkg:build`), use the `tarball` argument
53
+ EOM
54
+
55
+ task :auto, [:release,
56
+ :iso_paths,
57
+ :tarball,
58
+ :output_dir,
59
+ :do_checksum,
60
+ :key_name,
61
+ :packer_vars,
62
+ :verbose] do |t, args|
63
+ # set up data
64
+ # --------------------------------------------------------------------------
65
+
66
+ args.with_defaults(
67
+ :iso_paths => Dir.pwd,
68
+ :tarball => 'false',
69
+ :output_dir => '',
70
+ :do_checksum => 'false',
71
+ :key_name => 'dev',
72
+ :packer_vars => 'true',
73
+ :verbose => 'false',
74
+ )
75
+
76
+ # locals
77
+ target_release = args[:release]
78
+ iso_paths = File.expand_path(args[:iso_paths])
79
+ tarball = (args.tarball =~ /^(false|)$/ ? false : args.tarball)
80
+ output_dir = args[:output_dir].sub(/^$/, File.expand_path( 'SIMP_ISO', Dir.pwd ))
81
+ do_checksum = (args.do_checksum =~ /^$/ ? 'false' : args.do_checksum)
82
+ key_name = args[:key_name]
83
+ staging_dir = ENV.fetch('SIMP_BUILD_staging_dir',
84
+ File.expand_path( 'SIMP_ISO_STAGING', Dir.pwd ))
85
+ do_packer_vars = (args.packer_vars == 'false' ? false : true)
86
+ verbose = (args.verbose == 'false' ? false : true)
87
+
88
+ yaml_file = File.expand_path('build/release_mappings.yaml', @base_dir)
89
+ pwd = Dir.pwd
90
+ repo_root_dir = File.expand_path( @base_dir )
91
+ method = ENV.fetch('SIMP_BUILD_puppetfile','tracking')
92
+ do_rm_staging = ENV['SIMP_BUILD_rm_staging_dir'] == 'yes'
93
+ do_docs = ENV['SIMP_BUILD_docs'] == 'yes' ? 'true' : 'false'
94
+ do_merge = ENV['SIMP_BUILD_unpack_merge'] != 'no'
95
+ do_prune = ENV['SIMP_BUILD_prune'] != 'no' ? 'true' : 'false'
96
+ do_checkout = ENV['SIMP_BUILD_checkout'] != 'no'
97
+ do_bundle = ENV['SIMP_BUILD_bundle'] != 'no'
98
+ do_unpack = ENV['SIMP_BUILD_unpack'] != 'no'
99
+ full_iso_name = ENV.fetch('SIMP_BUILD_iso_name', false)
100
+ iso_name_tag = ENV.fetch('SIMP_BUILD_iso_tag', false)
101
+ @dirty_repos = nil
102
+ @simp_output_iso = nil
103
+
104
+
105
+ # Build environment sanity checks
106
+ # --------------------
107
+ if do_rm_staging && !do_unpack
108
+ fail SIMPBuildException, "ERROR: Mixing `SIMP_BUILD_rm_staging_dir=yes` and `SIMP_BUILD_unpack=no` is silly."
109
+ end
110
+
111
+ if File.exists?(output_dir) && !File.directory?(output_dir)
112
+ fail SIMPBuildException, "ERROR: ISO output dir exists but is not a directory:\n\n" +
113
+ " '#{output_dir}'\n\n"
114
+ end
115
+
116
+
117
+ # Look up ISOs against known build assets
118
+ # --------------------
119
+ target_data = get_target_data(target_release, iso_paths, yaml_file, do_checksum, verbose )
120
+
121
+ # IDEA: check for prequisite build tools
122
+
123
+ # check out subrepos
124
+ # --------------------
125
+ if do_checkout
126
+ puts
127
+ puts '='*80
128
+ puts "## Checking out subrepositories"
129
+ puts
130
+ puts " (skip with `SIMP_BUILD_checkout=no`)"
131
+ puts '='*80
132
+ Dir.chdir repo_root_dir
133
+ Rake::Task['deps:status'].invoke
134
+ if @dirty_repos && !ENV['SIMP_BUILD_force_dirty'] == 'yes'
135
+ raise SIMPBuildException, "ERROR: Dirty repos detected! I refuse to destroy uncommitted work."
136
+ else
137
+ puts
138
+ puts '-'*80
139
+ puts "#### Checking out subrepositories using method '#{method}'"
140
+ puts '-'*80
141
+ Rake::Task['deps:checkout'].invoke(method)
142
+ end
143
+
144
+ if do_bundle
145
+ puts
146
+ puts '-'*80
147
+ puts "#### Running bundler in all repos"
148
+ puts ' (Disable with `SIMP_BUILD_bundle=no`)'
149
+ puts '-'*80
150
+ Rake::Task['build:bundle'].invoke
151
+ else
152
+ puts
153
+ puts '-'*80
154
+ puts "#### SKIPPED: bundler in all repos"
155
+ puts ' (Force with `SIMP_BUILD_bundle=yes`)'
156
+ puts '-'*80
157
+ end
158
+ else
159
+ puts
160
+ puts '='*80
161
+ puts "#### skipping sub repository checkout (because `SIMP_BUILD_checkout=no`)"
162
+ puts
163
+ end
164
+
165
+ # build tarball
166
+ # --------------------
167
+ if tarball
168
+ puts
169
+ puts '-'*80
170
+ puts "#### Using pre-existing tarball:"
171
+ puts " '#{tarball}'"
172
+ puts
173
+ puts '-'*80
174
+
175
+ ver=target_data['os_version'].split('.').first
176
+ repo_pkglist_file = File.expand_path( "src/DVD/#{ver}-simp_pkglist.txt",
177
+ repo_root_dir
178
+ )
179
+ if File.file? repo_pkglist_file
180
+ puts "#### setting SIMP_PKGLIST_FILE=#{repo_pkglist_file}"
181
+ ENV['SIMP_PKGLIST_FILE']=repo_pkglist_file
182
+ else
183
+ puts "#### WARNING: repo pkglist file not found:"
184
+ puts " '#{repo_pkglist_file}'"
185
+ puts
186
+ end
187
+ else
188
+ puts
189
+ puts '='*80
190
+ puts "#### Running tar:build in all repos"
191
+ puts '='*80
192
+ @simp_tarballs = {}
193
+ Rake::Task['tar:build'].invoke(target_data['mock'],key_name,do_docs)
194
+ tarball = @simp_tarballs.fetch(target_data['flavor'])
195
+ end
196
+
197
+ # yum sync
198
+ # --------------------
199
+ puts
200
+ puts '-'*80
201
+ puts "#### rake build:yum:sync[#{target_data['flavor']},#{target_data['os_version']}]"
202
+ puts '-'*80
203
+ Rake::Task['build:yum:sync'].invoke(target_data['flavor'],target_data['os_version'])
204
+
205
+ # If you have previously downloaded packages from yum, you may need to run
206
+ # $ rake build:yum:clean_cache
207
+
208
+ # Optionally, you may drop in custom packages you wish to have available during an install into build/yum_data/SIMP<simp_version>_<CentOS or RHEL><os_version>_<architecture>/packages
209
+ # TODO: ENV var for optional packages
210
+
211
+ prepare_staging_dir( staging_dir, do_rm_staging, repo_root_dir, verbose )
212
+ Dir.chdir staging_dir
213
+
214
+ #
215
+ # --------------------
216
+ if do_unpack
217
+ puts
218
+ puts '='*80
219
+ puts "#### unpack ISOs into staging directory"
220
+ puts " staging area: '#{staging_dir}'"
221
+ puts
222
+ puts " (skip with `SIMP_BUILD_unpack=no`)"
223
+ puts '='*80
224
+ puts
225
+ target_data['isos'].each do |iso|
226
+ puts "---- rake unpack[#{iso}]"
227
+ Dir.glob( File.join(staging_dir, "#{target_data['flavor']}*/") ).each do |f|
228
+ FileUtils.rm_f( f , :verbose => verbose )
229
+ end
230
+ Rake::Task['unpack'].invoke(iso,do_merge,Dir.pwd,'isoinfo',target_data['os_version'])
231
+ end
232
+ else
233
+ puts
234
+ puts '='*80
235
+ puts "#### skipping ISOs unpack (because `SIMP_BUILD_unpack=no`)"
236
+ puts
237
+ end
238
+
239
+ puts
240
+ puts '='*80
241
+ puts "#### iso:build[#{tarball}]"
242
+ puts '='*80
243
+ puts
244
+
245
+ Rake::Task['iso:build'].invoke(tarball,staging_dir,do_prune)
246
+
247
+ output_file = full_iso_name ? full_iso_name : @simp_output_iso
248
+ if iso_name_tag
249
+ output_file.sub!(/\.iso$/i, "__#{iso_name_tag}.iso")
250
+ end
251
+
252
+ puts
253
+ puts '='*80
254
+ puts "#### Moving '#{@simp_output_iso}' into:"
255
+ puts " '#{output_dir}/#{output_file}'"
256
+ puts '='*80
257
+ puts
258
+
259
+ iso = File.join(output_dir,output_file)
260
+ FileUtils.mkdir_p File.dirname(iso), :verbose => verbose
261
+ FileUtils.mv(@simp_output_iso, iso, :verbose => verbose)
262
+
263
+ # write vars.json for packer build
264
+ # --------------------------------------
265
+ vars_file = iso.sub(/.iso$/, '.json')
266
+ puts
267
+ puts '='*80
268
+ puts "#### Checksumming #{iso}..."
269
+ puts '='*80
270
+ puts
271
+
272
+ sum = `sha256sum "#{iso}"`.split(/ +/).first
273
+
274
+ puts
275
+ puts '='*80
276
+ puts "#### Writing packer data to:"
277
+ puts " '#{vars_file}'"
278
+ puts '='*80
279
+ puts
280
+ packer_vars = {
281
+ 'box_simp_release' => target_release,
282
+ 'box_distro_release' => "SIMP-#{target_release}-#{target_data['isos'].first.sub(/\.iso$|-x86_64/,'')}",
283
+ 'iso_url' => iso,
284
+ 'iso_checksum' => sum,
285
+ 'iso_checksum_type' => 'sha256',
286
+ 'new_password' => 'suP3rP@ssw0r!suP3rP@ssw0r!suP3rP@ssw0r!',
287
+ 'output_directory' => './OUTPUT',
288
+ }
289
+ File.open(vars_file, 'w'){|f| f.puts packer_vars.to_json }
290
+
291
+ puts
292
+ puts '='*80
293
+ puts "#### FINIS!"
294
+ puts '='*80
295
+ puts
296
+ end
297
+
298
+ end
299
+
300
+ def get_target_data(target_release, iso_paths, yaml_file, do_checksum, verbose )
301
+ puts '='*80
302
+ puts "## validating ISOs for target:"
303
+ puts " '#{target_release}' in '#{iso_paths}'"
304
+ puts '='*80
305
+ puts
306
+
307
+ mapper = Simp::Build::ReleaseMapper.new(target_release, yaml_file, do_checksum == 'true')
308
+ mapper.verbose = true || verbose
309
+ target_data = mapper.autoscan_unpack_list( iso_paths )
310
+
311
+ puts '-'*80
312
+ puts "## target data:"
313
+ puts ''
314
+ puts " target release: '#{target_release}'"
315
+ puts " target flavor: '#{target_data['flavor']}'"
316
+ puts " source isos:"
317
+ target_data['isos'].each do |iso|
318
+ puts " - #{iso}"
319
+ end
320
+ puts '-'*80
321
+ puts
322
+ sleep 3
323
+
324
+ target_data
325
+ end
326
+
327
+
328
+ def prepare_staging_dir( staging_dir, do_rm_staging, repo_root_dir, verbose )
329
+ if ['','/',Dir.home,repo_root_dir].include? staging_dir
330
+ fail SIMPBuildException,
331
+ "ERROR: staging directoy path is too stupid to be believed:\n"+
332
+ " '#{staging_dir}'\n\n" +
333
+ " Use SIMP_BUILD_staging_dir='path/to/staging/dir'\n\n"
334
+ end
335
+ if do_rm_staging
336
+ puts
337
+ puts '-'*80
338
+ puts '#### Ensuring previous staging directory is removed:'
339
+ puts " '#{staging_dir}'"
340
+ puts
341
+ puts ' (disable this with `SIMP_BUILD_rm_staging_dir=no`)'
342
+ puts '-'*80
343
+
344
+ FileUtils.rm_rf staging_dir, :verbose => verbose
345
+ elsif File.exists? staging_dir
346
+ warn ''
347
+ warn '!'*80
348
+ warn '#### WARNING: staging dir already exists at:'
349
+ warn " '#{staging_dir}'"
350
+ warn ''
351
+ warn ' - Previously staged assets in this directory may cause problems.'
352
+ warn ' - Use `SIMP_BUILD_rm_staging_dir=yes` to remove it automatically.'
353
+ warn ''
354
+ warn '!'*80
355
+ warn ''
356
+ sleep 10
357
+ end
358
+ FileUtils.mkdir_p staging_dir, :verbose => verbose
359
+ end
360
+ end
361
+ end
362
+ end
363
+
@@ -0,0 +1,707 @@
1
+ require 'simp/rake'
2
+ require 'simp/rake/build/constants'
3
+
4
+ module Simp; end
5
+ module Simp::Rake; end
6
+ module Simp::Rake::Build
7
+ class SIMPBuildException < Exception
8
+ end
9
+
10
+ include Simp::Rake
11
+
12
+ class Build < ::Rake::TaskLib
13
+ include Simp::Rake::Build::Constants
14
+
15
+ def initialize( base_dir )
16
+ init_member_vars( base_dir )
17
+ define_tasks
18
+ end
19
+
20
+ # define rake tasks
21
+ def define_tasks
22
+ namespace :build do
23
+ desc <<-EOM
24
+ Run bundle at every level of the project.
25
+
26
+ This taks runs 'bundle' at each level of the Git subproject tree as well as the top level.
27
+
28
+ The intent is to ensure that the entire development space is up to date when starting work.
29
+
30
+ Arguments:
31
+ * :action => The action that you with bundle to take. Default => 'install'
32
+ * :verbose => Enable verbose reporting. Default => 'false'
33
+ EOM
34
+
35
+ task :bundle, [:action, :verbose, :method] do |t, args|
36
+ args.with_defaults(:action => 'install')
37
+ args.with_defaults(:verbose => 'false')
38
+ args.with_defaults(:method => 'tracking')
39
+
40
+ verbose = args.verbose == 'false' ? false : true
41
+
42
+ # Grab all currently tracked submodules.
43
+ fake_lp = FakeLibrarian.new("Puppetfile.#{args[:method]}")
44
+ $modules = fake_lp.modules
45
+ mod_list = []
46
+ fake_lp.each_module do |environment,name, path|
47
+ if Dir.exists?(path)
48
+ mod_list.push(path)
49
+ end
50
+ end
51
+
52
+ failed_mods = Parallel.map(
53
+ mod_list,
54
+ :in_processes => 1,
55
+ :progress => t.name
56
+ ) do |mod|
57
+
58
+ status = true
59
+ next unless File.exists?(File.join(mod,'Gemfile'))
60
+ puts "\n#{mod}\n" if verbose
61
+ Dir.chdir(mod) do
62
+ if File.exist?('Gemfile.lock')
63
+ puts "Cleaning Gemfile.lock from #{mod}" if verbose
64
+ rm('Gemfile.lock')
65
+ end
66
+ # Any ruby code that opens a subshell will automatically use the current Bundler environment.
67
+ # Clean env will give bundler the environment present before Bundler is activated.
68
+ Bundler.with_clean_env do
69
+ out = %x(bundle #{args.action} 2>&1)
70
+ status = $?.success?
71
+ puts out if verbose
72
+ end
73
+ mod unless status
74
+ end
75
+ end
76
+
77
+ failed_mods.compact!
78
+ warn %(The following modules failed bundle #{args.action}:\n * #{failed_mods.sort.join("\n *")})
79
+ end
80
+
81
+ namespace :yum do
82
+ @base_dir = File.join(@build_dir,'yum_data')
83
+ @build_arch = 'x86_64'
84
+
85
+ ##############################################################################
86
+ # Helpers
87
+ ##############################################################################
88
+
89
+ # This is a complete hack to get around the fact that yumdownloader doesn't
90
+ # cache properly when setting TMPDIR.
91
+ def clean_yumdownloader_cache_dir
92
+ require 'etc'
93
+
94
+ tmp_dir = '/tmp'
95
+ tmp_dir = ENV['TMPDIR'] if ENV['TMPDIR']
96
+
97
+ Dir.glob("#{tmp_dir}/yum-#{Etc.getlogin}-*").each do |cache_dir|
98
+ cache_dir = File.expand_path(cache_dir)
99
+
100
+ puts "Cleaning Yumdownloader Cache Dir '#{cache_dir}'"
101
+ begin
102
+ rm_rf(cache_dir)
103
+ rescue Exception => e
104
+ puts "Could not remove cache Dir: #{e}"
105
+ end
106
+ end
107
+ end
108
+
109
+ # Return the target directory
110
+ # Expects one argument wich is the 'arguments' hash to one of the tasks.
111
+ def get_target_dir(args)
112
+ fail("Error: You must specify 'os'") unless args.os
113
+ fail("Error: You must specify 'os_version'") unless args.os_version
114
+ fail("Error: You must specify both major and minor version for the OS") unless args.os_version =~ /^.+\..+$/
115
+ fail("Error: You must specify 'simp_version'") unless args.simp_version
116
+ fail("Error: You must specify 'arch'") unless args.arch
117
+
118
+ # Yes, this is a kluge but the amount of variable passing that would need
119
+ # to be done to support this is silly.
120
+ @build_arch = args.arch
121
+
122
+ return File.join(
123
+ @base_dir,
124
+ "SIMP#{args.simp_version}_#{args.os}#{args.os_version}_#{args.arch}"
125
+ )
126
+ end
127
+
128
+ # Return where YUM finds the passed RPM
129
+ def get_rpm_source(rpm,yum_conf)
130
+ # Do we have what we need?
131
+
132
+ unless @have_yumdownloader
133
+ %x(yumdownloader --version > /dev/null 2>&1)
134
+
135
+ if $?.exitstatus == 127
136
+ fail("Error: Could not find 'yumdownloader'. Please install it and try again.")
137
+ end
138
+ @have_yumdownloader = true
139
+ end
140
+
141
+ source = nil
142
+
143
+ puts("Looking up: #{rpm}")
144
+ sources = %x(yumdownloader -c #{yum_conf} --urls #{File.basename(rpm,'.rpm')} 2>/dev/null )
145
+
146
+ unless $?.success?
147
+ raise(SIMPBuildException,"Could not find a download source")
148
+ end
149
+
150
+ sources = sources.split("\n").grep(%r(\.rpm$))
151
+
152
+ if sources.empty?
153
+ raise(SIMPBuildException,'No Sources found')
154
+ else
155
+ native_sources = sources.grep(%r((#{@build_arch}|noarch)\.rpm$))
156
+
157
+ # One entry, one success
158
+ if native_sources.size == 1
159
+ source = native_sources.first
160
+ # More than one entry is no good.
161
+ elsif native_sources.size > 1
162
+ raise(SIMPBuildException,'More than one file found')
163
+ # The only entry found was for a non-native architecure
164
+ # This means that someone specified the arch explicitly at the
165
+ # command line and we should take it.
166
+ else
167
+ source = sources.first
168
+ end
169
+ end
170
+
171
+ return source
172
+ end
173
+
174
+ # Snag an RPM via YUM.
175
+ # Returns where the tool got the file from.
176
+ #
177
+ # If passed a source, simply downloads the file into the packages directory
178
+ def download_rpm(rpm, yum_conf, source=nil, distro_dir=Dir.pwd)
179
+ # We're doing this so that we can be 100% sure that we're pulling the RPM
180
+ # from where the last command indicated. YUM can choose multiple sources
181
+ # and we definitely want the one that we actually state!
182
+ source = get_rpm_source(rpm,yum_conf) unless source
183
+
184
+ Dir.chdir('packages') do
185
+ full_pkg = source.split('/').last
186
+ unless File.exist?(full_pkg)
187
+ puts("Downloading: #{full_pkg}")
188
+ %x(curl -L --max-redirs 10 -s -o "#{full_pkg}" -k "#{source}")
189
+
190
+ unless $?.success?
191
+ raise(SIMPBuildException,"Could not download")
192
+ end
193
+
194
+ begin
195
+ validate_rpm(full_pkg)
196
+ rescue SIMPBuildException
197
+ rm(full_pkg) if File.exist?(full_pkg)
198
+ raise(SIMPBuildException,"#{rpm} could not be downloaded")
199
+ end
200
+ end
201
+ end
202
+
203
+ return source
204
+ end
205
+
206
+ # Check to see if an RPM is actually a valid RPM
207
+ # Optionally remove any invalid RPMS.
208
+ #
209
+ # Returns true if the rm is valid raises a SIMPBuildException otherwise
210
+ def validate_rpm(rpm, clean=true)
211
+ # Check to see if what we got is actually valid
212
+ %x(rpm -K --nosignature "#{rpm}" 2>&1 > /dev/null)
213
+
214
+ unless $?.success?
215
+ errmsg = "RPM '#{rpm}' is invalid"
216
+
217
+ if clean
218
+ errmsg += ', removing'
219
+ FileUtils.rm(rpm)
220
+ end
221
+
222
+ raise(SIMPBuildException,errmsg)
223
+ end
224
+
225
+ true
226
+ end
227
+
228
+ # Create the YUM config file
229
+ # * yum_tmp => The directory in which to store the YUM DB and any other
230
+ # temporary files.
231
+ #
232
+ # Returns the location of the YUM Configuration
233
+ def generate_yum_conf(distro_dir=Dir.pwd)
234
+ yum_conf_template = <<-EOM.gsub(/^\s+/,'')
235
+ [main]
236
+ keepcache = 1
237
+ persistdir = <%= yum_cache %>
238
+ logfile = <%= yum_logfile %>
239
+ exactarch = 1
240
+ obsoletes = 0
241
+ gpgcheck = 1
242
+ plugins = 1
243
+ reposdir = <%= repo_dirs.join(' ') %>
244
+ assumeyes = 1
245
+ EOM
246
+
247
+ yum_conf = nil
248
+ Dir.chdir(distro_dir) do
249
+ # Create the target directory
250
+ yum_tmp = File.join('packages','yum_tmp')
251
+ mkdir_p(yum_tmp) unless File.directory?(yum_tmp)
252
+
253
+ yum_cache = File.expand_path(File.join(yum_tmp,'yum_cache'))
254
+ mkdir_p(yum_cache) unless File.directory?(yum_cache)
255
+
256
+ yum_logfile = File.expand_path(File.join(yum_tmp,'yum.log'))
257
+
258
+ repo_dirs = []
259
+ # Add the global directory
260
+ repo_dirs << File.expand_path('../my_repos')
261
+ if File.directory?('my_repos')
262
+ # Add the local user repos if they exist
263
+ repo_dirs << File.expand_path('my_repos')
264
+ else
265
+ # Add the default Internet repos otherwise
266
+ repo_dirs << File.expand_path('repos')
267
+ end
268
+
269
+ # Create our YUM config file
270
+ yum_conf = File.join(yum_tmp,'yum.conf')
271
+ File.open(yum_conf,'w') do |fh|
272
+ fh.write(ERB.new(yum_conf_template,nil,'-').result(binding))
273
+ end
274
+ end
275
+
276
+ return yum_conf
277
+ end
278
+
279
+ def get_known_packages(target_dir=Dir.pwd)
280
+ known_package_hash = {}
281
+
282
+ Dir.chdir(target_dir) do
283
+ if File.exist?('packages.yaml')
284
+ known_package_hash = YAML::load_file('packages.yaml')
285
+ end
286
+ end
287
+
288
+ unless known_package_hash.empty?
289
+ unless known_package_hash.first.last[:rpm_name]
290
+ # Convert from Legacy
291
+ # This is imperfect since we can't accurately determine the RPM sort
292
+ # name but the code should straighten everything out since we rewrite
293
+ # the entire file based on what has been downloaded.
294
+ new_package_hash = known_package_hash.dup
295
+
296
+ known_package_hash.each_key { |k|
297
+ new_package_hash[k][:rpm_name] = k
298
+ }
299
+
300
+ known_package_hash = new_package_hash
301
+ end
302
+ end
303
+
304
+ return known_package_hash
305
+ end
306
+
307
+ def get_downloaded_packages(target_dir=Dir.pwd)
308
+ downloaded_packages = {}
309
+
310
+ Dir.chdir(target_dir) do
311
+ Dir.glob('packages/*.rpm').each do |pkg|
312
+ downloaded_packages[Simp::RPM.get_info(pkg)[:name]] = { :rpm_name => File.basename(pkg) }
313
+ end
314
+ end
315
+
316
+ return downloaded_packages
317
+ end
318
+
319
+ # Update the packages.yaml and packages/ directories
320
+ # * target_dir => The actual distribution directory where packages.yaml and
321
+ # packages/ reside.
322
+ def update_packages(target_dir,bootstrap=false)
323
+ # This really should never happen....
324
+ unless File.directory?(target_dir)
325
+ fail <<-EOM
326
+ Error: Could not update packages.
327
+
328
+ Target directory '#{target_dir}' does not exist!
329
+ EOM
330
+ end
331
+
332
+ Dir.chdir(target_dir) do
333
+ unless File.exist?('packages.yaml') || File.directory?('packages')
334
+ fail <<-EOM
335
+ Error: Either 'pacakges.yaml' or the 'packages/' directory need to exist under '#{target_dir}
336
+ EOM
337
+ end
338
+
339
+ yum_conf = generate_yum_conf
340
+
341
+ known_package_hash = get_known_packages
342
+ downloaded_package_hash = get_downloaded_packages
343
+
344
+ # This holds packages for which we could not find a source.
345
+ unknown_package_hash = {}
346
+
347
+ known_packages = known_package_hash.keys.collect{ |pkg|
348
+ pkg = known_package_hash[pkg][:rpm_name]
349
+ }.compact
350
+
351
+ downloaded_packages = downloaded_package_hash.keys.collect{ |pkg|
352
+ pkg = downloaded_package_hash[pkg][:rpm_name]
353
+ }.compact
354
+
355
+ if known_packages.empty? && downloaded_packages.empty?
356
+ fail <<-EOM
357
+ Error: Could not find anything to do!
358
+
359
+ In #{target_dir}:
360
+ No packages in either packages.yaml or the packages/ directory
361
+ EOM
362
+ end
363
+
364
+ failed_updates = {}
365
+
366
+ # Kill any pre-existing invalid packages that might be hanging around
367
+ downloaded_packages.dup.each do |package|
368
+ begin
369
+ validate_rpm(%(packages/#{package}))
370
+ rescue SIMPBuildException => e
371
+ rm(%(packages/#{package})) if File.exist?(%(packages/#{package}))
372
+ downloaded_packages.delete(package)
373
+ failed_updates[package] = e
374
+ end
375
+ end
376
+
377
+ # Let's go ahead and grab everything that we know the source for
378
+ (known_packages - downloaded_packages).sort.each do |package_to_download|
379
+ begin
380
+ # Do we have a valid external source?
381
+ package_source = known_package_hash.find{|k,h| h[:rpm_name] == package_to_download}.last[:source]
382
+ if package_source && (package_source =~ %r(^[a-z]+://))
383
+ download_rpm(package_to_download,yum_conf,package_source)
384
+ else
385
+ # If you get here, then you'll need to have an internal mirror of the
386
+ # repositories in question. This covers things like private RPMs as
387
+ # well as Commercial RPMs from Red Hat.
388
+ download_rpm(package_to_download,yum_conf)
389
+ end
390
+ rescue SIMPBuildException => e
391
+ base_package_name = known_package_hash.find{|k,h| h[:rpm_name] == package_to_download}.first
392
+ updated_package = update_rpm(base_package_name,yum_conf,true)
393
+
394
+ if updated_package
395
+ updated_package_rpm_name = updated_package[base_package_name][:rpm_name]
396
+
397
+ # We just got a new one! Replace the old one.
398
+ puts "Updating: #{package_to_download} with #{updated_package_rpm_name}"
399
+
400
+ # We now know about this
401
+ downloaded_packages.delete(package_to_download)
402
+ downloaded_package_hash.merge!(updated_package)
403
+ known_package_hash.merge!(updated_package)
404
+ known_packages << updated_package_rpm_name
405
+ else
406
+ failed_updates[package_to_download] = e
407
+ end
408
+ end
409
+ end
410
+
411
+ # Now, let's update the known_packages data structure for anything that's
412
+ # new!
413
+ (downloaded_packages - known_packages).each do |package|
414
+ downloaded_package_hash.keys.each do |key|
415
+ if downloaded_package_hash[key][:rpm_name] == package
416
+ begin
417
+ rpm_source = get_rpm_source(package,yum_conf)
418
+ known_package_hash[key] = downloaded_package_hash[key]
419
+ known_package_hash[key][:source] = rpm_source
420
+ rescue SIMPBuildException => e
421
+ unknown_package_hash[key] = {} unless unknown_package_hash[key]
422
+ unknown_package_hash[key][:rpm_name] = package
423
+ failed_updates[package] = e
424
+ end
425
+ break
426
+ end
427
+ end
428
+ end
429
+
430
+ # OK! In theory, we're done with all of this nonsense! Let's update the
431
+ # YAML file.
432
+ File.open('packages.yaml','w') do |fh|
433
+ sorted_packages = {}
434
+ known_package_hash.keys.sort.each do |k|
435
+ # Make sure we don't capture any legacy malformed info
436
+ if known_package_hash[k][:rpm_name][-4..-1] == '.rpm'
437
+ sorted_packages[k] = known_package_hash[k]
438
+ end
439
+ end
440
+ fh.puts(sorted_packages.to_yaml)
441
+ end
442
+
443
+ if unknown_package_hash.empty?
444
+ rm('unknown_packages.yaml') if File.exist?('unknown_packages.yaml')
445
+ else
446
+ # Next, let's freshen up the unknown packages reference file
447
+ File.open('unknown_packages.yaml','w') do |fh|
448
+ sorted_packages = {}
449
+ unknown_package_hash.keys.sort.each do |k|
450
+ sorted_packages[k] = unknown_package_hash[k]
451
+ end
452
+ fh.puts(sorted_packages.to_yaml)
453
+ end
454
+ end
455
+
456
+ # Now, let's tell the user what went wrong.
457
+ unless failed_updates.empty?
458
+ $stderr.puts("Warning: There were errors updating some files:")
459
+
460
+ failed_updates.keys.sort.each do |k|
461
+ $stderr.puts(" * #{k} => #{failed_updates[k]}")
462
+ end
463
+ end
464
+ end
465
+ end
466
+
467
+ # Downloads a packge into the *current working directory*
468
+ #
469
+ # Will create a directory called 'obsolete' and move any old packages into
470
+ # that location if a newer one is found.
471
+ #
472
+ # Arguments:
473
+ # * pkg => The name of the package to download. YUM supported globs
474
+ # are allowed.
475
+ # * verbose => If true, identify potential actions of note.
476
+ #
477
+ # Returns a hash of the new package information if found.
478
+ #
479
+ def update_rpm(pkg,yum_conf,verbose=false)
480
+ updated_pkg = nil
481
+
482
+ begin
483
+ new_pkg_source = download_rpm(pkg, yum_conf)
484
+ new_pkg = new_pkg_source.split('/').last
485
+
486
+ Dir.chdir('packages') do
487
+ new_pkg_info = Simp::RPM.new(new_pkg)
488
+
489
+ # Find any old packages and move them into the 'obsolete' directory.
490
+ Dir.glob("#{new_pkg_info.basename}*.rpm").each do |old_pkg|
491
+ old_pkg_info = Simp::RPM.new(old_pkg)
492
+ # Don't obsolete yourself!
493
+ next unless new_pkg_info.basename == old_pkg_info.basename
494
+
495
+ %x(rpmdev-vercmp #{new_pkg_info.full_version} #{old_pkg_info.full_version})
496
+ exit_status = $?.exitstatus
497
+ if exit_status == 11
498
+ mkdir('obsolete') unless File.directory?('obsolete')
499
+
500
+ puts("Retiring #{old_pkg}") if verbose
501
+
502
+ mv(old_pkg,'obsolete')
503
+ end
504
+ end
505
+
506
+ updated_pkg = {
507
+ new_pkg_info.basename => {
508
+ :source => new_pkg_source,
509
+ :rpm_name => new_pkg
510
+ }
511
+ }
512
+ end
513
+ rescue SIMPBuildException => e
514
+ puts("Failed to update #{pkg} -> #{e}") if verbose
515
+ end
516
+
517
+ updated_pkg
518
+ end
519
+
520
+ ##############################################################################
521
+ # Main tasks
522
+ ##############################################################################
523
+
524
+ desc <<-EOM
525
+ Create a workspace for a new distribution.
526
+
527
+ Creates a YUM scaffold under
528
+ #{@build_dir}/yum_data/SIMP{:simp_version}_{:os}{:os_version}_{:arch}.
529
+
530
+ * :os - The Operating System that you wish to use.
531
+ Supported OSs: #{@target_dists}.join(', ')
532
+ * :os_version - The Major and Minor version of the OS. Ex: 6.6, 7.0, etc...
533
+ * :simp_version - The Full version of SIMP that you are going to support. Do
534
+ not enter the revision number. Ex: 5.1.0, 4.2.0.
535
+ Default: Auto
536
+
537
+ * :arch - The architecture that you support. Default: x86_64
538
+ EOM
539
+ task :scaffold,[:os,:os_version,:simp_version,:arch] do |t,args|
540
+ # @simp_version is set in the main Rakefile
541
+ args.with_defaults(:simp_version => @simp_version.split('-').first)
542
+ args.with_defaults(:arch => @build_arch)
543
+
544
+ target_dir = get_target_dir(args)
545
+
546
+ unless File.exist?(target_dir)
547
+ mkdir_p(target_dir)
548
+ puts("Created #{target_dir}")
549
+ end
550
+
551
+ # Put together the rest of the scaffold directories
552
+ Dir.chdir(@base_dir) do
553
+ mkdir('my_repos') unless File.exist?('my_repos')
554
+ end
555
+
556
+ Dir.chdir(target_dir) do
557
+ mkdir('repos') unless File.exist?('repos')
558
+ mkdir('packages') unless File.exist?('packages')
559
+ end
560
+ end
561
+
562
+ desc <<-EOM
563
+ Download ALL THE PACKAGES.
564
+
565
+ Downloads everything as appropriate for the distribution at
566
+ #{@build_dir}/yum_data/SIMP{:simp_version}_{:os}{:os_version}_{:arch}.
567
+
568
+ * :os - The Operating System that you wish to use.
569
+ Supported OSs: #{@target_dists}.join(', ')
570
+ * :os_version - The Major and Minor version of the OS. Ex: 6.6, 7.0, etc...
571
+ * :simp_version - The Full version of SIMP that you are going to support. Do
572
+ not enter the revision number. Ex: 5.1.0, 4.2.0.
573
+ Default: Auto
574
+
575
+ * :arch - The architecture that you support. Default: x86_64
576
+ EOM
577
+ task :sync,[:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
578
+ # @simp_version is set in the main Rakefile
579
+ args.with_defaults(:simp_version => @simp_version.split('-').first)
580
+ args.with_defaults(:arch => @build_arch)
581
+
582
+ target_dir = get_target_dir(args)
583
+
584
+ update_packages(target_dir)
585
+ end
586
+
587
+ desc <<-EOM
588
+ Display the difference between record and download.
589
+
590
+ Provides a list of the differences between what we have recorded in
591
+ 'packages.yaml' and the downloads in the 'packages' directory.
592
+
593
+ * :os - The Operating System that you wish to use.
594
+ Supported OSs: #{@target_dists}.join(', ')
595
+ * :os_version - The Major and Minor version of the OS. Ex: 6.6, 7.0, etc...
596
+ * :simp_version - The Full version of SIMP that you are going to support. Do
597
+ not enter the revision number. Ex: 5.1.0, 4.2.0.
598
+ Default: Auto
599
+
600
+ * :arch - The architecture that you support. Default: x86_64
601
+ EOM
602
+ task :diff,[:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
603
+ args.with_defaults(:simp_version => @simp_version.split('-').first)
604
+ args.with_defaults(:arch => @build_arch)
605
+
606
+ differences_found = false
607
+
608
+ target_dir = get_target_dir(args)
609
+
610
+ known_package_hash = get_known_packages(target_dir)
611
+ downloaded_package_hash = get_downloaded_packages(target_dir)
612
+
613
+ known_packages = known_package_hash.keys.compact
614
+ downloaded_packages = downloaded_package_hash.keys.compact
615
+
616
+ known_not_downloaded = (known_packages - downloaded_packages).sort
617
+ unless known_not_downloaded.empty?
618
+ differences_found = true
619
+
620
+ puts("=== Packages Not Downloaded ===")
621
+ known_not_downloaded.each do |package|
622
+ puts " - #{package}"
623
+ end
624
+ end
625
+
626
+ downloaded_not_known = (downloaded_packages - known_packages).sort
627
+ unless downloaded_not_known.empty?
628
+ differences_found = true
629
+
630
+ puts ("=== Pacakges Downloaded not Recorded ===")
631
+ downloaded_not_known.each do |package|
632
+ puts " ~ #{downloaded_package_hash[package][:rpm_name]}"
633
+ end
634
+ end
635
+
636
+ if differences_found
637
+ exit 1
638
+ else
639
+ puts("=== No Differences Found ===")
640
+ exit 0
641
+ end
642
+ end
643
+
644
+ desc <<-EOM
645
+ Download an RPM for the given distribution.
646
+
647
+ Fetches the *latest* version of an RPM from the included sources and places
648
+ it in the downloads directory.
649
+
650
+ Any old versions are moved into an 'obsolete' directory.
651
+
652
+ This does *not* update the packages.yaml file.
653
+
654
+ Note: for convienience, you can specify the output of yum_diff as your input
655
+ package and it will try to pull down everything in the file.
656
+ * If you do this, it must be the *full path* to the file.
657
+
658
+ * :pkg - The package that you wish to download.
659
+ * :os - The Operating System that you wish to use.
660
+ Supported OSs: #{@target_dists}.join(', ')
661
+ * :os_version - The Major and Minor version of the OS. Ex: 6.6, 7.0, etc...
662
+ * :simp_version - The Full version of SIMP that you are going to support. Do
663
+ not enter the revision number. Ex: 5.1.0, 4.2.0.
664
+ Default: Auto
665
+
666
+ * :arch - The architecture that you support. Default: x86_64
667
+ EOM
668
+ task :fetch,[:pkg,:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
669
+ args.with_defaults(:simp_version => @simp_version.split('-').first)
670
+ args.with_defaults(:arch => @build_arch)
671
+
672
+ fail("Error: You must specify 'pkg'") unless args.pkg
673
+
674
+ pkgs = []
675
+ # Handle the output of build:yum_diff
676
+ if File.readable?(args.pkg)
677
+ File.read(args.pkg).each_line do |line|
678
+ if line =~ /\s+~\s+(.*)/
679
+ pkgs << $1.split(/-\d+/).first
680
+ end
681
+ end
682
+ else
683
+ # Handle the default case
684
+ pkgs = [args.pkg]
685
+ end
686
+
687
+ Dir.chdir(get_target_dir(args)) do
688
+ pkgs.each do |pkg|
689
+ # Pull down the RPM
690
+ update_rpm(pkg,generate_yum_conf,true)
691
+ end
692
+ end
693
+ end
694
+
695
+ desc <<-EOM
696
+ Clean the Yumdownloader cache.
697
+
698
+ Use this if you're having strange issues fetching packages.
699
+ EOM
700
+ task :clean_cache do
701
+ clean_yumdownloader_cache_dir
702
+ end
703
+ end
704
+ end
705
+ end
706
+ end
707
+ end