simp-rake-helpers 1.0.15 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,287 @@
1
+ require 'simp/rake'
2
+ require 'simp/rake/build/constants'
3
+
4
+ module Simp; end
5
+ module Simp::Rake; end
6
+ module Simp::Rake::Build
7
+
8
+ class Iso < ::Rake::TaskLib
9
+ include Simp::Rake
10
+ include Simp::Rake::Build::Constants
11
+
12
+ def initialize( base_dir )
13
+ init_member_vars( base_dir )
14
+ @mock = ENV['mock'] || '/usr/bin/mock'
15
+ define_tasks
16
+ end
17
+
18
+ def define_tasks
19
+
20
+ File.umask(0007)
21
+
22
+ namespace :iso do
23
+
24
+ # Remove packages from the given directory. The goal of this method is to help
25
+ # get the distro to be as small as possible.
26
+ # [:from_dir] Root directory to remove packages from (removes recursively)
27
+ # [:exclude_dirs] Array of directories to not remove any packages from
28
+ # [:exclude_pkgs] Array of packages to not remove
29
+ def prune_packages(from_dir,exclude_dirs,exclude_pkgs,mkrepo='createrepo -p',use_hack=true)
30
+ $stderr.puts "Starting to prune..."
31
+ Dir.chdir(from_dir) do
32
+ prune_count = 0
33
+
34
+ Find.find('.') do |path|
35
+ Find.prune if exclude_dirs.include?(File.basename(path))
36
+
37
+ if File.basename(path) =~ /.*\.rpm/
38
+ # Get the package name from the RPM.
39
+ # Note: an alternative method may be to just simply check the names
40
+ # of the RPMs themselves instead of the names of the packages.
41
+ pkg = nil
42
+ if use_hack
43
+ # The proper way (defined below) is way too slow, so this hack helps
44
+ # speed up the process by reading the file directly. If the code is
45
+ # not working, attempt this without using the hack, just be ready
46
+ # to wait a long time for the code to complete.
47
+ pkgname = File.basename(path).split('-').first
48
+ File.open(path,'r').each_line do |line|
49
+ if encode_line(line) =~ /C\000(\S+\000)?(#{Regexp.escape(pkgname)}\S*)\000/
50
+ pkg = $2.split(/\000/).first
51
+ break
52
+ end
53
+ end
54
+ else
55
+ # Proper way to obtain the RPM's package name, but WAY too slow
56
+ pkg = %x{rpm -qp --qf "%{NAME}" #{path} 2>/dev/null}.chomp
57
+ end
58
+
59
+ unless exclude_pkgs.include?(pkg)
60
+ rm(path)
61
+ prune_count += 1
62
+ end
63
+ end
64
+ end
65
+ $stderr.puts "Info: Pruned #{prune_count} packages from #{from_dir}"
66
+
67
+ if prune_count > 0
68
+ # Recreate the now-pruned repos
69
+ basepath = '.'
70
+ if (File.basename(from_dir) =~ /^RHEL/)
71
+ # This covers old versions of RHEL that don't follow the new
72
+ # way of doing things.
73
+ unless Dir.glob("Server/*.rpm").empty?
74
+ basepath = 'Server'
75
+ end
76
+ end
77
+
78
+ Dir.chdir(basepath) do
79
+ cp(Dir.glob("repodata/*comps*.xml").first,"simp_comps.xml")
80
+ sh %{#{mkrepo} -g simp_comps.xml .}
81
+ rm("simp_comps.xml")
82
+ end
83
+ end
84
+ end
85
+ end # End of prune_packages
86
+
87
+ desc <<-EOM
88
+ Build the SIMP ISO(s).
89
+ * :tarball - Path of the source tarball or directory containing the source
90
+ tarballs.
91
+ * :unpacked_dvds - Path of the directory containing the unpacked base OS
92
+ directories. Default is the current directory.
93
+ * :prune - Flag for whether unwanted packages should be pruned prior to
94
+ building the ISO. Default is true.
95
+ EOM
96
+ task :build,[:tarball,:unpacked_dvds,:prune] do |t,args|
97
+ args.with_defaults(:unpacked_dvds => "#{@run_dir}", :prune => 'true')
98
+
99
+ if args.tarball.nil?
100
+ fail("Error: You must specify a source tarball or tarball directory!")
101
+ else
102
+ tarball = File.expand_path(args.tarball)
103
+ unless File.exist?(tarball)
104
+ fail("Error: Could not find tarball at '#{tarball}'!")
105
+ end
106
+ end
107
+
108
+ tarfiles = File.directory?(tarball) ?
109
+ Dir.glob("#{tarball}/*.tar.gz") : [tarball]
110
+ vermap = YAML::load_file("#{@base_dir}/rakefiles/vermap.yaml")
111
+
112
+ tarfiles.each do |tarball|
113
+ namepieces = File.basename(tarball,".tar.gz").split('-')
114
+ simpver = namepieces[3..-1].join('-')
115
+ baseos = namepieces[2]
116
+
117
+ iso_dirs = Dir.glob("#{File.expand_path(args.unpacked_dvds)}/#{baseos}*")
118
+ if iso_dirs.empty?
119
+ fail("Error: No unpacked DVD directories found for '#{baseos}' under '#{File.expand_path(args.unpacked_dvds)}'")
120
+ end
121
+
122
+ # Process each unpacked base OS ISO directory found
123
+ iso_dirs.each do |dir|
124
+ baseosver = '???'
125
+ arch = '???'
126
+
127
+ # read the .treeinfo file (INI format)
128
+ # ------------------------------------
129
+ require 'puppet'
130
+ require 'puppet/util/inifile'
131
+
132
+ file = "#{dir}/.treeinfo"
133
+ fail("ERROR: no file '#{file}'") unless File.file?(file)
134
+
135
+ ini = Puppet::Util::IniConfig::PhysicalFile.new(file)
136
+ ini.read
137
+ sections = ini.sections.map{ |s| s.name }
138
+
139
+ # NOTE: RHEL7 discs claim [general] section is deprecated.
140
+ if sections.include?('general')
141
+ h = Hash[ ini.get_section('general').entries ]
142
+ arch = h.fetch('arch', arch).strip
143
+ baseosver = h.fetch('version', baseosver).strip
144
+ baseosver += '.0' if (baseosver.count('.') < 1)
145
+ end
146
+ # ------------------------------------
147
+
148
+ # Skip if SIMP version doesn't match target base OS version
149
+ next unless vermap[simpver.split('.').first].eql?(baseosver.split('.').first)
150
+
151
+ mkrepo = baseosver.split('.').first == '5' ? 'createrepo -s sha -p' : 'createrepo -p'
152
+
153
+ @simp_dvd_dirs.each do |clean_dir|
154
+ if File.directory?("#{dir}/#{clean_dir}")
155
+ rm_rf("#{dir}/#{clean_dir}")
156
+ elsif File.file?("#{dir}/#{clean_dir}")
157
+ fail("Error: #{dir}/#{clean_dir} is a file, expecting directory!")
158
+ end
159
+ end
160
+
161
+ # Prune unwanted packages
162
+ begin
163
+ system("tar --no-same-permissions -C #{dir} -xzf #{tarball} *simp_pkglist.txt")
164
+ rescue
165
+ # Does not matter if the command fails
166
+ end
167
+ pkglist_file = ENV.fetch(
168
+ 'SIMP_PKGLIST_FILE',
169
+ File.join(dir,"#{baseosver.split('.').first}-simp_pkglist.txt")
170
+ )
171
+ puts
172
+ puts '-'*80
173
+ puts "### Pruning packages not in file '#{pkglist_file}'"
174
+ puts
175
+ puts ' (override this with `SIMP_PKGLIST_FILE=<file>`)'
176
+ puts
177
+ puts '-'*80
178
+ puts
179
+
180
+ if (args.prune.casecmp("false") != 0) && File.exist?(pkglist_file)
181
+ exclude_pkgs = Array.new
182
+ File.read(pkglist_file).each_line do |line|
183
+ next if line =~ /^(\s+|#.*)$/
184
+ exclude_pkgs.push(line.chomp)
185
+ end
186
+ prune_packages(dir,['SIMP'],exclude_pkgs,mkrepo)
187
+ end
188
+
189
+ # Add the SIMP code
190
+ system("tar --no-same-permissions -C #{dir} -xzf #{tarball}")
191
+
192
+ Dir.chdir("#{dir}/SIMP") do
193
+ # Add the SIMP Dependencies
194
+ simp_base_ver = simpver.split('-').first
195
+ simp_dep_src = %(SIMP#{simp_base_ver}_#{baseos}#{baseosver}_#{arch})
196
+ yum_dep_location = File.join(@build_dir,'yum_data',simp_dep_src,'packages')
197
+
198
+ unless File.directory?(yum_dep_location)
199
+ fail("Could not find dependency directory at #{yum_dep_location}")
200
+ end
201
+
202
+ yum_dep_rpms = Dir.glob(File.join(yum_dep_location,'*.rpm'))
203
+ if yum_dep_rpms.empty?
204
+ fail("Could not find any dependency RPMs at #{yum_dep_location}")
205
+ end
206
+
207
+ # Add any one-off RPMs that you might want to add to your own build
208
+ # These are *not* checked to make sure that they actually match your
209
+ # environment
210
+ aux_packages = File.join(@build_dir,'yum_data',simp_dep_src,'aux_packages')
211
+ if File.directory?(aux_packages)
212
+ yum_dep_rpms += Dir.glob(File.join(aux_packages,'*.rpm'))
213
+ end
214
+
215
+ yum_dep_rpms.each do |rpm|
216
+ rpm_arch = rpm.split('.')[-2]
217
+
218
+ unless File.directory?(rpm_arch)
219
+ mkdir(rpm_arch)
220
+ end
221
+
222
+ # Just in case this is a symlink, broken, or some other nonsense.
223
+ target_file = File.join(rpm_arch,File.basename(rpm))
224
+ rm_f(target_file) if File.exist?(target_file)
225
+
226
+ cp(rpm,rpm_arch)
227
+ end
228
+
229
+ fail("Could not find architecture '#{arch}' in the SIMP distribution") unless File.directory?(arch)
230
+ # Get everything set up properly...
231
+ Dir.chdir(arch) do
232
+ Dir.glob('../*') do |rpm_dir|
233
+ # Don't dive into ourselves
234
+ next if File.basename(rpm_dir) == arch
235
+
236
+ Dir.glob(%(#{rpm_dir}/*.rpm)) do |source_rpm|
237
+ link_target = File.basename(source_rpm)
238
+ if File.exist?(source_rpm) && File.exist?(link_target)
239
+ next if Pathname.new(source_rpm).realpath == Pathname.new(link_target).realpath
240
+ end
241
+
242
+ ln_sf(source_rpm,link_target)
243
+ end
244
+ end
245
+
246
+ fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
247
+ end
248
+ end
249
+
250
+ # Make sure we have all of the necessary RPMs!
251
+ Rake::Task['pkg:repoclosure'].invoke(File.expand_path(dir))
252
+
253
+ # Do some sane chmod'ing and build ISO
254
+ system("chmod -fR u+rwX,g+rX,o=g #{dir}")
255
+ @simp_output_iso = "SIMP-#{simpver}-#{baseos}-#{baseosver}-#{arch}.iso"
256
+ system("mkisofs -uid 0 -gid 0 -o #{@simp_output_iso} -b isolinux/isolinux.bin -c boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -r -m TRANS.TBL #{dir}")
257
+ end
258
+ end # End of tarfiles loop
259
+ end
260
+
261
+ desc <<-EOM
262
+ Build the source ISO.
263
+ Note: The process clobbers the temporary and built files, rebuilds the
264
+ tarball(s) and packages the source ISO. Therefore it will take a
265
+ while.
266
+ * :key - The GPG key to sign the RPMs with. Defaults to 'prod'.
267
+ * :chroot - An optional Mock Chroot. If this is passed, the tar:build task will be called.
268
+ EOM
269
+ task :src,[:key,:chroot] do |t,args|
270
+ args.with_defaults(:key => 'prod')
271
+
272
+ if Dir.glob("#{@dvd_dir}/*.gz").empty? && !args.chroot
273
+ fail("Error: Could not find compiled source tarballs, please pass a chroot.")
274
+ end
275
+
276
+ Rake::Task['tar:build'].invoke(args.chroot) if args.chroot
277
+
278
+ Dir.chdir(@base_dir) do
279
+ File.basename(Dir.glob("#{@dvd_dir}/*.tar.gz").first,'.tar.gz') =~ /SIMP-DVD-[^-]+-(.+)/
280
+ name = "SIMP-#{$1}"
281
+ sh %{mkisofs -uid 0 -gid 0 -D -A #{name} -J -joliet-long -m ".git*" -m "./build/tmp" -m "./build/SRPMS" -m "./build/RPMS" -m "./build/build_keys" -o #{name}.src.iso .}
282
+ end
283
+ end
284
+ end
285
+ end
286
+ end
287
+ end
@@ -0,0 +1,731 @@
1
+ #!/usr/bin/rake -T
2
+
3
+ require 'simp/rake/build/constants'
4
+
5
+ module Simp; end
6
+ module Simp::Rake; end
7
+ module Simp::Rake::Build
8
+
9
+ class Pkg < ::Rake::TaskLib
10
+ include Simp::Rake::Build::Constants
11
+
12
+ def initialize( base_dir )
13
+ init_member_vars( base_dir )
14
+ @mock = ENV['mock'] || '/usr/bin/mock'
15
+ define_tasks
16
+ end
17
+
18
+ def define_tasks
19
+ task :help do
20
+ puts <<-EOF.gsub(/^ /, '')
21
+ SIMP_RAKE_CHOWN_EVERYTHING=(Y|N)
22
+ If 'Y', builds are preceded by a massive chown -R mock on the entire source tree
23
+
24
+ EOF
25
+ end
26
+
27
+ namespace :pkg do
28
+
29
+ ##############################################################################
30
+ # Main tasks
31
+ ##############################################################################
32
+
33
+ # Have to get things set up inside the proper namespace
34
+ task :prep do
35
+ @build_dirs = {
36
+ :modules => get_module_dirs,
37
+ :aux => [
38
+ "#{@build_dir}/GPGKEYS",
39
+ "#{@src_dir}/puppet/bootstrap",
40
+ "#{@src_dir}/rsync",
41
+ "#{@src_dir}/utils"
42
+ ],
43
+ :doc => "#{@src_dir}/doc",
44
+ :simp_cli => "#{@src_dir}/rubygems/simp-cli",
45
+ :simp => "#{@src_dir}",
46
+ }
47
+
48
+ @pkg_dirs = {
49
+ :simp => "#{@build_dir}/SIMP",
50
+ :ext => "#{@build_dir}/Ext_*"
51
+ }
52
+ end
53
+
54
+ task :mock_prep do
55
+ chown_everything = ENV.fetch( 'SIMP_RAKE_CHOWN_EVERYTHING', 'Y' ).chomp.index( %r{^(1|Y|true|yes)$}i ) || false
56
+
57
+ verbose(true) do
58
+ next if not chown_everything
59
+ # Set the permissions properly for mock to dig through your source
60
+ # directories.
61
+ chown_R(nil,'mock',@base_dir)
62
+ # Ruby >= 1.9.3 chmod_R('g+rXs',@base_dir)
63
+ Find.find(@base_dir) do |path|
64
+ if File.directory?(path)
65
+ %x{chmod g+rXs #{Shellwords.escape(path)}}
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ task :clean,[:chroot] => [:prep] do |t,args|
72
+ validate_in_mock_group?
73
+ @build_dirs.each_pair do |k,dirs|
74
+ Parallel.map(
75
+ Array(dirs),
76
+ :in_processes => get_cpu_limit,
77
+ :progress => t.name
78
+ ) do |dir|
79
+ Dir.chdir(dir) do
80
+ begin
81
+ sh %{rake clean[#{args.chroot}]}
82
+ rescue Exception => e
83
+ raise Parallel::Kill
84
+ end
85
+
86
+ end
87
+ end
88
+ end
89
+
90
+ # FIXME: not thread-safe
91
+ %x{mock -r #{args.chroot} --scrub=all} if args.chroot
92
+ end
93
+
94
+ task :clobber,[:chroot] => [:prep] do |t,args|
95
+ validate_in_mock_group?
96
+ @build_dirs.each_pair do |k,dirs|
97
+ Parallel.map(
98
+ Array(dirs),
99
+ :in_processes => get_cpu_limit,
100
+ :progress => t.name
101
+ ) do |dir|
102
+ Dir.chdir(dir) do
103
+ sh %{rake clobber[#{args.chroot}]}
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ desc <<-EOM
110
+ Prepare the GPG key space for a SIMP build.
111
+
112
+ If passed anything but 'dev', will fail if the directory is not present in
113
+ the 'build/build_keys' directory.
114
+ EOM
115
+ task :key_prep,[:key] do |t,args|
116
+ require 'securerandom'
117
+
118
+ args.with_defaults(:key => 'dev')
119
+
120
+ Dir.chdir("#{@build_dir}/build_keys") {
121
+ if (args.key != 'dev')
122
+ fail("Could not find GPG keydir '#{args.key}' in '#{Dir.pwd}'") unless File.directory?(args.key)
123
+ end
124
+
125
+ mkdir('dev') unless File.directory?('dev')
126
+ chmod(0700,'dev')
127
+
128
+ Dir.chdir('dev') {
129
+ dev_email = 'gatekeeper@simp.development.key'
130
+ current_key = `gpg --homedir=#{Dir.pwd} --list-keys #{dev_email} 2>/dev/null`
131
+ days_left = 0
132
+ if !current_key.empty?
133
+ lasts_until = current_key.lines.first.strip.split("\s").last.delete(']')
134
+ days_left = (Date.parse(lasts_until) - DateTime.now).to_i
135
+ end
136
+
137
+ if days_left > 0
138
+ puts "GPG key will expire in #{days_left} days."
139
+ else
140
+ puts "Generating new GPG key"
141
+
142
+ Dir.glob('*').each do |todel|
143
+ rm_rf(todel)
144
+ end
145
+
146
+ expire_date = (DateTime.now + 14)
147
+ now = Time.now.to_i.to_s
148
+ dev_email = 'gatekeeper@simp.development.key'
149
+ passphrase = SecureRandom.base64(500)
150
+
151
+ gpg_infile = <<-EOM
152
+ %echo Generating Development GPG Key
153
+ %echo
154
+ %echo This key will expire on #{expire_date}
155
+ %echo
156
+ Key-Type: RSA
157
+ Key-Length: 4096
158
+ Key-Usage: sign
159
+ Name-Real: SIMP Development
160
+ Name-Comment: Development key #{now}
161
+ Name-Email: #{dev_email}
162
+ Expire-Date: 2w
163
+ Passphrase: #{passphrase}
164
+ %pubring pubring.gpg
165
+ %secring secring.gpg
166
+ # The following creates the key, so we can print "Done!" afterwards
167
+ %commit
168
+ %echo New GPG Development Key Created
169
+ EOM
170
+
171
+ gpg_agent_script = <<-EOM
172
+ #!/bin/sh
173
+
174
+ gpg-agent --homedir=#{Dir.pwd} --batch --daemon --pinentry-program /usr/bin/pinentry-curses < /dev/null &
175
+ EOM
176
+
177
+ File.open('gengpgkey','w'){ |fh| fh.puts(gpg_infile) }
178
+ File.open('run_gpg_agent','w'){ |fh| fh.puts(gpg_agent_script) }
179
+ chmod(0755,'run_gpg_agent')
180
+
181
+ gpg_agent_pid = nil
182
+ gpg_agent_socket = nil
183
+
184
+ if File.exist?(%(#{ENV['HOME']}/.gnupg/S.gpg-agent))
185
+ gpg_agent_socket = %(#{ENV['HOME']}/.gnupg/S.gpg-agent)
186
+ gpg_agent_socket = %(#{ENV['HOME']}/.gnupg/S.gpg-agent)
187
+ end
188
+
189
+ begin
190
+ unless gpg_agent_socket
191
+ gpg_agent_output = %x(./run_gpg_agent).strip
192
+
193
+ if gpg_agent_output.empty?
194
+ # This is a working version of gpg-agent, that means we need to
195
+ # connect to it to figure out what's going on
196
+
197
+ gpg_agent_socket = %x(#{Dir.pwd}/S.gpg-agent)
198
+ gpg_agent_pid_info = %x(gpg-agent --homedir=#{Dir.pwd} /get serverpid).strip
199
+ gpg_agent_pid_info =~ %r(\[(\d+)\])
200
+ gpg_agent_pid = $1
201
+ else
202
+ # Are we running a broken version of the gpg-agent? If so, we'll
203
+ # get back info on the command line.
204
+
205
+ gpg_agent_info = gpg_agent_output.split(';').first.split('=').last.split(':')
206
+ gpg_agent_socket = gpg_agent_info[0]
207
+ gpg_agent_pid = gpg_agent_info[1].strip.to_i
208
+
209
+ if not File.exist? (%(#{Dir.pwd}/#{File.basename(gpg_agent_socket)})) then
210
+ ln_s(gpg_agent_socket,%(#{Dir.pwd}/#{File.basename(gpg_agent_socket)}))
211
+ end
212
+ end
213
+ end
214
+
215
+ sh %{gpg --homedir=#{Dir.pwd} --batch --gen-key gengpgkey}
216
+ sh %{gpg --homedir=#{Dir.pwd} --armor --export #{dev_email} > RPM-GPG-KEY-SIMP-Dev}
217
+ ensure
218
+ begin
219
+ rm('S.gpg-agent') if File.symlink?('S.gpg-agent')
220
+
221
+ if gpg_agent_pid
222
+ Process.kill(0,gpg_agent_pid)
223
+ Process.kill(15,gpg_agent_pid)
224
+ end
225
+ rescue Errno::ESRCH
226
+ # Not Running, Nothing to do!
227
+ end
228
+ end
229
+ end
230
+ }
231
+
232
+ Dir.chdir(args.key) {
233
+ rpm_build_keys = Dir.glob('RPM-GPG-KEY-*')
234
+ target_dir = '../../GPGKEYS'
235
+
236
+ fail("Could not find any RPM-GPG-KEY files in '#{Dir.pwd}'") if rpm_build_keys.empty?
237
+ fail("No GPGKEYS directory at '#{Dir.pwd}/#{target_dir}") unless File.directory?(target_dir)
238
+
239
+ dkfh = File.open("#{target_dir}/.dropped_keys",'w')
240
+
241
+ rpm_build_keys.each do |gpgkey|
242
+ cp(gpgkey,target_dir)
243
+ dkfh.puts(gpgkey)
244
+ end
245
+
246
+ dkfh.flush
247
+ dkfh.close
248
+ }
249
+ }
250
+ end
251
+
252
+ desc <<-EOM
253
+ Build the entire SIMP release.
254
+
255
+ Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
256
+ * :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
257
+ * :docs - Build the docs. Set this to false if you wish to skip building the docs.
258
+ * :key - The GPG key to sign the RPMs with. Defaults to 'dev'.
259
+ EOM
260
+ task :build,[:chroot,:docs,:key,:snapshot_release] => [:prep,:mock_prep,:key_prep] do |t,args|
261
+ validate_in_mock_group?
262
+
263
+ args.with_defaults(:key => 'dev')
264
+ args.with_defaults(:docs => true)
265
+
266
+ output_dir = @pkg_dirs[:simp]
267
+
268
+ check_dvd_env
269
+
270
+ Rake::Task['pkg:modules'].invoke(args.chroot)
271
+ Rake::Task['pkg:aux'].invoke(args.chroot)
272
+ if "#{args.docs}" == 'true'
273
+ Rake::Task['pkg:doc'].invoke(args.chroot)
274
+ end
275
+ Rake::Task['pkg:simp_cli'].invoke(args.chroot)
276
+
277
+ # The main SIMP RPM must be built last!
278
+ Rake::Task['pkg:simp'].invoke(args.chroot,args.snapshot_release)
279
+
280
+ # Prepare for the build!
281
+ rm_rf(output_dir)
282
+
283
+ # Copy all the resulting files into the target output directory
284
+ mkdir_p(output_dir)
285
+
286
+ @build_dirs.each_pair do |k,dirs|
287
+ Array(dirs).each do |dir|
288
+ rpms = Dir.glob("#{dir}/dist/*.rpm")
289
+ srpms = []
290
+ rpms.delete_if{|x|
291
+ del = false
292
+ if x =~ /\.src\.rpm$/
293
+ del = true
294
+ srpms << x
295
+ end
296
+
297
+ del
298
+ }
299
+
300
+ srpms.each do |srpm|
301
+ out_dir = "#{output_dir}/SRPMS"
302
+ mkdir_p(out_dir) unless File.directory?(out_dir)
303
+
304
+ if not uptodate?("#{out_dir}/#{File.basename(srpm)}",[srpm])
305
+ cp(srpm,out_dir)
306
+ end
307
+ end
308
+
309
+ rpms.each do |rpm|
310
+ out_dir = "#{output_dir}/RPMS/#{rpm.split('.')[-2]}"
311
+ mkdir_p(out_dir) unless File.directory?(out_dir)
312
+
313
+ if not uptodate?("#{out_dir}/#{File.basename(rpm)}",[rpm])
314
+ cp(rpm,out_dir)
315
+ end
316
+ end
317
+ end
318
+ end
319
+
320
+ Rake::Task['pkg:signrpms'].invoke(args.key)
321
+ end
322
+
323
+ desc <<-EOM
324
+ Build the Puppet module RPMs.
325
+
326
+ This also builds the simp-mit RPM due to its location.
327
+ Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
328
+ * :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
329
+ EOM
330
+ task :modules,[:chroot] => [:prep,:mock_prep] do |t,args|
331
+ build(args.chroot,@build_dirs[:modules],t)
332
+ end
333
+
334
+ desc <<-EOM
335
+ Build simp config rubygem RPM.
336
+ EOM
337
+ task :simp_cli,[:chroot] => [:prep,:mock_prep] do |t,args|
338
+ build(args.chroot,@build_dirs[:simp_cli],t)
339
+ end
340
+
341
+ desc <<-EOM
342
+ Build the SIMP non-module RPMs.
343
+
344
+ Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
345
+ * :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
346
+ EOM
347
+ task :aux,[:chroot] => [:prep,:mock_prep] do |t,args|
348
+ build(args.chroot,@build_dirs[:aux],t)
349
+ end
350
+
351
+ desc <<-EOM
352
+ Build the SIMP documentation.
353
+
354
+ Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
355
+ * :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
356
+ EOM
357
+ task :doc,[:chroot] => [:prep,:mock_prep] do |t,args|
358
+ build(args.chroot,@build_dirs[:doc],t)
359
+ end
360
+
361
+ desc <<-EOM
362
+ Build the main SIMP RPM.
363
+
364
+ Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
365
+ * :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
366
+ * :snapshot_release - Will add a define to the Mock to set snapshot_release to current date and time.
367
+ EOM
368
+ task :simp,[:chroot,:snapshot_release] => [:prep,:mock_prep] do |t,args|
369
+ build(args.chroot,@build_dirs[:simp],t,false,args.snapshot_release)
370
+ end
371
+
372
+ desc "Sign the RPMs."
373
+ task :signrpms,[:key,:rpm_dir,:force] => [:prep,:key_prep,:mock_prep] do |t,args|
374
+ args.with_defaults(:key => 'dev')
375
+ args.with_defaults(:rpm_dir => "#{@build_dir}/SIMP/*RPMS")
376
+ args.with_default(:force => false)
377
+
378
+ force = (args.force.to_s == 'false' ? false : true)
379
+
380
+ rpm_dirs = Dir.glob(args.rpm_dir)
381
+ to_sign = []
382
+
383
+ rpm_dirs.each do |rpm_dir|
384
+ Find.find(rpm_dir) do |rpm|
385
+ next unless File.readable?(rpm)
386
+ to_sign << rpm if rpm =~ /\.rpm$/
387
+ end
388
+ end
389
+
390
+ Parallel.map(
391
+ to_sign,
392
+ :in_processes => get_cpu_limit,
393
+ :progress => t.name
394
+ ) do |rpm|
395
+ rpminfo = %x{rpm -qip #{rpm} 2>/dev/null}.split("\n")
396
+ if (force || !rpminfo.grep(/Signature\s+:\s+\(none\)/).empty?)
397
+ Simp::RPM.signrpm(rpm,"#{@build_dir}/build_keys/#{args.key}")
398
+ end
399
+ end
400
+ end
401
+
402
+ desc <<-EOM
403
+ Check that RPMs are signed.
404
+
405
+ Checks all RPM files in a directory to see if they are trusted.
406
+ * :rpm_dir - A directory containing RPM files to check. Default #{@build_dir}/SIMP
407
+ * :key_dir - The path to the GPG keys you want to check the packages against. Default #{@build_dir}/GPGKEYS
408
+ EOM
409
+ task :checksig,[:rpm_dir,:key_dir] => [:prep] do |t,args|
410
+ begin
411
+ args.with_defaults(:rpm_dir => @pkg_dirs[:ext])
412
+ args.with_defaults(:key_dir => "#{@build_dir}/GPGKEYS")
413
+
414
+ rpm_dirs = Dir.glob(args.rpm_dir)
415
+
416
+ fail("Could not find files at #{args.rpm_dir}!") if rpm_dirs.empty?
417
+
418
+ temp_gpg_dir = Dir.mktmpdir
419
+
420
+ rpm_cmd = %{rpm --dbpath #{temp_gpg_dir}}
421
+
422
+ sh %{#{rpm_cmd} --initdb}
423
+
424
+ # Only import thngs that look like GPG keys...
425
+ Dir.glob("#{args.key_dir}/*").each do |key|
426
+ next if File.directory?(key) or not File.readable?(key)
427
+
428
+ File.read(key).each_line do |line|
429
+ if line =~ /-----BEGIN PGP PUBLIC KEY BLOCK-----/
430
+ sh %{#{rpm_cmd} --import #{key}}
431
+ break
432
+ end
433
+ end
434
+ end
435
+
436
+ bad_rpms = []
437
+ rpm_dirs.each do |rpm_dir|
438
+ Find.find(rpm_dir) do |path|
439
+ if (path =~ /.*\.rpm$/)
440
+ result = %x{#{rpm_cmd} --checksig #{path}}.strip
441
+ if result !~ /:.*\(\S+\).* OK$/
442
+ bad_rpms << path.split(/\s/).first
443
+ end
444
+ end
445
+ end
446
+ end
447
+
448
+ if !bad_rpms.empty?
449
+ bad_rpms.map!{|x| x = " * #{x}"}
450
+ bad_rpms.unshift("ERROR: Untrusted RPMs found in the repository:")
451
+
452
+ fail(bad_rpms.join("\n"))
453
+ else
454
+ puts "Checksig succeeded"
455
+ end
456
+ ensure
457
+ remove_entry_secure temp_gpg_dir
458
+ end
459
+ end
460
+
461
+ desc <<-EOM
462
+ Run repoclosure on RPM files.
463
+
464
+ Finds all rpm files in the target dir and all of its subdirectories, then
465
+ reports which packages have unresolved dependencies. This needs to be run
466
+ after rake tasks tar:build and unpack if operating on the base SIMP repo.
467
+ * :target_dir - The directory to assess. Default #{@build_dir}/SIMP.
468
+ * :aux_dir - Auxillary repo glob to use when assessing. Default #{@build_dir}/Ext_*.
469
+ Defaults to ''(empty) if :target_dir is not the system default.
470
+
471
+ EOM
472
+ task :repoclosure,[:target_dir,:aux_dir] => [:prep] do |t,args|
473
+ default_target = @pkg_dirs[:simp]
474
+ args.with_defaults(:target_dir => default_target)
475
+ if args.target_dir == default_target
476
+ args.with_defaults(:aux_dir => @pkg_dirs[:ext])
477
+ else
478
+ args.with_defaults(:aux_dir => '')
479
+ end
480
+
481
+ yum_conf_template = <<-EOF
482
+ [main]
483
+ keepcache=0
484
+ exactarch=1
485
+ obsoletes=1
486
+ gpgcheck=0
487
+ plugins=1
488
+ installonly_limit=5
489
+
490
+ <% repo_files.each do |repo| -%>
491
+ include=file://<%= repo %>
492
+ <% end -%>
493
+ EOF
494
+
495
+ yum_repo_template = <<-EOF
496
+ [<%= repo_name %>]
497
+ name=<%= repo_name %>
498
+ baseurl=file://<%= repo_path %>
499
+ enabled=1
500
+ gpgcheck=0
501
+ protect=1
502
+ EOF
503
+
504
+ fail "#{args.target_dir} does not exist!" if not File.directory?(args.target_dir)
505
+
506
+ begin
507
+ temp_pkg_dir = Dir.mktmpdir
508
+
509
+ mkdir_p("#{temp_pkg_dir}/repos/base")
510
+ mkdir_p("#{temp_pkg_dir}/repos/lookaside")
511
+ mkdir_p("#{temp_pkg_dir}/repodata")
512
+
513
+ Dir.glob(args.target_dir).each do |base_dir|
514
+ Find.find(base_dir) do |path|
515
+ if (path =~ /.*\.rpm$/) and (path !~ /.*.src\.rpm$/)
516
+ sym_path = "#{temp_pkg_dir}/repos/base/#{File.basename(path)}"
517
+ ln_s(path,sym_path) unless File.exists?(sym_path)
518
+ end
519
+ end
520
+ end
521
+
522
+ Dir.glob(args.aux_dir).each do |aux_dir|
523
+ Find.find(aux_dir) do |path|
524
+ if (path =~ /.*\.rpm$/) and (path !~ /.*.src\.rpm$/)
525
+ sym_path = "#{temp_pkg_dir}/repos/lookaside/#{File.basename(path)}"
526
+ ln_s(path,sym_path) unless File.exists?(sym_path)
527
+ end
528
+ end
529
+ end
530
+
531
+ Dir.chdir(temp_pkg_dir) do
532
+ repo_files = []
533
+ Dir.glob('repos/*').each do |repo|
534
+ if File.directory?(repo)
535
+ Dir.chdir(repo) { sh %{createrepo .} }
536
+
537
+ repo_name = File.basename(repo)
538
+ repo_path = File.expand_path(repo)
539
+ conf_file = "#{temp_pkg_dir}/#{repo_name}.conf"
540
+
541
+ File.open(conf_file,'w') do |file|
542
+ file.write(ERB.new(yum_repo_template,nil,'-').result(binding))
543
+ end
544
+
545
+ repo_files << conf_file
546
+ end
547
+ end
548
+
549
+ File.open('yum.conf', 'w') do |file|
550
+ file.write(ERB.new(yum_conf_template,nil,'-').result(binding))
551
+ end
552
+
553
+ cmd = 'repoclosure -c repodata -n -t -r base -l lookaside -c yum.conf'
554
+
555
+ if ENV['SIMP_BUILD_verbose'] == 'yes'
556
+ puts
557
+ puts '-'*80
558
+ puts "#### RUNNING: `#{cmd}`"
559
+ puts " in path '#{Dir.pwd}'"
560
+ puts '-'*80
561
+ end
562
+ sh cmd
563
+ end
564
+ ensure
565
+ remove_entry_secure temp_pkg_dir
566
+ end
567
+ end
568
+
569
+ ##############################################################################
570
+ # Helper methods
571
+ ##############################################################################
572
+
573
+ # Takes a list of directories to hop into and perform builds within
574
+ # Needs to be passed the chroot path as well
575
+ #
576
+ # The task must be passed so that we can output the calling name in the
577
+ # status bar.
578
+ def build(chroot,dirs,task,add_to_autoreq=true,snapshot_release=false)
579
+ validate_in_mock_group?
580
+
581
+ mock_pre_check(chroot)
582
+
583
+ # Default package metadata for reference
584
+ default_metadata = YAML.load(File.read("#{@src_dir}/build/package_metadata_defaults.yaml"))
585
+
586
+ metadata = Parallel.map(
587
+ Array(dirs),
588
+ :in_processes => get_cpu_limit,
589
+ :progress => task.name
590
+ ) do |dir|
591
+ result = []
592
+
593
+ Dir.chdir(dir) do
594
+ if File.exist?('Rakefile')
595
+ # Read the package metadata file and proceed accordingly.
596
+ module_metadata = default_metadata
597
+ if File.exist?('build/package_metadata.yaml')
598
+ module_metadata.merge!(YAML.load(File.read('build/package_metadata.yaml')))
599
+ end
600
+
601
+ # @simp_version should be set in the main Rakefile
602
+ build_module = false
603
+ Array(module_metadata['valid_versions']).each do |version_regex|
604
+ build_module = Regexp.new("^#{version_regex}$").match(@simp_version)
605
+ break if build_module
606
+ end
607
+
608
+ if build_module
609
+ unique_build = (get_cpu_limit != 1)
610
+ sh %{rake pkg:rpm[#{chroot},unique_build,#{snapshot_release}]}
611
+
612
+ # Glob all generated rpms, and add their metadata to a result array.
613
+ pkginfo = Hash.new
614
+ Dir.glob('dist/*.rpm') do |rpm|
615
+ if not rpm =~ /.*.src.rpm/ then
616
+ # get_info from each generated rpm, not the spec file, so macros in the
617
+ # metadata have already been resolved in the mock chroot.
618
+ pkginfo = Simp::RPM.get_info(rpm)
619
+ result << [pkginfo,module_metadata]
620
+ end
621
+ end
622
+ else
623
+ puts "Warning: #{Simp::RPM.get_info(Dir.glob('build/*.spec').first)[:name]} is not \
624
+ valid against SIMP version #{@simp_version.gsub("%{?snapshot_release}","")} and will not be built."
625
+ end
626
+ else
627
+ puts "Warning: Could not find Rakefile in '#{dir}'"
628
+ end
629
+ end
630
+
631
+ result
632
+ end
633
+
634
+ metadata.each do |i|
635
+ # Each module could generate multiple rpms, each with its own metadata.
636
+ # Iterate over them to add all built rpms to autorequires.
637
+ i.each do |module_pkginfo,module_metadata|
638
+ next unless (module_pkginfo and module_metadata)
639
+
640
+ # Set up the autorequires
641
+ if add_to_autoreq and not module_metadata['optional']
642
+ # Register the package with the autorequires
643
+ mode = 'r+'
644
+ mode = 'w+' unless File.exist?("#{@src_dir}/build/autorequires")
645
+ autoreq_fh = File.open("#{@src_dir}/build/autorequires",mode)
646
+
647
+ begin
648
+ autorequires = []
649
+ autorequires += autoreq_fh.read.split("\n")
650
+ autoreq_fh.rewind
651
+ autoreq_fh.truncate(0)
652
+
653
+ # The SIMP Rakefile expects the autorequires to be in this format.
654
+ autorequires << "#{module_pkginfo[:name]} #{module_pkginfo[:version]} #{module_pkginfo[:release]}"
655
+ autoreq_fh.puts(autorequires.sort.uniq.join("\n"))
656
+ ensure
657
+ autoreq_fh.flush
658
+ autoreq_fh.close
659
+ end
660
+ end
661
+ end
662
+ end
663
+ end
664
+
665
+ #desc "Checks the environment for building the DVD tarball
666
+ def check_dvd_env
667
+ ["#{@dvd_src}/isolinux","#{@dvd_src}/ks"].each do |dir|
668
+ File.directory?(dir)or raise "Environment not suitable: Unable to find directory '#{dir}'"
669
+ end
670
+ end
671
+
672
+ # Return a list of all puppet module directories with Rakefiles
673
+ def get_module_dirs
674
+ moddir = "#{@src_dir}/puppet/modules"
675
+
676
+ toret = []
677
+
678
+ Dir.glob("#{moddir}/*").map { |x| x = File.basename(x).chomp }.sort.each do |mod|
679
+ if File.exist?("#{moddir}/#{mod}/Rakefile")
680
+ toret << "#{moddir}/#{mod}"
681
+ end
682
+ end
683
+
684
+ toret
685
+ end
686
+
687
+ # Get a list of all of the mock configs available on the system.
688
+ def get_mock_configs
689
+ Dir.glob('/etc/mock/*.cfg').sort.map{ |x| x = File.basename(x,'.cfg')}
690
+ end
691
+
692
+ # Run some pre-checks to make sure that mock will work properly.
693
+ # Pass init=false if you do not want the function to initialize.
694
+ #
695
+ # Returns 'true' if the space is already initialized.
696
+ # FIXME: unique_name doesn't work
697
+ # FIXME: unique_name is never called
698
+ # FIXME: which is fortunate, because PKGNAME is never defined
699
+ def mock_pre_check(chroot,unique_name=false,init=true)
700
+ raise(Exception,"Could not find mock on your system, exiting") unless File.executable?('/usr/bin/mock')
701
+
702
+ mock_configs = get_mock_configs
703
+
704
+ if not chroot
705
+ fail("Error: No mock chroot provided. Your choices are:\n#{mock_configs.join("\n ")}"
706
+ )
707
+ end
708
+ if not mock_configs.include?(chroot)
709
+ fail("Error: Invalid mock chroot provided. Your choices are:\n#{mock_configs.join("\n ")}"
710
+ )
711
+ end
712
+
713
+ # Allow for building all modules in parallel.
714
+ @mock = "#{@mock} --uniqueext=#{PKGNAME}" if unique_name
715
+
716
+ # A simple test to see if the chroot is initialized
717
+ %x{#{@mock} -q --root #{chroot} --chroot "test -d /tmp" --quiet &> /dev/null}
718
+ initialized = $?.success?
719
+
720
+ if init and not initialized
721
+ cmd = %{#{@mock} --root #{chroot} --init}
722
+ sh cmd
723
+ end
724
+
725
+ initialized
726
+ end
727
+ end
728
+ end
729
+ end
730
+ end
731
+