simp-rake-helpers 4.1.1 → 5.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/lib/simp/rake.rb +4 -19
- data/lib/simp/rake/build/auto.rb +241 -506
- data/lib/simp/rake/build/build.rb +22 -155
- data/lib/simp/rake/build/clean.rb +3 -11
- data/lib/simp/rake/build/constants.rb +16 -78
- data/lib/simp/rake/build/deps.rb +8 -1
- data/lib/simp/rake/build/iso.rb +8 -6
- data/lib/simp/rake/build/pkg.rb +372 -304
- data/lib/simp/rake/build/rpmdeps.rb +2 -2
- data/lib/simp/rake/build/spec.rb +2 -1
- data/lib/simp/rake/build/tar.rb +11 -13
- data/lib/simp/rake/build/unpack.rb +2 -1
- data/lib/simp/rake/build/upload.rb +2 -1
- data/lib/simp/rake/helpers.rb +23 -0
- data/lib/simp/rake/helpers/version.rb +1 -1
- data/lib/simp/rake/pkg.rb +177 -312
- data/lib/simp/rake/pupmod/helpers.rb +13 -10
- data/lib/simp/rpm.rb +348 -65
- data/lib/simp/yum.rb +209 -0
- data/spec/lib/simp/files/simp_build/src/{build → assets/simp/build}/simp.spec +0 -0
- data/spec/lib/simp/files/testpackage-1-0.el7.noarch.rpm +0 -0
- data/spec/lib/simp/files/testpackage-dist.spec +42 -0
- data/spec/lib/simp/files/testpackage-multi.spec +7 -5
- data/spec/lib/simp/rpm_spec.rb +289 -14
- data/spec/spec_helper.rb +3 -0
- metadata +6 -3
data/lib/simp/rake/build/pkg.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
#!/usr/bin/rake -T
|
2
2
|
|
3
|
+
require 'simp/yum'
|
3
4
|
require 'simp/rake/pkg'
|
4
5
|
require 'simp/rake/build/constants'
|
5
6
|
require 'simp/rake/build/rpmdeps'
|
@@ -15,19 +16,13 @@ module Simp::Rake::Build
|
|
15
16
|
def initialize( base_dir )
|
16
17
|
init_member_vars( base_dir )
|
17
18
|
|
18
|
-
@
|
19
|
+
@verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
|
20
|
+
@rpm_build_metadata = 'last_rpm_build_metadata.yaml'
|
21
|
+
|
19
22
|
define_tasks
|
20
23
|
end
|
21
24
|
|
22
25
|
def define_tasks
|
23
|
-
task :help do
|
24
|
-
puts <<-EOF.gsub(/^ /, '')
|
25
|
-
SIMP_RAKE_CHOWN_EVERYTHING=(Y|N)
|
26
|
-
If 'Y', builds are preceded by a massive chown -R mock on the entire source tree
|
27
|
-
|
28
|
-
EOF
|
29
|
-
end
|
30
|
-
|
31
26
|
namespace :pkg do
|
32
27
|
##############################################################################
|
33
28
|
# Main tasks
|
@@ -47,16 +42,10 @@ module Simp::Rake::Build
|
|
47
42
|
@build_dirs = {
|
48
43
|
:modules => get_module_dirs(args[:method]),
|
49
44
|
:aux => [
|
50
|
-
"#{@src_dir}/rsync",
|
51
45
|
# Anything in here gets built!
|
52
|
-
"#{@src_dir}/assets/*"
|
53
|
-
# Legacy Compat
|
54
|
-
"#{@src_dir}/utils",
|
55
|
-
"#{@src_dir}/puppet/bootstrap"
|
46
|
+
"#{@src_dir}/assets/*"
|
56
47
|
],
|
57
|
-
:doc => "#{@src_dir}/doc"
|
58
|
-
:simp_cli => "#{@src_dir}/rubygems/simp_cli",
|
59
|
-
:simp => "#{@src_dir}",
|
48
|
+
:doc => "#{@src_dir}/doc"
|
60
49
|
}
|
61
50
|
|
62
51
|
@build_dirs[:aux].map!{|dir| dir = Dir.glob(dir)}
|
@@ -68,68 +57,10 @@ module Simp::Rake::Build
|
|
68
57
|
}
|
69
58
|
end
|
70
59
|
|
71
|
-
task :mock_prep => [:prep] do
|
72
|
-
chown_everything = ENV.fetch( 'SIMP_RAKE_CHOWN_EVERYTHING', 'Y' ).chomp.index( %r{^(1|Y|true|yes)$}i ) || false
|
73
|
-
|
74
|
-
verbose(true) do
|
75
|
-
next unless chown_everything
|
76
|
-
|
77
|
-
dirs_to_mod = []
|
78
|
-
files_to_mod = []
|
79
|
-
|
80
|
-
Find.find(@base_dir) do |path|
|
81
|
-
Find.prune if File.symlink?(path)
|
82
|
-
Find.prune if (File.basename(path) == '.git')
|
83
|
-
Find.prune if (File.basename(path) =~ /^\..*cache/)
|
84
|
-
|
85
|
-
safe_path = Shellwords.escape(path)
|
86
|
-
|
87
|
-
mock_gid = Etc.getgrnam('mock').gid
|
88
|
-
|
89
|
-
file_stat = File.stat(path)
|
90
|
-
|
91
|
-
file_mode = file_stat.mode.to_s(2)[-12..-1].scan(/.{3}/)
|
92
|
-
file_group_perms = file_mode[-2]
|
93
|
-
file_group_read = (file_group_perms[0] == '1')
|
94
|
-
file_group_execute = (file_group_perms[2] == '1')
|
95
|
-
file_sgid = (file_mode[-4][1] == '1')
|
96
|
-
|
97
|
-
if File.directory?(path)
|
98
|
-
if (mock_gid != file_stat.gid) ||
|
99
|
-
!file_group_read ||!file_group_execute || !file_sgid
|
100
|
-
dirs_to_mod << safe_path
|
101
|
-
end
|
102
|
-
else
|
103
|
-
if (mock_gid != file_stat.gid) || !file_group_read
|
104
|
-
files_to_mod << safe_path
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
# Set the permissions properly for mock to dig through your source
|
110
|
-
Parallel.map(
|
111
|
-
dirs_to_mod,
|
112
|
-
:in_processes => get_cpu_limit,
|
113
|
-
:progress => 'Set "mock" dir perms'
|
114
|
-
) do |dir|
|
115
|
-
%x{chgrp 'mock' #{dir} && chmod g+rXs #{dir}}
|
116
|
-
end
|
117
|
-
|
118
|
-
Parallel.map(
|
119
|
-
files_to_mod,
|
120
|
-
:in_processes => get_cpu_limit,
|
121
|
-
:progress => 'Set "mock" file perms'
|
122
|
-
) do |file|
|
123
|
-
%x{chgrp 'mock' #{file} && chmod g+r #{file}}
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
60
|
clean_failures = []
|
129
61
|
clean_failures_lock = Mutex.new
|
130
62
|
|
131
|
-
task :clean
|
132
|
-
validate_in_mock_group?
|
63
|
+
task :clean => [:prep] do |t,args|
|
133
64
|
@build_dirs.each_pair do |k,dirs|
|
134
65
|
Parallel.map(
|
135
66
|
Array(dirs),
|
@@ -139,7 +70,7 @@ module Simp::Rake::Build
|
|
139
70
|
Dir.chdir(dir) do
|
140
71
|
begin
|
141
72
|
rake_flags = Rake.application.options.trace ? '--trace' : ''
|
142
|
-
%x{rake clean
|
73
|
+
%x{rake clean #{rake_flags}}
|
143
74
|
clean_failures_lock.synchronize do
|
144
75
|
clean_failures << dir unless $?.success?
|
145
76
|
end
|
@@ -156,12 +87,9 @@ module Simp::Rake::Build
|
|
156
87
|
unless clean_failures.empty?
|
157
88
|
fail(%(Error: The following directories had failures in #{task.name}:\n * #{clean_failures.join("\n * ")}))
|
158
89
|
end
|
159
|
-
|
160
|
-
%x{mock -r #{args[:chroot]} --scrub=all} if args[:chroot]
|
161
90
|
end
|
162
91
|
|
163
|
-
task :clobber
|
164
|
-
validate_in_mock_group?
|
92
|
+
task :clobber => [:prep] do |t,args|
|
165
93
|
@build_dirs.each_pair do |k,dirs|
|
166
94
|
Parallel.map(
|
167
95
|
Array(dirs),
|
@@ -170,12 +98,13 @@ module Simp::Rake::Build
|
|
170
98
|
) do |dir|
|
171
99
|
Dir.chdir(dir) do
|
172
100
|
rake_flags = Rake.application.options.trace ? '--trace' : ''
|
173
|
-
sh %{rake clobber
|
101
|
+
sh %{rake clobber #{rake_flags}}
|
174
102
|
end
|
175
103
|
end
|
176
104
|
end
|
177
105
|
end
|
178
106
|
|
107
|
+
=begin
|
179
108
|
desc <<-EOM
|
180
109
|
Prepare the GPG key space for a SIMP build.
|
181
110
|
|
@@ -185,6 +114,7 @@ module Simp::Rake::Build
|
|
185
114
|
ENV vars:
|
186
115
|
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
187
116
|
EOM
|
117
|
+
=end
|
188
118
|
task :key_prep,[:key] => [:prep] do |t,args|
|
189
119
|
require 'securerandom'
|
190
120
|
_verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
|
@@ -330,78 +260,102 @@ module Simp::Rake::Build
|
|
330
260
|
end
|
331
261
|
end
|
332
262
|
|
333
|
-
|
334
|
-
|
263
|
+
def populate_rpm_dir(rpm_dir)
|
264
|
+
srpm_dir = File.join(File.dirname(rpm_dir), 'SRPMS')
|
335
265
|
|
336
|
-
|
337
|
-
|
338
|
-
* :docs - Build the docs. Set this to false if you wish to skip building the docs.
|
339
|
-
* :key - The GPG key to sign the RPMs with. Defaults to 'dev'.
|
266
|
+
FileUtils.mkdir_p(rpm_dir)
|
267
|
+
FileUtils.mkdir_p(srpm_dir)
|
340
268
|
|
341
|
-
|
342
|
-
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
343
|
-
EOM
|
344
|
-
task :build,[:chroot,:docs,:key,:snapshot_release] => [:prep,:mock_prep,:key_prep] do |t,args|
|
269
|
+
rpm_metadata = %x(find -P #{@src_dir} -xdev -type f -name #{@rpm_build_metadata}).lines.map(&:strip).sort
|
345
270
|
|
346
|
-
|
347
|
-
_verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
|
271
|
+
fail("No #{@rpm_build_metadata} files found under #{@src_dir}") if rpm_metadata.empty?
|
348
272
|
|
349
|
-
|
350
|
-
|
273
|
+
rpm_metadata.each do |mf|
|
274
|
+
metadata = YAML.load_file(mf)
|
275
|
+
rpms = metadata['rpms']
|
276
|
+
srpms = metadata['srpms']
|
351
277
|
|
352
|
-
|
278
|
+
fail("No RPMs found at #{rpm_dir}") if (rpms.nil? || rpms.empty?)
|
353
279
|
|
354
|
-
|
280
|
+
have_signed_rpm = false
|
355
281
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
Rake::Task['pkg:modules'].invoke(args[:chroot])
|
362
|
-
|
363
|
-
# The main SIMP RPM must be built last!
|
364
|
-
Rake::Task['pkg:simp'].invoke(args[:chroot],args[:snapshot_release])
|
282
|
+
Dir.chdir(rpm_dir) do
|
283
|
+
rpms.each_key do |rpm|
|
284
|
+
if @verbose
|
285
|
+
puts "Copying #{rpm} to #{rpm_dir}"
|
286
|
+
end
|
365
287
|
|
366
|
-
|
367
|
-
|
288
|
+
arch = rpms[rpm]['metadata'][:arch]
|
289
|
+
FileUtils.mkdir_p(arch)
|
368
290
|
|
369
|
-
|
370
|
-
mkdir_p(output_dir)
|
291
|
+
FileUtils.cp(rpms[rpm]['path'], arch)
|
371
292
|
|
372
|
-
|
373
|
-
|
374
|
-
rpms = Dir.glob("#{dir}/dist/*.rpm")
|
375
|
-
srpms = []
|
376
|
-
rpms.delete_if{|x|
|
377
|
-
del = false
|
378
|
-
if x =~ /\.src\.rpm$/
|
379
|
-
del = true
|
380
|
-
srpms << x
|
293
|
+
if rpms[rpm][:signature]
|
294
|
+
have_signed_rpm = true
|
381
295
|
end
|
296
|
+
end
|
297
|
+
end
|
382
298
|
|
383
|
-
|
384
|
-
|
299
|
+
Dir.chdir(srpm_dir) do
|
300
|
+
srpms.each_key do |srpm|
|
301
|
+
if have_signed_rpm && !srpms[srpm][:signature]
|
302
|
+
if @verbose
|
303
|
+
puts "Found signed RPM - skipping copy of '#{srpm}'"
|
304
|
+
end
|
385
305
|
|
386
|
-
|
387
|
-
|
388
|
-
mkdir_p(out_dir, :verbose => _verbose) unless File.directory?(out_dir)
|
306
|
+
next
|
307
|
+
end
|
389
308
|
|
390
|
-
|
391
|
-
|
309
|
+
if @verbose
|
310
|
+
puts "Copying '#{srpm}' to '#{srpm_dir}'"
|
392
311
|
end
|
393
|
-
end
|
394
312
|
|
395
|
-
|
396
|
-
|
397
|
-
mkdir_p(out_dir, :verbose => _verbose) unless File.directory?(out_dir)
|
313
|
+
arch = srpms[srpm]['metadata'][:arch]
|
314
|
+
FileUtils.mkdir_p(arch)
|
398
315
|
|
399
|
-
|
400
|
-
cp(rpm, out_dir, :verbose => _verbose)
|
401
|
-
end
|
316
|
+
FileUtils.cp(srpms[srpm]['path'], arch)
|
402
317
|
end
|
403
318
|
end
|
404
319
|
end
|
320
|
+
end
|
321
|
+
|
322
|
+
|
323
|
+
=begin
|
324
|
+
desc <<-EOM
|
325
|
+
Build the entire SIMP release.
|
326
|
+
|
327
|
+
* :docs - Build the docs. Set this to false if you wish to skip building the docs.
|
328
|
+
* :key - The GPG key to sign the RPMs with. Defaults to 'dev'.
|
329
|
+
|
330
|
+
ENV vars:
|
331
|
+
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
332
|
+
- Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
|
333
|
+
build-specific YUM cache
|
334
|
+
EOM
|
335
|
+
=end
|
336
|
+
task :build,[:docs,:key] => [:prep,:key_prep] do |t,args|
|
337
|
+
|
338
|
+
_verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
|
339
|
+
|
340
|
+
args.with_defaults(:key => 'dev')
|
341
|
+
args.with_defaults(:docs => 'true')
|
342
|
+
|
343
|
+
check_dvd_env
|
344
|
+
|
345
|
+
begin
|
346
|
+
yum_helper = Simp::YUM.new(
|
347
|
+
Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
|
348
|
+
ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
|
349
|
+
rescue Simp::YUM::Error
|
350
|
+
end
|
351
|
+
|
352
|
+
Rake::Task['pkg:aux'].invoke
|
353
|
+
if "#{args.docs}" == 'true'
|
354
|
+
Rake::Task['pkg:doc'].invoke
|
355
|
+
end
|
356
|
+
Rake::Task['pkg:modules'].invoke
|
357
|
+
|
358
|
+
populate_rpm_dir(@rpm_dir)
|
405
359
|
|
406
360
|
Rake::Task['pkg:signrpms'].invoke(args[:key])
|
407
361
|
end
|
@@ -409,35 +363,32 @@ module Simp::Rake::Build
|
|
409
363
|
desc <<-EOM
|
410
364
|
Build the Puppet module RPMs.
|
411
365
|
|
412
|
-
|
413
|
-
|
414
|
-
* :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
|
415
|
-
* :method - The Puppetfile from which the repository information should be read. Defaults to 'tracking'
|
366
|
+
* :method - The Puppetfile from which the repository information
|
367
|
+
should be read. Defaults to 'tracking'
|
416
368
|
|
417
369
|
ENV vars:
|
418
370
|
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
419
371
|
EOM
|
420
|
-
task :modules,[:
|
421
|
-
build(
|
372
|
+
task :modules,[:method] => [:prep] do |t,args|
|
373
|
+
build(@build_dirs[:modules],t)
|
422
374
|
end
|
423
375
|
|
424
376
|
desc <<-EOM
|
425
377
|
Build a single Puppet Module RPM.
|
426
378
|
|
427
|
-
Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
|
428
|
-
|
429
379
|
* :name - The path, or name, of the module to build. If a name is
|
430
380
|
given, the Puppetfile.<method> will be used to find the
|
431
381
|
module.
|
432
382
|
Note: This is the *short* name. So, to build
|
433
383
|
puppetlabs-stdlib, you would just enter 'stdlib'
|
434
|
-
* :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
|
435
384
|
* :method - The Puppetfile from which the repository information should be read. Defaults to 'tracking'
|
436
385
|
|
437
386
|
ENV vars:
|
438
387
|
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
388
|
+
- Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
|
389
|
+
build-specific YUM cache
|
439
390
|
EOM
|
440
|
-
task :single,[:name,:
|
391
|
+
task :single,[:name,:method] => [:prep] do |t,args|
|
441
392
|
fail("You must pass :name to '#{t.name}'") unless args[:name]
|
442
393
|
|
443
394
|
mod_path = File.absolute_path(args[:name])
|
@@ -455,70 +406,45 @@ module Simp::Rake::Build
|
|
455
406
|
mod_path = local_module[:path]
|
456
407
|
end
|
457
408
|
|
409
|
+
begin
|
410
|
+
yum_helper = Simp::YUM.new(
|
411
|
+
Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
|
412
|
+
ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
|
413
|
+
rescue Simp::YUM::Error
|
414
|
+
end
|
415
|
+
|
458
416
|
ENV['SIMP_PKG_rand_name'] = 'yes'
|
459
|
-
build(
|
417
|
+
build(Array(mod_path), t)
|
460
418
|
|
461
419
|
puts("Your packages can be found in '#{mod_path}/dist'")
|
462
420
|
end
|
463
421
|
|
464
|
-
desc <<-EOM
|
465
|
-
Build simp config rubygem RPM.
|
466
|
-
|
467
|
-
* :method - The Puppetfile from which the repository information should be read. Defaults to 'tracking'
|
468
|
-
|
469
|
-
ENV vars:
|
470
|
-
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
471
|
-
EOM
|
472
|
-
task :simp_cli,[:chroot] => [:prep,:mock_prep] do |t,args|
|
473
|
-
build(args[:chroot], @build_dirs[:simp_cli], t)
|
474
|
-
end
|
475
|
-
|
476
422
|
desc <<-EOM
|
477
423
|
Build the SIMP non-module RPMs.
|
478
424
|
|
479
|
-
Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
|
480
|
-
|
481
425
|
ENV vars:
|
482
426
|
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
483
|
-
* :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
|
484
427
|
EOM
|
485
|
-
task :aux
|
486
|
-
build(
|
428
|
+
task :aux => [:prep] do |t,args|
|
429
|
+
build(@build_dirs[:aux],t)
|
487
430
|
end
|
488
431
|
|
489
432
|
desc <<-EOM
|
490
433
|
Build the SIMP documentation.
|
491
434
|
|
492
|
-
Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
|
493
|
-
* :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
|
494
|
-
|
495
435
|
ENV vars:
|
496
436
|
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
497
437
|
EOM
|
498
|
-
task :doc
|
499
|
-
build(
|
500
|
-
end
|
501
|
-
|
502
|
-
desc <<-EOM
|
503
|
-
Build the main SIMP RPM.
|
504
|
-
|
505
|
-
Building this environment requires a working Mock setup (http://fedoraproject.org/wiki/Projects/Mock)
|
506
|
-
* :chroot - The Mock chroot configuration to use. See the '--root' option in mock(1).
|
507
|
-
* :snapshot_release - Will add a define to the Mock to set snapshot_release to current date and time.
|
508
|
-
|
509
|
-
ENV vars:
|
510
|
-
- Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
|
511
|
-
EOM
|
512
|
-
task :simp,[:chroot,:snapshot_release] => [:prep,:mock_prep] do |t,args|
|
513
|
-
build(args[:chroot],@build_dirs[:simp],t,false,args[:snapshot_release])
|
438
|
+
task :doc => [:prep] do |t,args|
|
439
|
+
build(@build_dirs[:doc],t)
|
514
440
|
end
|
515
441
|
|
516
442
|
desc "Sign the RPMs."
|
517
|
-
task :signrpms,[:key,:rpm_dir,:force] => [:prep,:key_prep
|
443
|
+
task :signrpms,[:key,:rpm_dir,:force] => [:prep,:key_prep] do |t,args|
|
518
444
|
which('rpmsign') || raise(StandardError, 'Could not find rpmsign on your system. Exiting.')
|
519
445
|
|
520
446
|
args.with_defaults(:key => 'dev')
|
521
|
-
args.with_defaults(:rpm_dir =>
|
447
|
+
args.with_defaults(:rpm_dir => File.join(File.dirname(@rpm_dir), '*RPMS'))
|
522
448
|
args.with_default(:force => 'false')
|
523
449
|
|
524
450
|
force = (args[:force].to_s == 'false' ? false : true)
|
@@ -538,36 +464,62 @@ module Simp::Rake::Build
|
|
538
464
|
:in_processes => get_cpu_limit,
|
539
465
|
:progress => t.name
|
540
466
|
) do |rpm|
|
541
|
-
|
542
|
-
|
543
|
-
Simp::RPM.signrpm(rpm,"#{@build_dir}/build_keys/#{args[:key]}")
|
544
|
-
end
|
467
|
+
rpm_info = Simp::RPM.new(rpm)
|
468
|
+
Simp::RPM.signrpm(rpm, "#{@build_dir}/build_keys/#{args[:key]}") unless rpm_info.signature
|
545
469
|
end
|
546
470
|
end
|
547
471
|
|
472
|
+
=begin
|
548
473
|
desc <<-EOM
|
549
474
|
Check that RPMs are signed.
|
550
475
|
|
551
476
|
Checks all RPM files in a directory to see if they are trusted.
|
552
477
|
* :rpm_dir - A directory containing RPM files to check. Default #{@build_dir}/SIMP
|
553
|
-
* :key_dir - The path to the GPG keys you want to check the packages against. Default #{@src_dir}/assets/
|
478
|
+
* :key_dir - The path to the GPG keys you want to check the packages against. Default #{@src_dir}/assets/gpgkeys/
|
554
479
|
EOM
|
480
|
+
=end
|
555
481
|
task :checksig,[:rpm_dir,:key_dir] => [:prep] do |t,args|
|
556
482
|
begin
|
483
|
+
|
484
|
+
default_key_dir = File.join(@src_dir, 'assets', 'gpgkeys', 'GPGKEYS')
|
557
485
|
args.with_defaults(:rpm_dir => @pkg_dirs[:simp])
|
558
|
-
args.with_defaults(:key_dir =>
|
486
|
+
args.with_defaults(:key_dir => default_key_dir)
|
559
487
|
|
560
488
|
rpm_dirs = Dir.glob(args[:rpm_dir])
|
561
489
|
|
562
490
|
fail("Could not find files at #{args[:rpm_dir]}!") if rpm_dirs.empty?
|
563
491
|
|
564
492
|
temp_gpg_dir = Dir.mktmpdir
|
493
|
+
at_exit { FileUtils.remove_entry(temp_gpg_dir) if File.exist?(temp_gpg_dir) }
|
565
494
|
|
566
495
|
rpm_cmd = %{rpm --dbpath #{temp_gpg_dir}}
|
567
496
|
|
568
497
|
%x{#{rpm_cmd} --initdb}
|
569
498
|
|
570
499
|
public_keys = Dir.glob(File.join(args[:key_dir], '*'))
|
500
|
+
|
501
|
+
if public_keys.empty?
|
502
|
+
key_dirs_tried = [ args[:key_dir] ]
|
503
|
+
|
504
|
+
# try path to GPG keys for SIMP 6.1+
|
505
|
+
if (args[:key_dir] != default_key_dir) and File.exist?(default_key_dir)
|
506
|
+
key_dirs_tried << default_key_dir
|
507
|
+
public_keys = Dir.glob(File.join(default_key_dir, '*'))
|
508
|
+
end
|
509
|
+
|
510
|
+
if public_keys.empty?
|
511
|
+
# try path to GPG keys for SIMP 6.0
|
512
|
+
old_key_dir = File.join(@src_dir, 'assets', 'simp-gpgkeys', 'GPGKEYS')
|
513
|
+
if File.exist?(old_key_dir)
|
514
|
+
key_dirs_tried << old_key_dir
|
515
|
+
public_keys = Dir.glob(File.join(old_key_dir, '*'))
|
516
|
+
end
|
517
|
+
end
|
518
|
+
|
519
|
+
if public_keys.empty?
|
520
|
+
$stderr.puts "pkg:checksig: Warning no GPG keys found in #{key_dirs_tried}"
|
521
|
+
end
|
522
|
+
end
|
571
523
|
public_keys += Dir.glob(File.join(@build_dir, 'build_keys', '*', 'RPM-GPG-KEY*'))
|
572
524
|
|
573
525
|
# Only import thngs that look like GPG keys...
|
@@ -728,94 +680,271 @@ protect=1
|
|
728
680
|
end
|
729
681
|
end
|
730
682
|
|
683
|
+
desc <<-EOM
|
684
|
+
Print published status of all project RPMs
|
685
|
+
EOM
|
686
|
+
task :check_published => [:prep] do |t,args|
|
687
|
+
begin
|
688
|
+
yum_helper = Simp::YUM.new(
|
689
|
+
Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
|
690
|
+
ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
|
691
|
+
rescue Simp::YUM::Error
|
692
|
+
end
|
693
|
+
|
694
|
+
Parallel.map(
|
695
|
+
# Allow for shell globs
|
696
|
+
Array(@build_dirs.values).flatten.sort,
|
697
|
+
:in_processes => 1
|
698
|
+
) do |dir|
|
699
|
+
fail("Could not find directory #{dir}") unless Dir.exist?(dir)
|
700
|
+
|
701
|
+
require_rebuild?(dir, yum_helper, { :verbose => true, :check_git => true, :prefix => '' })
|
702
|
+
end
|
703
|
+
end
|
704
|
+
|
731
705
|
##############################################################################
|
732
706
|
# Helper methods
|
733
707
|
##############################################################################
|
734
708
|
|
709
|
+
# Generate a random string suitable for a rake task namespace
|
710
|
+
#
|
711
|
+
# This is used as a workaround for Parallelization
|
712
|
+
def generate_namespace
|
713
|
+
return (0...24).map{ (65 + rand(26)).chr }.join.downcase
|
714
|
+
end
|
715
|
+
|
716
|
+
# Check and see if 'dir' requires a rebuild based on published packages
|
717
|
+
#
|
718
|
+
# If 'fetch' is true => Download the RPM
|
719
|
+
# If 'verbose' is true => Print helpful information to stderr
|
720
|
+
# If 'check_git' is true => Print the git tag status if 'verbose' is true
|
721
|
+
# 'prefix' is used to prepend verbose messages
|
722
|
+
#
|
723
|
+
# FIXME
|
724
|
+
# - Method is too long
|
725
|
+
# - Method needs to be passed in class variables (@xxx) so that it
|
726
|
+
# can be pulled out into a library that is easily unit-testable
|
727
|
+
def require_rebuild?(dir, yum_helper, opts={ :unique_namespace => generate_namespace, :fetch => false, :verbose => @verbose, :check_git => false, :prefix => '' })
|
728
|
+
result = false
|
729
|
+
|
730
|
+
|
731
|
+
rpm_dependency_file = File.join(@base_dir, 'build', 'rpm', 'dependencies.yaml')
|
732
|
+
|
733
|
+
if File.exist?(rpm_dependency_file)
|
734
|
+
rpm_metadata = YAML.load(File.read(rpm_dependency_file))
|
735
|
+
end
|
736
|
+
|
737
|
+
Dir.chdir(dir) do
|
738
|
+
if File.exist?('metadata.json')
|
739
|
+
# Generate RPM metadata files
|
740
|
+
# - 'build/rpm_metadata/requires' file containing RPM
|
741
|
+
# dependency/obsoletes information from the
|
742
|
+
# 'dependencies.yaml' and the module's
|
743
|
+
# 'metadata.json'; always created
|
744
|
+
# - 'build/rpm_metadata/release' file containing RPM
|
745
|
+
# release qualifier from the 'dependencies.yaml';
|
746
|
+
# only created if release qualifier if specified in
|
747
|
+
# the 'dependencies.yaml'
|
748
|
+
Simp::Rake::Build::RpmDeps::generate_rpm_meta_files(dir, rpm_metadata) if rpm_metadata
|
749
|
+
|
750
|
+
new_rpm = Simp::Rake::Pkg.new(Dir.pwd, opts[:unique_namespace], @simp_version)
|
751
|
+
new_rpm_info = Simp::RPM.new(new_rpm.spec_file)
|
752
|
+
else
|
753
|
+
spec_file = Dir.glob(File.join('build', '*.spec'))
|
754
|
+
fail("No spec file found in #{dir}/build") if spec_file.empty?
|
755
|
+
new_rpm_info = Simp::RPM.new(spec_file.first)
|
756
|
+
end
|
757
|
+
|
758
|
+
if opts[:check_git]
|
759
|
+
require_tag = false
|
760
|
+
|
761
|
+
#FIXME The check below is insufficient. See logic in compare_latest_tag,
|
762
|
+
# which does a git diff between files checked out and latest tag to see
|
763
|
+
# if any changes to mission-relevant files have been made and if the
|
764
|
+
# version has been bumped, when such changes have been made.
|
765
|
+
#
|
766
|
+
# We remove any leading 'v' chars since some external projects use them
|
767
|
+
latest_tag = %x(git describe --abbrev=0 --tags 2>/dev/null).strip.gsub(/^v/,'')
|
768
|
+
|
769
|
+
# Legacy munge
|
770
|
+
# We remove any leading 'simp-', leading 'simp6.0.0-', or trailing
|
771
|
+
# '-post1' strings used previously for some projects. This munge
|
772
|
+
# logic can be remove in SIMP 7.
|
773
|
+
latest_tag.gsub!(/^simp-|^simp6.0.0-|-post1$/,'')
|
774
|
+
|
775
|
+
begin
|
776
|
+
rpm_version = Gem::Version.new(new_rpm_info.version)
|
777
|
+
rescue ArgumentError
|
778
|
+
$stderr.puts ">>#{new_rpm_info.basename}: Could not determine RPM version"
|
779
|
+
end
|
780
|
+
|
781
|
+
begin
|
782
|
+
if latest_tag.empty?
|
783
|
+
require_tag = true
|
784
|
+
else
|
785
|
+
latest_tag_version = Gem::Version.new(latest_tag)
|
786
|
+
end
|
787
|
+
rescue ArgumentError
|
788
|
+
$stderr.puts ">>#{new_rpm_info.basename}: Invalid git tag version '#{latest_tag}' "
|
789
|
+
end
|
790
|
+
|
791
|
+
if rpm_version && latest_tag_version
|
792
|
+
if rpm_version > latest_tag_version
|
793
|
+
require_tag = true
|
794
|
+
end
|
795
|
+
end
|
796
|
+
|
797
|
+
if opts[:verbose] && require_tag
|
798
|
+
$stderr.puts "#{opts[:prefix]}Git Release Tag Required: #{new_rpm_info.basename} #{latest_tag} => #{new_rpm_info.version}"
|
799
|
+
end
|
800
|
+
end
|
801
|
+
|
802
|
+
# Pull down any newer versions of the target RPM if we've been
|
803
|
+
# given a yum_helper
|
804
|
+
#
|
805
|
+
# Just build from scratch if something goes wrong
|
806
|
+
if yum_helper
|
807
|
+
# Most RPM spec files generate a single package, but we have
|
808
|
+
# a handful that generate multiple (sub-)packages
|
809
|
+
new_rpm_info.packages.each do |package|
|
810
|
+
begin
|
811
|
+
published_rpm = yum_helper.available_package(package)
|
812
|
+
|
813
|
+
if published_rpm
|
814
|
+
if new_rpm_info.package_newer?(package, published_rpm)
|
815
|
+
if opts[:verbose]
|
816
|
+
$stderr.puts "#{opts[:prefix]}RPM Publish Required: #{published_rpm} => #{new_rpm_info.rpm_name(package)}"
|
817
|
+
|
818
|
+
end
|
819
|
+
result = true
|
820
|
+
else
|
821
|
+
$stderr.puts "#{opts[:prefix]}Found Existing Remote RPM: #{published_rpm}" if opts[:verbose]
|
822
|
+
if opts[:fetch]
|
823
|
+
# Download remote RPM, unless already downloaded it.
|
824
|
+
if File.exist?(File.join('dist', published_rpm))
|
825
|
+
$stderr.puts "#{opts[:prefix]}#{published_rpm} previously downloaded" if opts[:verbose]
|
826
|
+
else
|
827
|
+
# We know the package exists. So in a brute-force fashion,
|
828
|
+
# we're going to retry a couple of times.
|
829
|
+
# (Real fix is for user to update retry and timeout parameters
|
830
|
+
# in their yum config).
|
831
|
+
tries = ENV.fetch('SIMP_YUM_retries','3').to_i
|
832
|
+
begin
|
833
|
+
yum_helper.download("#{package}", :target_dir => 'dist')
|
834
|
+
$stderr.puts "#{opts[:prefix]}Downloaded #{published_rpm}" if opts[:verbose]
|
835
|
+
rescue Simp::YUM::Error
|
836
|
+
tries -= 1
|
837
|
+
if tries > 0
|
838
|
+
retry
|
839
|
+
else
|
840
|
+
$stderr.puts ">>Failed to download existing remote RPM: #{published_rpm}. RPM will be locally rebuilt"
|
841
|
+
result = true
|
842
|
+
end
|
843
|
+
end
|
844
|
+
end
|
845
|
+
end
|
846
|
+
end
|
847
|
+
else
|
848
|
+
if opts[:verbose]
|
849
|
+
$stderr.puts "#{opts[:prefix]}RPM Publish Required: #{new_rpm_info.rpm_name(package)}"
|
850
|
+
end
|
851
|
+
result = true
|
852
|
+
end
|
853
|
+
rescue Simp::YUM::Error => e
|
854
|
+
$stderr.puts e if opts[:verbose]
|
855
|
+
end
|
856
|
+
end
|
857
|
+
elsif opts[:verbose]
|
858
|
+
$stderr.puts '>>Issue creating YUM configuration. Packages will be locally generated.'
|
859
|
+
|
860
|
+
result = true
|
861
|
+
end
|
862
|
+
end
|
863
|
+
|
864
|
+
return result
|
865
|
+
end
|
866
|
+
|
735
867
|
# Takes a list of directories to hop into and perform builds within
|
736
|
-
# Needs to be passed the chroot path as well
|
737
868
|
#
|
738
869
|
# The task must be passed so that we can output the calling name in the
|
739
870
|
# status bar.
|
740
|
-
def build(
|
741
|
-
validate_in_mock_group?
|
871
|
+
def build(dirs, task, rebuild_for_arch=false, remake_yum_cache = false)
|
742
872
|
_verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
|
873
|
+
dbg_prefix = ' ' # prefix for debug messages
|
743
874
|
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
875
|
+
begin
|
876
|
+
yum_helper = Simp::YUM.new(
|
877
|
+
Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data'))
|
878
|
+
)
|
879
|
+
rescue Simp::YUM::Error
|
749
880
|
end
|
750
881
|
|
751
|
-
|
882
|
+
Parallel.map(
|
752
883
|
# Allow for shell globs
|
753
884
|
Array(dirs),
|
754
885
|
:in_processes => get_cpu_limit,
|
755
886
|
:progress => task.name
|
756
887
|
) do |dir|
|
757
|
-
result = []
|
758
|
-
|
759
888
|
fail("Could not find directory #{dir}") unless Dir.exist?(dir)
|
760
889
|
|
761
890
|
Dir.chdir(dir) do
|
762
891
|
built_rpm = false
|
763
|
-
unique_build = (get_cpu_limit != 1).to_s
|
764
892
|
|
765
893
|
if _verbose
|
766
|
-
$stderr.puts("
|
894
|
+
$stderr.puts("\nPackaging #{File.basename(dir)}")
|
767
895
|
end
|
768
896
|
|
769
897
|
# We're building a module, override anything down there
|
770
898
|
if File.exist?('metadata.json')
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
# only created if release qualifier if specified in
|
780
|
-
# the 'dependencies.yaml'
|
781
|
-
Simp::Rake::Build::RpmDeps::generate_rpm_meta_files(dir, rpm_metadata)
|
782
|
-
|
783
|
-
unique_namespace = (0...24).map{ (65 + rand(26)).chr }.join.downcase
|
784
|
-
|
785
|
-
Simp::Rake::Pkg.new(Dir.pwd, nil, unique_namespace, @simp_version)
|
786
|
-
|
787
|
-
Rake::Task["#{unique_namespace}:pkg:rpm"].invoke(chroot, unique_build, snapshot_release)
|
899
|
+
unique_namespace = generate_namespace
|
900
|
+
if require_rebuild?(dir, yum_helper, { :unique_namespace => unique_namespace, :fetch => true, :verbose => _verbose, :prefix => dbg_prefix})
|
901
|
+
$stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' on #{File.basename(dir)}") if _verbose
|
902
|
+
Rake::Task["#{unique_namespace}:pkg:rpm"].invoke
|
903
|
+
else
|
904
|
+
# Record metadata for the downloaded RPM
|
905
|
+
Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
|
906
|
+
end
|
788
907
|
|
789
908
|
built_rpm = true
|
790
909
|
|
791
910
|
# We're building one of the extra assets and should honor its Rakefile
|
911
|
+
# and RPM spec file.
|
792
912
|
elsif File.exist?('Rakefile')
|
913
|
+
if require_rebuild?(dir, yum_helper, { :fetch => true, :verbose => _verbose, :prefix => dbg_prefix })
|
914
|
+
$stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' in #{File.basename(dir)}") if _verbose
|
915
|
+
rake_flags = Rake.application.options.trace ? '--trace' : ''
|
916
|
+
cmd = %{SIMP_BUILD_version=#{@simp_version} rake pkg:rpm #{rake_flags} 2>&1}
|
793
917
|
|
794
|
-
|
795
|
-
|
796
|
-
|
918
|
+
build_success = true
|
919
|
+
begin
|
920
|
+
::Bundler.with_clean_env do
|
921
|
+
%x{#{cmd}}
|
922
|
+
build_success = $?.success?
|
923
|
+
end
|
797
924
|
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
%x{#{cmd}}
|
802
|
-
build_success = $?.success?
|
925
|
+
built_rpm = true
|
926
|
+
rescue
|
927
|
+
build_success = false
|
803
928
|
end
|
804
929
|
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
930
|
+
unless build_success
|
931
|
+
if _verbose
|
932
|
+
$stderr.puts("First 'rake pkg:rpm' attempt for #{dir} failed, running bundle and trying again.")
|
933
|
+
end
|
809
934
|
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
end
|
935
|
+
::Bundler.with_clean_env do
|
936
|
+
%x{bundle install --with development}
|
937
|
+
output = %x{#{cmd} 2>&1}
|
814
938
|
|
815
|
-
|
816
|
-
|
817
|
-
|
939
|
+
unless $?.success?
|
940
|
+
raise("Error in #{dir} running #{cmd}\n#{output}")
|
941
|
+
end
|
942
|
+
end
|
818
943
|
end
|
944
|
+
else
|
945
|
+
# Record metadata for the downloaded RPM
|
946
|
+
Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
|
947
|
+
built_rpm = true
|
819
948
|
end
|
820
949
|
else
|
821
950
|
puts "Warning: '#{dir}' could not be built via Rake"
|
@@ -823,8 +952,7 @@ protect=1
|
|
823
952
|
|
824
953
|
if built_rpm
|
825
954
|
tarballs = Dir.glob('dist/*.tar.gz')
|
826
|
-
|
827
|
-
rpms = (Dir.glob('dist/*.rpm') - srpms)
|
955
|
+
rpms = Dir.glob('dist/*.rpm').delete_if{|x| x =~ %r(\.src\.rpm$)}
|
828
956
|
|
829
957
|
# Not all items generate tarballs
|
830
958
|
tarballs.each do |pkg|
|
@@ -832,74 +960,14 @@ protect=1
|
|
832
960
|
raise("Empty Tarball '#{pkg}' generated for #{dir}")
|
833
961
|
end
|
834
962
|
end
|
835
|
-
raise("No SRPMs generated for #{dir}") if srpms.empty?
|
836
|
-
raise("No RPMs generated for #{dir}") if rpms.empty?
|
837
|
-
|
838
|
-
last_build = {
|
839
|
-
'git_hash' => %x{git show-ref --head HEAD}.chomp,
|
840
|
-
'rpms' => {}
|
841
|
-
}
|
842
|
-
|
843
|
-
# Glob all generated rpms, and add their metadata to a result array.
|
844
|
-
rpms.each do |rpm|
|
845
|
-
# get_info from each generated rpm, not the spec file, so macros in the
|
846
|
-
# metadata have already been resolved in the mock chroot.
|
847
|
-
metadata = Simp::RPM.get_info(rpm)
|
848
|
-
|
849
|
-
if File.exist?('build/package_metadata.yaml')
|
850
|
-
metadata.merge!(YAML.load_file('build/package_metadata.yaml'))
|
851
|
-
end
|
852
|
-
|
853
|
-
rpm_stat = File.stat(rpm)
|
854
|
-
|
855
|
-
last_build['rpms'][rpm] = {
|
856
|
-
'metadata' => metadata,
|
857
|
-
'size' => rpm_stat.size
|
858
|
-
}
|
859
963
|
|
860
|
-
|
861
|
-
end
|
862
|
-
|
863
|
-
File.open('dist/.last_build_metadata', 'w') do |fh|
|
864
|
-
fh.puts(last_build.to_yaml)
|
865
|
-
end
|
964
|
+
raise("No RPMs generated for #{dir}") if rpms.empty?
|
866
965
|
end
|
867
966
|
|
868
967
|
if _verbose
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
result
|
874
|
-
end
|
875
|
-
|
876
|
-
metadata.each do |mod|
|
877
|
-
# Each module could generate multiple rpms, each with its own metadata.
|
878
|
-
# Iterate over them to add all built rpms to autorequires.
|
879
|
-
mod.each do |module_pkginfo|
|
880
|
-
next unless (module_pkginfo && module_pkginfo.is_a?(Hash))
|
881
|
-
|
882
|
-
# Set up the autorequires
|
883
|
-
if add_to_autoreq
|
884
|
-
# Register the package with the autorequires
|
885
|
-
mode = 'r+'
|
886
|
-
mode = 'w+' unless File.exist?("#{@src_dir}/build/autorequires")
|
887
|
-
autoreq_fh = File.open("#{@src_dir}/build/autorequires",mode)
|
888
|
-
|
889
|
-
begin
|
890
|
-
# Reads the autorequires file, then empties it
|
891
|
-
autorequires = []
|
892
|
-
autorequires += autoreq_fh.read.split("\n")
|
893
|
-
autoreq_fh.rewind
|
894
|
-
autoreq_fh.truncate(0)
|
895
|
-
|
896
|
-
# The SIMP Rakefile expects the autorequires to be in this format.
|
897
|
-
autorequires << "#{module_pkginfo[:name]} #{module_pkginfo[:version]} #{module_pkginfo[:release]}"
|
898
|
-
autoreq_fh.puts(autorequires.sort.uniq.join("\n"))
|
899
|
-
ensure
|
900
|
-
autoreq_fh.flush
|
901
|
-
autoreq_fh.close
|
902
|
-
end
|
968
|
+
rpms = Dir.glob('dist/*.rpm')
|
969
|
+
# $stderr.puts("#{dbg_prefix}RPMS: #{rpms.join("\n#{dbg_prefix} ")}")
|
970
|
+
$stderr.puts("Finished #{File.basename(dir)}")
|
903
971
|
end
|
904
972
|
end
|
905
973
|
end
|