puppet 2.7.20 → 2.7.21

data/ext/packaging/tasks/mock.rake

@@ -11,7 +11,7 @@
 # pkg/<dist>/<version>/{products,devel,dependencies}/<arch>/*.rpm
 # e.g.,
 # pkg/el/5/products/i386/*.rpm
-# pkg/fedora/f15/products/i386/*.rpm
+# pkg/fedora/f16/products/i386/*.rpm
 #
 # For PE, the directory structure is flatter:
 # pkg/<dist>-<version>-<arch>/*.rpm
@@ -19,8 +19,29 @@
 # pkg/el-5-i386/*.rpm
 
 def mock(mock_config, srpm)
-  check_tool('mock')
-  sh "mock -r #{mock_config} #{srpm}"
+  unless mock = find_tool('mock')
+    warn "mock is required for building rpms with mock. Please install mock and try again."
+    exit 1
+  end
+  if @build.random_mockroot
+    basedir = get_temp
+    chown("#{ENV['USER']}", "mock", basedir)
+    # Mock requires the sticky bit be set on the basedir
+    chmod(02775, basedir)
+    mockfile = File.join('/', 'etc', 'mock', "#{mock_config}.cfg")
+    puts "Setting mock basedir to #{basedir}"
+    config = mock_with_basedir(mockfile, basedir)
+    configdir = setup_mock_config_dir(config)
+    # Clean up the new mock config
+    rm_r  File.dirname(config)
+    configdir_arg = " --configdir #{configdir}"
+    mock << configdir_arg
+  end
+  sh "#{mock} -r #{mock_config} #{srpm}"
+  # Clean up the configdir
+  rm_r configdir
+
+  basedir
 end
 
 def srpm_file
@@ -30,116 +51,176 @@ def srpm_file
 end
 
 def mock_el_family(mock_config)
-  if @build_pe
+  if @build.build_pe
     family = mock_config.split('-')[2][/[a-z]+/]
   else
-    family = mock_config.split('-')[0]
-    family = 'el' if family == 'pl'
+    first, second = mock_config.split('-')
+    if (first == 'el' || first == 'fedora')
+      family = first
+    elsif first == 'pl'
+      if second.match(/^\d+$/)
+        family = 'el'
+      else
+        family = second
+      end
+    end
   end
   family
 end
 
 def mock_el_ver(mock_config)
-  if @build_pe
+  if @build.build_pe
     version = mock_config.split('-')[2][/[0-9]+/]
   else
-    version = mock_config.split('-')[1]
-    version = "f#{version}" if mock_config.split('-')[0] == 'fedora'
+    first, second, third = mock_config.split('-')
+    if (first == 'el' || first == 'fedora') || (first == 'pl' && second.match(/^\d+$/))
+      version = second
+    else
+      version = third
+    end
+  end
+  if [first,second].include?('fedora')
+    version = "f#{version}"
   end
   version
 end
 
 def mock_arch(mock_config)
-  if @build_pe
+  if @build.build_pe
     arch = mock_config.split('-')[3]
   else
     arch = mock_config.split('-')[2]
   end
 end
 
-def build_rpm_with_mock(mocks, is_rc, subdir)
+def build_rpm_with_mock(mocks, is_rc)
   mocks.split(' ').each do |mock_config|
     family  = mock_el_family(mock_config)
     version = mock_el_ver(mock_config)
     arch    = mock_arch(mock_config)
-    result  = "/var/lib/mock/#{mock_config}/result/*.rpm"
+    subdir  = is_rc ? 'devel' : 'products'
+    bench = Benchmark.realtime do
+      resultdir = mock(mock_config, srpm_file)
+      result  = "#{resultdir}/#{mock_config}/result/*.rpm"
 
-    mock(mock_config, srpm_file)
-
-    Dir[result].each do |rpm|
-      rpm.strip!
-      unless ENV['RC_OVERRIDE'] == '1'
-        if is_rc == FALSE and rpm =~ /[0-9]+rc[0-9]+\./
-          puts "It looks like you might be trying to ship an RC to the production repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
-          next
-        elsif is_rc and rpm !~ /[0-9]+rc[0-9]+\./
-          puts "It looks like you might be trying to ship a production release to the development repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
-          next
+      Dir[result].each do |rpm|
+        rpm.strip!
+        unless ENV['RC_OVERRIDE'] == '1'
+          if is_rc == FALSE and rpm =~ /[0-9]+rc[0-9]+\./
+            puts "It looks like you might be trying to ship an RC to the production repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
+            next
+          elsif is_rc and rpm !~ /[0-9]+rc[0-9]+\./
+            puts "It looks like you might be trying to ship a production release to the development repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
+            next
+          end
         end
-      end
 
-      if @build_pe
-        case File.basename(rpm)
-          when /debuginfo/
-            rm_rf(rpm)
-          when /src\.rpm/
-            cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-srpms")
-          when /i.?86/
-            cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
-          when /x66_64/
-            cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-x86_64")
-          when /noarch/
-            cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
-            ln("pkg/pe/rpm/#{family}-#{version}-i386/#{File.basename(rpm)}", "pkg/pe/rpm/#{family}-#{version}-x86_64/")
+        if @build.build_pe
+          %x{mkdir -p pkg/pe/rpm/#{family}-#{version}-{srpms,i386,x86_64}}
+          case File.basename(rpm)
+            when /debuginfo/
+              rm_rf(rpm)
+            when /src\.rpm/
+              cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-srpms")
+            when /i.?86/
+              cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
+            when /x66_64/
+              cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-x86_64")
+            when /noarch/
+              cp_pr(rpm, "pkg/pe/rpm/#{family}-#{version}-i386")
+              ln("pkg/pe/rpm/#{family}-#{version}-i386/#{File.basename(rpm)}", "pkg/pe/rpm/#{family}-#{version}-x86_64/")
+          end
+        else
+          %x{mkdir -p pkg/#{family}/#{version}/#{subdir}/{SRPMS,i386,x86_64}}
+          case File.basename(rpm)
+            when /debuginfo/
+              rm_rf(rpm)
+            when /src\.rpm/
+              cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/SRPMS")
+            when /i.?86/
+              cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/i386")
+            when /x86_64/
+              cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/x86_64")
+            when /noarch/
+              cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/i386")
+              ln("pkg/#{family}/#{version}/#{subdir}/i386/#{File.basename(rpm)}", "pkg/#{family}/#{version}/#{subdir}/x86_64/")
+          end
         end
-      else
-        case File.basename(rpm)
-          when /debuginfo/
-            rm_rf(rpm)
-          when /src\.rpm/
-            cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/SRPMS")
-          when /i.?86/
-            cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/i386")
-          when /x86_64/
-            cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/x86_64")
-          when /noarch/
-            cp_pr(rpm, "pkg/#{family}/#{version}/#{subdir}/i386")
-            ln("pkg/#{family}/#{version}/#{subdir}/i386/#{File.basename(rpm)}", "pkg/#{family}/#{version}/#{subdir}/x86_64/")
+      end
+      # To avoid filling up the system with our random mockroots, we should
+      # clean up. However, this requires sudo. If we don't have sudo, we'll
+      # just fail and not clean up, but warn the user about it.
+      if @build.random_mockroot
+        %x{sudo -n echo 'Cleaning build root.'}
+        if $?.success?
+          sh "sudo -n rm -r #{resultdir}" unless resultdir.nil?
+        else
+          warn "Couldn't clean #{resultdir} without sudo. Leaving."
         end
       end
     end
+    add_metrics({ :dist => "#{family}-#{version}", :bench => bench }) if @build.benchmark
   end
 end
 
+# With the advent of using Jenkins to parallelize builds, it becomes critical
+# that we be able to use the same mock at the same time for > 1 builds without
+# clobbering the mock root every time. Here we add a method that takes the full
+# path to a mock configuration and a path, and adds a base directory
+# configuration directive to the mock to use the path as the directory for the
+# mock build root. The new mock config is written to a temporary space, and its
+# location is returned.  This allows us to create mock configs with randomized
+# temporary mock roots.
+#
+def mock_with_basedir(mock, basedir)
+  config = IO.readlines(mock)
+  basedir = "config_opts['basedir'] = '#{basedir}'"
+  config.unshift(basedir)
+  tempdir = get_temp
+  newmock = File.join(tempdir, File.basename(mock))
+  File.open(newmock, 'w') { |f| f.puts config }
+  newmock
+end
+
+# Mock accepts an alternate configuration directory to /etc/mock for mock
+# configs, but the directory has to include both site-defaults.cfg and
+# logging.ini. This is a simple utility method to set a mock configuration dir
+# by copying a mock and the required defaults to a temporary directory and
+# returning that directory. This method takes the full path to a mock
+# configuration file and returns the path to the new configuration dir.
+#
+def setup_mock_config_dir(mock)
+  tempdir = get_temp
+  cp File.join('/', 'etc', 'mock', 'site-defaults.cfg'), tempdir
+  cp File.join('/', 'etc', 'mock', 'logging.ini'), tempdir
+  cp mock, tempdir
+  tempdir
+end
 
 namespace :pl do
-  task :setup_el_dirs do
-    if @build_pe
-      %x{mkdir -p pkg/pe/rpm/sles-11-{i586,x86_64,srpms}}
-      %x{mkdir -p pkg/pe/rpm/el-{5,6}-{i386,x86_64,srpms}}
-    else
-      %x{mkdir -p pkg/el/{5,6}/{products,devel,dependencies}/{SRPMS,i386,x86_64}}
-      %x{mkdir -p pkg/fedora/{f15,f16,f17}/{products,devel,dependencies}/{SRPMS,i386,x86_64}}
-    end
+  desc "Use default mock to make a final rpm, keyed to PL infrastructure, pass MOCK to specify config"
+  task :mock => "package:srpm" do
+    # If default mock isn't specified, just take the first one in the @build.final_mocks list
+    @build.default_mock ||= @build.final_mocks.split(' ')[0]
+    build_rpm_with_mock(@build.default_mock, is_rc?)
+    post_metrics if @build.benchmark
   end
 
-  desc "Use default mock to make a final rpm, keyed to PL infrastructure, pass MOCK to specify config"
-  task :mock => [ "package:srpm", "pl:setup_el_dirs" ] do
-    # If default mock isn't specified, just take the first one in the @final_mocks list
-    @default_mock ||= @final_mocks.split(' ')[0]
-    subdir = ENV['subdir'] || 'products'
-    build_rpm_with_mock(@default_mock, FALSE, subdir)
+  task :mock_final => "package:srpm" do
+    deprecate("pl:mock_final", "pl:mock_all")
+    build_rpm_with_mock(@build.final_mocks, FALSE)
+    post_metrics if @build.benchmark
   end
 
-  desc "Use specified mocks to make final rpms, keyed to PL infrastructure, pass MOCK to specifiy config"
-  task :mock_final => [ "package:srpm", "pl:setup_el_dirs" ] do
-    subdir = ENV['subdir'] || 'products'
-    build_rpm_with_mock(@final_mocks, FALSE, subdir)
+  task :mock_rc => "package:srpm" do
+    deprecate("pl:mock_rc", "pl:mock_all")
+    build_rpm_with_mock(@build.rc_mocks, TRUE)
+    post_metrics if @build.benchmark
   end
 
-  desc "Use specified mocks to make RC rpms, keyed to PL infrastructure, pass MOCK to specify config"
-  task :mock_rc => [ "package:srpm", "pl:setup_el_dirs" ] do
-    subdir = 'devel'
-    build_rpm_with_mock(@rc_mocks, TRUE, subdir)
+  desc "Use specified mocks to make rpms, keyed to PL infrastructure, pass MOCK to specifiy config"
+  task :mock_all => "package:srpm" do
+    build_rpm_with_mock(@build.final_mocks, is_rc?)
+    post_metrics if @build.benchmark
   end
 end

data/ext/packaging/tasks/pe_deb.rake

@@ -3,9 +3,9 @@
 # e.g., pe:local_deb is the task to build a deb on the local host,
 # while pe:deb is the task for building on the remote builder host
 
-if @build_pe
+if @build.build_pe
   namespace :pe do
-    desc "Create a PE deb from this repo using the default cow #{@default_cow}."
+    desc "Create a PE deb from this repo using the default cow #{@build.default_cow}."
     task :local_deb => "pl:deb"
 
     desc "Create PE debs from this git repository using all cows specified in build_defaults yaml"

data/ext/packaging/tasks/pe_remote.rake

@@ -3,44 +3,47 @@
 # e.g., pe:local_deb is the task to build a deb on the local host,
 # while pe:deb is the task for building on the remote builder host
 
-if @build_pe
+if @build.build_pe
   namespace :pe do
     desc "Execute remote debian build using default cow on builder and retrieve package"
-    task :deb => ['pl:fetch', 'pl:load_extras'] do
-      ENV['PE_VER'] ||= @pe_version
+    task :deb => 'pl:fetch' do
+      ENV['PE_VER'] ||= @build.pe_version
       check_var('PE_VER', ENV['PE_VER'])
-      Rake::Task["pl:remote_build"].reenable
-      Rake::Task["pl:remote_build"].invoke(@deb_build_host, 'HEAD', "pe:local_deb PE_VER=#{ENV['PE_VER']}")
+      Rake::Task["pl:remote:build"].reenable
+      Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:local_deb PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
     end
 
     desc "Execute remote debian build using ALL cows on builder and retrieve packages"
-    task :deb_all => ['pl:fetch', 'pl:load_extras'] do
-      ENV['PE_VER'] ||= @pe_version
+    task :deb_all => 'pl:fetch' do
+      ENV['PE_VER'] ||= @build.pe_version
       check_var('PE_VER', ENV['PE_VER'])
-      Rake::Task["pl:remote_build"].reenable
-      Rake::Task["pl:remote_build"].invoke(@deb_build_host, 'HEAD', "pe:local_deb_all PE_VER=#{ENV['PE_VER']}")
+      Rake::Task["pl:remote:build"].reenable
+      Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:local_deb_all PE_BUILD=#{@build.build_pe} COW='#{@build.cows}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
     end
 
     desc "Execute remote rpm build using default mock on builder and retrieve package"
-    task :mock => ['pl:fetch', 'pl:load_extras'] do
-      Rake::Task["pl:remote_build"].reenable
-      Rake::Task["pl:remote_build"].invoke(@rpm_build_host, 'HEAD', "pe:local_mock TEAM=#{@team}")
+    task :mock => 'pl:fetch' do
+      ENV['PE_VER'] ||= @build.pe_version
+      Rake::Task["pl:remote:build"].reenable
+      Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:local_mock PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
     end
 
     desc "Execute remote rpm build with ALL mocks on builder and retrieve packages"
-    task :mock_all => ['pl:fetch', 'pl:load_extras'] do
-      Rake::Task["pl:remote_build"].reenable
-      Rake::Task["pl:remote_build"].invoke(@rpm_build_host, 'HEAD', "pe:local_mock_final TEAM=#{@team}")
+    task :mock_all => 'pl:fetch' do
+      ENV['PE_VER'] ||= @build.pe_version
+      Rake::Task["pl:remote:build"].reenable
+      Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:local_mock_all PE_BUILD=#{@build.build_pe} MOCK='#{@build.final_mocks}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
     end
 
     desc "Execute remote sles rpm build and retrieve package"
-    task :sles => ['pl:fetch', 'pl:load_extras'] do
-      Rake::Task["pl:remote_build"].reenable
-      Rake::Task["pl:remote_build"].invoke(@sles_build_host, 'HEAD', "pe:local_sles TEAM=#{@team}")
+    task :sles => 'pl:fetch' do
+      ENV['PE_VER'] ||= @build.pe_version
+      Rake::Task["pl:remote:build"].reenable
+      Rake::Task["pl:remote:build"].invoke(@build.sles_build_host, 'HEAD', "pe:local_sles PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
     end
 
     desc "Execute remote debian, el, and sles builds, sign, and ship pkgs"
-    task :all => ['clean', 'pl:fetch', 'pl:load_extras'] do
+    task :all => ['clean', 'pl:fetch'] do
       ['pe:deb', 'pe:mock_all', 'pe:sles', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
         Rake::Task[task].execute
       end

data/ext/packaging/tasks/pe_rpm.rake

@@ -1,18 +1,18 @@
-if @build_pe
+if @build.build_pe
   namespace :pe do
     desc "Build a PE rpm using rpmbuild (requires all BuildRequires, rpmbuild, etc)"
     task :local_rpm => "package:rpm"
 
     desc "Build rpms using ALL final mocks in build_defaults yaml, keyed to PL infrastructure, pass MOCK to override"
-    task :local_mock_final => ["pl:fetch", "pl:load_extras", "pl:mock_final"] do
-      if @team == 'release'
+    task :local_mock_all => ["pl:fetch", "pl:mock_all"] do
+      if @build.team == 'release'
         Rake::Task["pe:sign_rpms"].invoke
       end
     end
 
     desc "Build a PE rpm using the default mock"
-    task :local_mock => ["pl:fetch", "pl:load_extras", "pl:mock" ] do
-      if @team == 'release'
+    task :local_mock => ["pl:fetch", "pl:mock"] do
+      if @build.team == 'release'
         Rake::Task["pe:sign_rpms"].invoke
       end
     end

data/ext/packaging/tasks/pe_ship.rake

@@ -1,37 +1,47 @@
-if @build_pe
+if @build.build_pe
   namespace :pe do
-    desc "ship PE rpms to #{@yum_host}"
-    task :ship_rpms => ["pl:load_extras"] do
+    desc "ship PE rpms to #{@build.yum_host}"
+    task :ship_rpms => "pl:fetch" do
       if empty_dir?("pkg/pe/rpm")
         STDERR.puts "The 'pkg/pe/rpm' directory has no packages. Did you run rake pe:deb?"
+        exit 1
       else
-        rsync_to('pkg/pe/rpm/', @yum_host, "#{@yum_repo_path}/#{@pe_version}/repos/")
-        Rake::Task["pe:remote_update_yum_repo"].invoke
+        target_path = ENV['YUM_REPO'] ? ENV['YUM_REPO'] : "#{@build.yum_repo_path}/#{@build.pe_version}/repos/"
+        rsync_to('pkg/pe/rpm/', @build.yum_host, target_path)
+        if @build.team == 'release'
+          Rake::Task["pe:remote:update_yum_repo"].invoke
+        end
       end
     end
 
-    desc "Update remote rpm repodata for PE on #{@yum_host}"
-    task :remote_update_yum_repo => "pl:load_extras" do
-      remote_ssh_cmd(@yum_host, "for dir in  $(find #{@apt_repo_path}/#{@pe_version}/repos/el* -type d | grep -v repodata | grep -v cache | xargs)  ; do   pushd $dir; sudo rm -rf repodata; createrepo -q -d .; popd &> /dev/null ; done; sync")
-    end
-
-    desc "Ship PE debs to #{@apt_host}"
-    task :ship_debs => "pl:load_extras" do
-      dist = @default_cow.split('-')[1]
+    desc "Ship PE debs to #{@build.apt_host}"
+    task :ship_debs => "pl:fetch" do
+      dist = @build.default_cow.split('-')[1]
       if empty_dir?("pkg/pe/deb/#{dist}")
         STDERR.puts "The 'pkg/pe/deb/#{dist}' directory has no packages. Did you run rake pe:deb?"
+        exit 1
       else
-        rsync_to("pkg/pe/deb/#{dist}/", @apt_host, "#{@apt_repo_path}/#{@pe_version}/repos/incoming/unified/")
-        Rake::Task["pe:remote_freight"].invoke
+        target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/unified/"
+        rsync_to("pkg/pe/deb/#{dist}/", @build.apt_host, target_path)
+        if @build.team == 'release'
+          Rake::Task["pe:remote:freight"].invoke
+        end
       end
     end
 
-    # This is particularly hacky. The 'pe-the-things' script resides on the @apt_host and takes packages placed
-    # in the directory/structure shown in the rsync target of pe:ship_debs and adds them to the remote PE
-    # freight repository and updates the apt repo metadata
-    desc "remote freight PE packages to #{@apt_host}"
-    task :remote_freight => "pl:load_extras" do
-      remote_ssh_cmd(@apt_host, "sudo pe-the-things #{@pe_version} #{@apt_repo_path} #{@freight_conf}")
+    namespace :remote do
+      desc "Update remote rpm repodata for PE on #{@build.yum_host}"
+      task :update_yum_repo => "pl:fetch" do
+        remote_ssh_cmd(@build.yum_host, "for dir in  $(find #{@build.apt_repo_path}/#{@build.pe_version}/repos/el* -type d | grep -v repodata | grep -v cache | xargs)  ; do   pushd $dir; sudo rm -rf repodata; createrepo -q -d .; popd &> /dev/null ; done; sync")
+      end
+
+      # This is particularly hacky. The 'pe-the-things' script resides on the @build.apt_host and takes packages placed
+      # in the directory/structure shown in the rsync target of pe:ship_debs and adds them to the remote PE
+      # freight repository and updates the apt repo metadata
+      desc "remote freight PE packages to #{@build.apt_host}"
+      task :freight => "pl:fetch" do
+        remote_ssh_cmd(@build.apt_host, "sudo pe-the-things #{@build.pe_version} #{@build.apt_repo_path} #{@build.freight_conf}")
+      end
     end
   end
 end