puppet 2.7.22 → 2.7.23

data/ext/packaging/tasks/jenkins_dynamic.rake

@@ -0,0 +1,111 @@
+# Rake Task to dynamically create a Jenkins job to model the
+# pl:jenkins:uber_build set of tasks in a Matrix job where each cell is an
+# individual build to be run. This would be nice if we only had to create one job,
+# but alas, we're actually creating three jobs.
+# 1) a packaging job that builds the packages
+#                     |
+#                     V
+# 2) a repo creation job that creates repos from those packages
+#                     |
+#                     V
+# 3) (optional) a job to proxy the downstream job passed in via DOWNSTREAM_JOB
+#
+
+namespace :pl do
+  namespace :jenkins do
+    desc "Dynamic Jenkins UBER build: Build all the things with ONE job"
+    task :uber_build => "pl:fetch" do
+      # If we have a dirty source, bail, because changes won't get reflected in
+      # the package builds
+      fail_on_dirty_source
+
+      # Use JSON to parse the json part of the submission, so we want to fail
+      # here also if JSON isn't available
+      require_library_or_fail 'json'
+
+      # The uber_build.xml.erb file is an XML erb template that will define a
+      # job in Jenkins with all of the appropriate tasks
+      work_dir           = get_temp
+      template_dir       = File.join(File.dirname(__FILE__), '..', 'templates')
+      templates          = ['repo.xml.erb', 'packaging.xml.erb']
+      templates.unshift('downstream.xml.erb') if ENV['DOWNSTREAM_JOB']
+
+      # Generate an XML file for every job configuration erb and attempt to
+      # create a jenkins job from that XML config
+      templates.each do |t|
+        erb_file  = File.join(template_dir, t)
+        xml_file = File.join(work_dir, t.gsub('.erb', ''))
+        erb(erb_file, xml_file)
+        job_name  = "#{@build.project}-#{t.gsub('.xml.erb','')}-#{@build.build_date}-#{@build.ref}"
+        puts "Checking for existence of #{job_name}..."
+        if jenkins_job_exists?(job_name)
+          raise "Job #{job_name} already exists on #{@build.jenkins_build_host}"
+        else
+          retry_on_fail(:times => 3) do
+            url = create_jenkins_job(job_name, xml_file)
+            puts "Verifying job created successfully..."
+            unless jenkins_job_exists?(job_name)
+              raise "Unable to verify Jenkins job, trying again..."
+            end
+            puts "Jenkins job created at #{url}"
+          end
+        end
+      end
+      rm_r work_dir
+      packaging_name = "#{@build.project}-packaging-#{@build.build_date}-#{@build.ref}"
+      invoke_task("pl:jenkins:trigger_dynamic_job", packaging_name)
+    end
+
+    # Task to trigger the jenkins job we just created. This uses a lot of the
+    # same logic in jenkins.rake, with different parameters.
+    # TODO make all this replicated code a better, more abstract method
+    task :trigger_dynamic_job, :name do |t, args|
+      name = args.name
+
+      properties = @build.params_to_yaml
+      bundle = git_bundle('HEAD')
+
+      # Construct the parameters, which is an array of hashes we turn into JSON
+      parameters = [{ "name" => "BUILD_PROPERTIES", "file"  => "file0" },
+                    { "name" => "PROJECT_BUNDLE",   "file"  => "file1" },
+                    { "name" => "PROJECT",          "value" => "#{@build.project}" }]
+
+      # Contruct the json string
+      json = JSON.generate("parameter" => parameters)
+
+      # The args array that holds  all of the arguments we pass
+      # to the curl utility method.
+      curl_args =  [
+      "-Fname=BUILD_PROPERTIES", "-Ffile0=@#{properties}",
+      "-Fname=PROJECT_BUNDLE"  , "-Ffile1=@#{bundle}",
+      "-Fname=PROJECT"         , "-Fvalue=#{@build.project}",
+      "-FSubmit=Build",
+      "-Fjson=#{json.to_json}",
+      ]
+
+      # Contstruct the job url
+      trigger_url = "#{@build.jenkins_build_host}/job/#{name}/build"
+
+      if curl_form_data(trigger_url, curl_args)
+        print_url_info("#{@build.jenkins_build_host}/job/#{name}")
+        puts "Your packages will be available at #{@build.distribution_server}:#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}"
+      else
+        fail "An error occurred submitting the job to jenkins. Take a look at the preceding http response for more info."
+      end
+
+      # Clean up after ourselves
+      rm bundle
+      rm properties
+    end
+  end
+end
+
+namespace :pe do
+  namespace :jenkins do
+    desc "Dynamic Jenkins UBER build: Build all the things with ONE job"
+    task :uber_build do
+      check_var("PE_VER", @build.pe_version)
+      invoke_task("pl:jenkins:uber_build")
+    end
+  end
+end

data/ext/packaging/tasks/mock.rake

@@ -17,40 +17,50 @@
 # pkg/<dist>-<version>-<arch>/*.rpm
 # e.g.,
 # pkg/el-5-i386/*.rpm
-
-def mock(mock_config, srpm)
-  configdir = nil
+def mock_artifact(mock_config, cmd_args)
   unless mock = find_tool('mock')
-    warn "mock is required for building rpms with mock. Please install mock and try again."
-    exit 1
+    raise "mock is required for building srpms with mock. Please install mock and try again."
   end
-  if @build.random_mockroot
-    basedir = get_temp
-    chown("#{ENV['USER']}", "mock", basedir)
-    # Mock requires the sticky bit be set on the basedir
-    chmod(02775, basedir)
-    mockfile = File.join('/', 'etc', 'mock', "#{mock_config}.cfg")
-    puts "Setting mock basedir to #{basedir}"
-    config = mock_with_basedir(mockfile, basedir)
-    configdir = setup_mock_config_dir(config)
-    # Clean up the new mock config
-    rm_r  File.dirname(config)
+  randomize = @build.random_mockroot
+  configdir = nil
+  basedir = File.join('var', 'lib', 'mock')
+
+  if randomize
+    basedir, configdir = randomize_mock_config_dir(mock_config)
     configdir_arg = " --configdir #{configdir}"
-    mock << configdir_arg
   end
-  sh "#{mock} -r #{mock_config} #{srpm}"
+
+  sh "#{mock} -r #{mock_config} #{configdir_arg} #{cmd_args}"
   # Clean up the configdir
   rm_r configdir unless configdir.nil?
 
-  basedir
+  # Return a FileList of the build artifacts
+  FileList[File.join(basedir, mock_config, 'result','*.rpm')]
 end
 
-def srpm_file
-  srpm = Dir["pkg/srpm/*.rpm"][0]
-  check_file(srpm)
-  srpm
+# Use mock to build an SRPM
+# Return the path to the srpm
+def mock_srpm(mock_config, spec, sources, defines=nil)
+  cmd_args = "--buildsrpm #{defines} --sources #{sources} --spec #{spec}"
+  srpms = mock_artifact(mock_config, cmd_args)
+
+  unless srpms.size == 1
+    fail "#{srpms} contains an unexpected number of artifacts."
+  end
+  srpms[0]
 end
 
+# Use mock to build rpms from an srpm
+# Return a FileList containing the built RPMs
+def mock_rpm(mock_config, srpm)
+  cmd_args = " #{srpm}"
+  mock_artifact(mock_config, cmd_args)
+end
+
+# Determine the "family" of the target distribution based on the mock config name,
+# e.g. pupent-3.0-el5-i386 = "el"
+# and  pl-fedora-17-i386 = "fedora"
+#
 def mock_el_family(mock_config)
   if @build.build_pe
     family = mock_config.split('-')[2][/[a-z]+/]
@@ -69,6 +79,10 @@ def mock_el_family(mock_config)
   family
 end
 
+# Determine the major version of the target distribution based on the mock config name,
+# e.g. pupent-3.0-el5-i386 = "5"
+# and "pl-fedora-17-i386" = "17"
+#
 def mock_el_ver(mock_config)
   if @build.build_pe
     version = mock_config.split('-')[2][/[0-9]+/]
@@ -86,32 +100,50 @@ def mock_el_ver(mock_config)
   version
 end
 
-def mock_arch(mock_config)
-  if @build.build_pe
-    arch = mock_config.split('-')[3]
-  else
-    arch = mock_config.split('-')[2]
+# Determine the appropriate rpm macro definitions based on the mock config name
+# Return a string of space separated macros prefixed with --define
+#
+def mock_defines(mock_config)
+  family = mock_el_family(mock_config)
+  version = mock_el_ver(mock_config)
+  defines = ""
+  if version == "5" or family == "sles"
+    defines = %Q{--define "%dist .#{family}#{version}" \
+      --define "_source_filedigest_algorithm 1" \
+      --define "_binary_filedigest_algorithm 1" \
+      --define "_binary_payload w9.gzdio" \
+      --define "_source_payload w9.gzdio" \
+      --define "_default_patch_fuzz 2"}
   end
+  defines
 end
 
-def build_rpm_with_mock(mocks, is_rc)
+def build_rpm_with_mock(mocks)
   mocks.split(' ').each do |mock_config|
     family  = mock_el_family(mock_config)
     version = mock_el_ver(mock_config)
-    arch    = mock_arch(mock_config)
-    subdir  = is_rc ? 'devel' : 'products'
+    subdir  = is_rc? ? 'devel' : 'products'
     bench = Benchmark.realtime do
-      resultdir = mock(mock_config, srpm_file)
-      result  = "#{resultdir}/#{mock_config}/result/*.rpm"
+      # Set up the rpmbuild dir in a temp space, with our tarball and spec
+      workdir = prep_rpm_build_dir
+      spec = Dir.glob(File.join(workdir, "SPECS", "*.spec"))[0]
+      sources = File.join(workdir, "SOURCES")
+      defines = mock_defines(mock_config)
+
+      # Build the srpm inside a mock chroot
+      srpm = mock_srpm(mock_config, spec, sources, defines)
 
-      Dir[result].each do |rpm|
+      # Now that we have the srpm, build the rpm in a mock chroot
+      rpms = mock_rpm(mock_config, srpm)
+
+      rpms.each do |rpm|
         rpm.strip!
         unless ENV['RC_OVERRIDE'] == '1'
-          if is_rc == FALSE and rpm =~ /[0-9]+rc[0-9]+\./
-            puts "It looks like you might be trying to ship an RC to the production repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
+          if is_rc? == FALSE and rpm =~ /[0-9]+rc[0-9]+\./
+            puts "It looks like you might be trying to ship an RC to the production repos. Leaving #{rpm}. Pass RC_OVERRIDE=1 to override."
             next
-          elsif is_rc and rpm !~ /[0-9]+rc[0-9]+\./
-            puts "It looks like you might be trying to ship a production release to the development repos. Leaving rpm in #{result}. Pass RC_OVERRIDE=1 to override."
+          elsif is_rc? and rpm !~ /[0-9]+rc[0-9]+\./
+            puts "It looks like you might be trying to ship a production release to the development repos. Leaving #{rpm}. Pass RC_OVERRIDE=1 to override."
             next
           end
         end
@@ -154,9 +186,11 @@ def build_rpm_with_mock(mocks, is_rc)
       if @build.random_mockroot
         %x{sudo -n echo 'Cleaning build root.'}
         if $?.success?
-          sh "sudo -n rm -r #{resultdir}" unless resultdir.nil?
+          sh "sudo -n rm -r #{File.dirname(srpm)}" unless File.dirname(srpm).nil?
+          sh "sudo -n rm -r #{File.dirname(rpms[0])}" unless File.dirname(rpms[0]).nil?
+          sh "sudo -n rm -r #{workdir}" unless workdir.nil?
         else
-          warn "Couldn't clean #{resultdir} without sudo. Leaving."
+          warn "Couldn't clean rpm build areas without sudo. Leaving."
         end
       end
     end
@@ -198,30 +232,41 @@ def setup_mock_config_dir(mock)
   tempdir
 end
 
+# Create a mock config file from an existing one, except insert the 'basedir'
+# option. 'basedir' will be set to a random directory we create. Move the new
+# mock config and the required default mock settings files into a new config
+# dir to pass to mock. Return the path to the config dir.
+#
+def randomize_mock_config_dir(mock_config)
+  # basedir will be the location of our temporary mock root
+  basedir = get_temp
+  chown("#{ENV['USER']}", "mock", basedir)
+  # Mock requires the sticky bit be set on the basedir
+  chmod(02775, basedir)
+  mockfile = File.join('/', 'etc', 'mock', "#{mock_config}.cfg")
+  puts "Setting mock basedir to #{basedir}"
+  # Create a new mock config file with 'basedir' set to our basedir
+  config = mock_with_basedir(mockfile, basedir)
+  # Setup a mock config dir, copying in our mock config and logging.ini etc
+  configdir = setup_mock_config_dir(config)
+  # Clean up the directory with the temporary mock config
+  rm_r File.dirname(config)
+  return basedir, configdir
+end
+
+
 namespace :pl do
   desc "Use default mock to make a final rpm, keyed to PL infrastructure, pass MOCK to specify config"
-  task :mock => "package:srpm" do
+  task :mock => "package:tar" do
     # If default mock isn't specified, just take the first one in the @build.final_mocks list
     @build.default_mock ||= @build.final_mocks.split(' ')[0]
-    build_rpm_with_mock(@build.default_mock, is_rc?)
-    post_metrics if @build.benchmark
-  end
-
-  task :mock_final => "package:srpm" do
-    deprecate("pl:mock_final", "pl:mock_all")
-    build_rpm_with_mock(@build.final_mocks, FALSE)
-    post_metrics if @build.benchmark
-  end
-
-  task :mock_rc => "package:srpm" do
-    deprecate("pl:mock_rc", "pl:mock_all")
-    build_rpm_with_mock(@build.rc_mocks, TRUE)
+    build_rpm_with_mock(@build.default_mock)
     post_metrics if @build.benchmark
   end
 
   desc "Use specified mocks to make rpms, keyed to PL infrastructure, pass MOCK to specifiy config"
-  task :mock_all => "package:srpm" do
-    build_rpm_with_mock(@build.final_mocks, is_rc?)
+  task :mock_all => "package:tar" do
+    build_rpm_with_mock(@build.final_mocks)
     post_metrics if @build.benchmark
   end
 end

data/ext/packaging/tasks/pe_deb.rake

@@ -1,14 +1,12 @@
-# For PE, the natural default tasks are the remote tasks, rather than
-# the local ones, in reflection of which will be most ideal for PE devs.
-# e.g., pe:local_deb is the task to build a deb on the local host,
-# while pe:deb is the task for building on the remote builder host
-
+# "Alias" tasks for PE - these just point at the standard pl: tasks. They exist
+# for ease of aggregation with PE-specific tasks that _are_ actually different
+# from their "pl" counterparts
 if @build.build_pe
   namespace :pe do
     desc "Create a PE deb from this repo using the default cow #{@build.default_cow}."
-    task :local_deb => "pl:deb"
+    task :deb => "pl:deb"
 
     desc "Create PE debs from this git repository using all cows specified in build_defaults yaml"
-    task :local_deb_all => "pl:deb_all"
+    task :deb_all => "pl:deb_all"
   end
 end

data/ext/packaging/tasks/pe_remote.rake

@@ -1,51 +1,39 @@
-# For PE, the natural default tasks are the remote tasks, rather than
-# the local ones, in reflection of which will be most ideal for PE devs.
-# e.g., pe:local_deb is the task to build a deb on the local host,
-# while pe:deb is the task for building on the remote builder host
-
+# PE remote tasks
+# We keep these around for backwards compatibility and as a backup in case the
+# jenkins infrastructure fails. We hide them to reduce task clutter
 if @build.build_pe
   namespace :pe do
-    desc "Execute remote debian build using default cow on builder and retrieve package"
-    task :deb => 'pl:fetch' do
-      ENV['PE_VER'] ||= @build.pe_version
-      check_var('PE_VER', ENV['PE_VER'])
-      Rake::Task["pl:remote:build"].reenable
-      Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:local_deb PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
-    end
-
-    desc "Execute remote debian build using ALL cows on builder and retrieve packages"
-    task :deb_all => 'pl:fetch' do
-      ENV['PE_VER'] ||= @build.pe_version
-      check_var('PE_VER', ENV['PE_VER'])
-      Rake::Task["pl:remote:build"].reenable
-      Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:local_deb_all PE_BUILD=#{@build.build_pe} COW='#{@build.cows}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
-    end
+    namespace :remote do
+      task :deb => 'pl:fetch' do
+        ENV['PE_VER'] ||= @build.pe_version
+        check_var('PE_VER', ENV['PE_VER'])
+        Rake::Task["pl:remote:build"].reenable
+        Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:deb PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
+      end
 
-    desc "Execute remote rpm build using default mock on builder and retrieve package"
-    task :mock => 'pl:fetch' do
-      ENV['PE_VER'] ||= @build.pe_version
-      Rake::Task["pl:remote:build"].reenable
-      Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:local_mock PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
-    end
+      task :deb_all => 'pl:fetch' do
+        ENV['PE_VER'] ||= @build.pe_version
+        check_var('PE_VER', ENV['PE_VER'])
+        Rake::Task["pl:remote:build"].reenable
+        Rake::Task["pl:remote:build"].invoke(@build.deb_build_host, 'HEAD', "pe:deb_all PE_BUILD=#{@build.build_pe} COW='#{@build.cows}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
+      end
 
-    desc "Execute remote rpm build with ALL mocks on builder and retrieve packages"
-    task :mock_all => 'pl:fetch' do
-      ENV['PE_VER'] ||= @build.pe_version
-      Rake::Task["pl:remote:build"].reenable
-      Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:local_mock_all PE_BUILD=#{@build.build_pe} MOCK='#{@build.final_mocks}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
-    end
+      task :mock => 'pl:fetch' do
+        ENV['PE_VER'] ||= @build.pe_version
+        Rake::Task["pl:remote:build"].reenable
+        Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:mock PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
+      end
 
-    desc "Execute remote sles rpm build and retrieve package"
-    task :sles => 'pl:fetch' do
-      ENV['PE_VER'] ||= @build.pe_version
-      Rake::Task["pl:remote:build"].reenable
-      Rake::Task["pl:remote:build"].invoke(@build.sles_build_host, 'HEAD', "pe:local_sles PE_BUILD=#{@build.build_pe} TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
-    end
+      task :mock_all => 'pl:fetch' do
+        ENV['PE_VER'] ||= @build.pe_version
+        Rake::Task["pl:remote:build"].reenable
+        Rake::Task["pl:remote:build"].invoke(@build.rpm_build_host, 'HEAD', "pe:mock_all PE_BUILD=#{@build.build_pe} MOCK='#{@build.final_mocks}' TEAM=#{@build.team} PE_VER=#{ENV['PE_VER']}")
+      end
 
-    desc "Execute remote debian, el, and sles builds, sign, and ship pkgs"
-    task :all => ['clean', 'pl:fetch'] do
-      ['pe:deb_all', 'pe:mock_all', 'pe:sles', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
-        Rake::Task[task].execute
+      task :all => ['clean', 'pl:fetch'] do
+        ['pe:remote:deb_all', 'pe:remote:mock_all', 'pe:ship_rpms', 'pe:ship_debs'].each do |task|
+          Rake::Task[task].execute
+        end
       end
     end
   end

data/ext/packaging/tasks/pe_rpm.rake

@@ -1,21 +1,13 @@
 if @build.build_pe
   namespace :pe do
     desc "Build a PE rpm using rpmbuild (requires all BuildRequires, rpmbuild, etc)"
-    task :local_rpm => "package:rpm"
+    task :rpm => "package:rpm"
 
     desc "Build rpms using ALL final mocks in build_defaults yaml, keyed to PL infrastructure, pass MOCK to override"
-    task :local_mock_all => ["pl:fetch", "pl:mock_all"] do
-      if @build.team == 'release'
-        Rake::Task["pe:sign_rpms"].invoke
-      end
-    end
+    task :mock_all => ["pl:fetch", "pl:mock_all"]
 
     desc "Build a PE rpm using the default mock"
-    task :local_mock => ["pl:fetch", "pl:mock"] do
-      if @build.team == 'release'
-        Rake::Task["pe:sign_rpms"].invoke
-      end
-    end
+    task :mock => ["pl:fetch", "pl:mock"]
   end
 end