puppet 2.7.23 → 2.7.24

data/ext/packaging/tasks/jenkins.rake

@@ -115,6 +115,28 @@ namespace :pl do
         when /tar/ then "tar"
         else raise "Could not determine build type for #{build_task}"
       end
+
+      # Create a string of metrics to send to Jenkins for data analysis
+      dist = case build_type
+        when /deb/ then @build.default_cow.split('-')[1]
+        when /rpm/
+          if @build.pe_version
+            @build.final_mocks.split(' ')[0].split('-')[2]
+          else
+            @build.final_mocks.split(' ')[0].split('-')[1..2].join("")
+          end
+        when /dmg/ then "apple"
+        when /gem/ then "gem"
+        when /sles/ then "sles"
+        when /tar/ then "tar"
+        else raise "Could not determine build type for #{build_task}"
+      end
+
+      if @build.pe_version
+        metrics = "#{ENV['USER']}~#{@build.version}~#{@build.pe_version}~#{dist}~#{@build.team}"
+      else
+        metrics = "#{ENV['USER']}~#{@build.version}~N/A~#{dist}~#{@build.team}"
+      end
       #
       # Create the data files to send to jenkins
       properties = @build.params_to_yaml
@@ -124,7 +146,8 @@ namespace :pl do
       parameters = [{ "name" => "BUILD_PROPERTIES", "file"  => "file0" },
                     { "name" => "PROJECT_BUNDLE",   "file"  => "file1" },
                     { "name" => "PROJECT",          "value" => "#{@build.project}" },
-                    { "name" => "BUILD_TYPE",       "label" => "#{build_type}" }]
+                    { "name" => "BUILD_TYPE",       "label" => "#{build_type}" },
+                    { "name" => "METRICS",          "value" => "#{metrics}"}]
 
       # Initialize the args array that will hold all of the arguments we pass
       # to the curl utility method.
@@ -149,6 +172,7 @@ namespace :pl do
       "-Fname=PROJECT_BUNDLE"  , "-Ffile1=@#{bundle}",
       "-Fname=PROJECT"         , "-Fvalue=#{@build.project}",
       "-Fname=BUILD_TYPE"      , "-Fvalue=#{build_type}",
+      "-Fname=METRICS"         , "-Fvalue=#{metrics}",
       "-FSubmit=Build",
       "-Fjson=#{json.to_json}",
       ]

data/ext/packaging/tasks/jenkins_dynamic.rake

@@ -65,10 +65,18 @@ namespace :pl do
       properties = @build.params_to_yaml
       bundle = git_bundle('HEAD')
 
+      # Create a string of metrics to send to Jenkins for data analysis
+      if @build.pe_version
+        metrics = "#{ENV['USER']}~#{@build.version}~#{@build.pe_version}~#{@build.team}"
+      else
+        metrics = "#{ENV['USER']}~#{@build.version}~N/A~#{@build.team}"
+      end
+
       # Construct the parameters, which is an array of hashes we turn into JSON
       parameters = [{ "name" => "BUILD_PROPERTIES", "file"  => "file0" },
                     { "name" => "PROJECT_BUNDLE",   "file"  => "file1" },
-                    { "name" => "PROJECT",          "value" => "#{@build.project}" }]
+                    { "name" => "PROJECT",          "value" => "#{@build.project}" },
+                    { "name" => "METRICS",          "value" => "#{metrics}"}]
 
       # Contruct the json string
       json = JSON.generate("parameter" => parameters)
@@ -79,6 +87,7 @@ namespace :pl do
       "-Fname=BUILD_PROPERTIES", "-Ffile0=@#{properties}",
       "-Fname=PROJECT_BUNDLE"  , "-Ffile1=@#{bundle}",
       "-Fname=PROJECT"         , "-Fvalue=#{@build.project}",
+      "-Fname=METRICS"         , "-Fvalue=#{metrics}",
       "-FSubmit=Build",
       "-Fjson=#{json.to_json}",
       ]

data/ext/packaging/tasks/mock.rake

@@ -30,12 +30,41 @@ def mock_artifact(mock_config, cmd_args)
     configdir_arg = " --configdir #{configdir}"
   end
 
-  sh "#{mock} -r #{mock_config} #{configdir_arg} #{cmd_args}"
-  # Clean up the configdir
-  rm_r configdir unless configdir.nil?
+  begin
+    sh "#{mock} -r #{mock_config} #{configdir_arg} #{cmd_args}"
 
-  # Return a FileList of the build artifacts
-  FileList[File.join(basedir, mock_config, 'result','*.rpm')]
+    # Return a FileList of the build artifacts
+    return FileList[File.join(basedir, mock_config, 'result','*.rpm')]
+
+  rescue RuntimeError => error
+    build_log = File.join(basedir, mock_config, 'result', 'build.log')
+    root_log  = File.join(basedir, mock_config, 'result', 'root.log')
+    content   = File.read(build_log) if File.readable?(build_log)
+
+    if File.readable?(root_log)
+      STDERR.puts File.read(root_log)
+    end
+    if content and content.lines.count > 2
+      STDERR.puts content
+    end
+
+    # Any useful info has now been gleaned from the logs in the case of a
+    # failure, so we can safely remove basedir if this is a randomized mockroot
+    # build. Scarily enough, because of mock permissions, we can't actually
+    # just remove it, we have to sudo remove it.
+
+    if randomize and basedir and File.directory?(basedir)
+      sh "sudo -n rm -r #{basedir}"
+    end
+
+    raise error
+  ensure
+    # Unlike basedir, which we keep in the success case, we don't need
+    # configdir anymore either way, so we always clean it up if we're using
+    # randomized mockroots.
+    #
+    rm_r configdir if randomize
+  end
 end
 
 # Use mock to build an SRPM
@@ -107,7 +136,7 @@ def mock_defines(mock_config)
   family = mock_el_family(mock_config)
   version = mock_el_ver(mock_config)
   defines = ""
-  if version == "5" or family == "sles"
+  if version =~ /^(4|5)$/ or family == "sles"
     defines = %Q{--define "%dist .#{family}#{version}" \
       --define "_source_filedigest_algorithm 1" \
       --define "_binary_filedigest_algorithm 1" \
@@ -122,7 +151,7 @@ def build_rpm_with_mock(mocks)
   mocks.split(' ').each do |mock_config|
     family  = mock_el_family(mock_config)
     version = mock_el_ver(mock_config)
-    subdir  = is_rc? ? 'devel' : 'products'
+    subdir  = is_final? ? 'products' : 'devel'
     bench = Benchmark.realtime do
       # Set up the rpmbuild dir in a temp space, with our tarball and spec
       workdir = prep_rpm_build_dir
@@ -138,15 +167,6 @@ def build_rpm_with_mock(mocks)
 
       rpms.each do |rpm|
         rpm.strip!
-        unless ENV['RC_OVERRIDE'] == '1'
-          if is_rc? == FALSE and rpm =~ /[0-9]+rc[0-9]+\./
-            puts "It looks like you might be trying to ship an RC to the production repos. Leaving #{rpm}. Pass RC_OVERRIDE=1 to override."
-            next
-          elsif is_rc? and rpm !~ /[0-9]+rc[0-9]+\./
-            puts "It looks like you might be trying to ship a production release to the development repos. Leaving #{rpm}. Pass RC_OVERRIDE=1 to override."
-            next
-          end
-        end
 
         if @build.build_pe
           %x{mkdir -p pkg/pe/rpm/#{family}-#{version}-{srpms,i386,x86_64}}
@@ -194,7 +214,7 @@ def build_rpm_with_mock(mocks)
         end
       end
     end
-    add_metrics({ :dist => "#{family}-#{version}", :bench => bench }) if @build.benchmark
+    puts "Finished building in: #{bench}"
   end
 end
 
@@ -261,12 +281,10 @@ namespace :pl do
     # If default mock isn't specified, just take the first one in the @build.final_mocks list
     @build.default_mock ||= @build.final_mocks.split(' ')[0]
     build_rpm_with_mock(@build.default_mock)
-    post_metrics if @build.benchmark
   end
 
   desc "Use specified mocks to make rpms, keyed to PL infrastructure, pass MOCK to specifiy config"
   task :mock_all => "package:tar" do
     build_rpm_with_mock(@build.final_mocks)
-    post_metrics if @build.benchmark
   end
 end

data/ext/packaging/tasks/pe_ship.rake

@@ -15,27 +15,125 @@ if @build.build_pe
     desc "Ship PE debs to #{@build.apt_host}"
     task :ship_debs => "pl:fetch" do
       empty_dir?("pkg/pe/deb") and fail "The 'pkg/pe/deb' directory has no packages!"
-      target_path = ENV['APT_REPO'] ? ENV['APT_REPO'] : "#{@build.apt_repo_path}/#{@build.pe_version}/repos/incoming/disparate/"
+      target_path = ENV['APT_REPO']
+
+      #   If APT_REPO isn't specified as an environment variable, we use a temporary one
+      #   created for this specific deb ship. This enables us to escape the conflicts
+      #   introduced with simultaneous deb ships.
+      #
+      unless target_path
+        puts "Creating temporary incoming dir on #{@build.apt_host}"
+        target_path = %x{ssh -t #{@build.apt_host} 'mktemp -d -t incoming-XXXXXX'}.chomp
+      end
+
+      #   For reprepro, we ship just the debs into an incoming dir. On the remote end,
+      #   reprepro will pull these debs in and add them to the repositories based on the
+      #   dist, e.g. lucid, architecture notwithstanding.
+      #
+      #   The layout that the reprepro library will expect is:
+      #
+      #     incoming_dir/{$dists}/*.deb
+      #
+      #   ex:
+      #     incoming_dir|
+      #                 |_lucid/*.deb
+      #                 |_squeeze/*.deb
+      #                 |_precise/*.deb
+      #                 |_wheezy/*.deb
+      #
+      puts "Shipping PE debs to apt repo 'incoming' dir on #{@build.apt_host}"
       retry_on_fail(:times => 3) do
-        rsync_to("pkg/pe/deb/", @build.apt_host, target_path)
+        cd "pkg/pe/deb" do
+          Dir["**/*.deb"].each do |deb|
+            rsync_to(deb, @build.apt_host, "#{target_path}/#{File.dirname(deb)}/")
+          end
+        end
       end
+
+      #   We also ship our PE artifacts to directories for archival purposes and to
+      #   ease the gathering of both debs and sources when we do PE compose and ship. For
+      #   this case, we ship everything to directories that mirror the legacy rpm
+      #   directory format:
+      #
+      #     repos/$dist-{$architecture|source}
+      #
+      #   ex:
+      #     repos|
+      #          |_squeeze-i386
+      #          |_squeeze-amd64
+      #          |_squeeze-source
+      #
+      #   We also have concerns about shipped artifacts getting accidentally overwritten
+      #   by newer ones. To handle this, we make everything we ship to the archive
+      #   directories immutable, after rsyncing out.
+      #
+      base_path = "#{@build.apt_repo_path}/#{@build.pe_version}/repos"
+
+      puts "Shipping all built artifacts to to archive directories on #{@build.apt_host}"
+
+      @build.cows.split(' ').map { |i| i.sub('.cow','') }.each do |cow|
+        _base, dist, arch = cow.split('-')
+        unless empty_dir? "pkg/pe/deb/#{dist}"
+          archive_path = "#{base_path}/#{dist}-#{arch}"
+
+          # Ship arch-specific debs to correct dir, e.g. 'squeeze-i386'
+          unless Dir["pkg/pe/deb/#{dist}/pe-*_#{arch}.deb"].empty?
+            rsync_to("pkg/pe/deb/#{dist}/pe-*_#{arch}.deb --ignore-existing", @build.apt_host, "#{archive_path}/" )
+          end
+
+          # Ship all-arch debs to same place
+          unless Dir["pkg/pe/deb/#{dist}/pe-*_all.deb"].empty?
+            rsync_to("pkg/pe/deb/#{dist}/pe-*_all.deb --ignore-existing", @build.apt_host, "#{base_path}/#{dist}-i386/")
+            rsync_to("pkg/pe/deb/#{dist}/pe-*_all.deb --ignore-existing", @build.apt_host, "#{base_path}/#{dist}-amd64/")
+          end
+
+          unless Dir["pkg/pe/deb/#{dist}/pe-*"].select { |i| i !~ /^.*\.deb$/ }.empty?
+            # Ship source files to source dir, e.g. 'squeeze-source'
+            rsync_to("pkg/pe/deb/#{dist}/pe-* --exclude *.deb --ignore-existing", @build.apt_host, "#{base_path}/#{dist}-source")
+          end
+
+          files = Dir["pkg/pe/deb/#{dist}/pe-*{_#{arch},all}.deb"].map { |f| "#{archive_path}/#{File.basename(f)}" }
+
+          files += Dir["pkg/pe/deb/#{dist}/pe-*"].select { |f| f !~ /^.*\.deb$/ }.map { |f| "#{base_path}/#{dist}-source/#{File.basename(f)}" }
+
+          unless files.empty?
+            remote_set_immutable(@build.apt_host, files)
+          end
+        end
+      end
+
+
       if @build.team == 'release'
-        Rake::Task["pe:remote:freight"].invoke
+        Rake::Task["pe:remote:apt"].invoke(target_path)
       end
     end
 
     namespace :remote do
       desc "Update remote rpm repodata for PE on #{@build.yum_host}"
       task :update_yum_repo => "pl:fetch" do
-        remote_ssh_cmd(@build.yum_host, "for dir in  $(find #{@build.apt_repo_path}/#{@build.pe_version}/repos/el* -type d | grep -v repodata | grep -v cache | xargs)  ; do   pushd $dir; sudo rm -rf repodata; createrepo -q -d .; popd &> /dev/null ; done; sync")
+        remote_ssh_cmd(@build.yum_host, "for dir in  $(find #{@build.apt_repo_path}/#{@build.pe_version}/repos/{sles,el}* -type d | grep -v repodata | grep -v cache | xargs)  ; do pushd $dir; sudo createrepo -q -d --update .; popd &> /dev/null ; done; sync")
       end
 
-      # This is hacky. The freight.rb script resides on the @build.apt_host and takes packages placed
-      # in the directory/structure shown in the rsync target of pe:ship_debs and adds them to the remote PE
-      # freight repository and updates the apt repo metadata
-      desc "remote freight PE packages to #{@build.apt_host}"
-      task :freight => "pl:fetch" do
-        remote_ssh_cmd(@build.apt_host, "ruby /opt/enterprise/bin/freight.rb --version #{@build.pe_version} --basedir #{@build.apt_repo_path} --config /etc/freight.conf.d/#{@build.freight_conf}")
+      #   the repsimple application is a small wrapper around reprepro, the purpose of
+      #   which is largely to limit the surface area and functionality of reprepro to
+      #   some very basic tasks - add, remove, and add all in a directory. The add_all
+      #   command expects an incoming directory option containing .deb files.
+      #   Per previous comments, the incoming directory must contain subdirectories named
+      #   for debian distributions.
+      desc "Remotely add shipped packages to apt repo on #{@build.apt_host}"
+      task :apt, :incoming do |t, args|
+        incoming_dir = args.incoming
+        incoming_dir or fail "Adding packages to apt repo requires an incoming directory"
+        invoke_task("pl:fetch")
+        remote_ssh_cmd(@build.apt_host, "/usr/bin/repsimple add_all \
+            --confdir /etc/reprepro/#{@build.pe_version} \
+            --basedir #{@build.apt_repo_path}/#{@build.pe_version}/repos/debian \
+            --databasedir /var/lib/reprepro/#{@build.pe_version} \
+            --incomingdir #{incoming_dir}")
+
+        puts "Cleaning up apt repo 'incoming' dir on #{@build.apt_host}"
+        remote_ssh_cmd(@build.apt_host, "rm -r #{incoming_dir}")
+
       end
     end
   end

data/ext/packaging/tasks/pe_sign.rake

@@ -1,4 +1,4 @@
-# The sign_el5 and sign_modern methods are defined in sign.rake
+# The sign_legacy_rpm and sign_rpm methods are defined in sign.rake
 # This is just adapted for the 'PE' layout
 
 if @build.build_pe
@@ -24,8 +24,8 @@ if @build.build_pe
           end
         end
       end
-      sign_el5(old_rpms) unless old_rpms.empty?
-      sign_modern(modern_rpms) unless modern_rpms.empty?
+      sign_legacy_rpm(old_rpms) unless old_rpms.empty?
+      sign_rpm(modern_rpms) unless modern_rpms.empty?
       # Now we hardlink them back in
       Dir["pkg/pe/rpm/*-*-i386/*.noarch.rpm"].each do |rpm|
         dir = rpm.split('/')[-2]

data/ext/packaging/tasks/retrieve.rake

@@ -31,3 +31,15 @@ namespace :pl do
     end
   end
 end
+
+if @build.build_pe
+  namespace :pe do
+    namespace :jenkins do
+      desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
+      task :retrieve, :target do |t, args|
+        target = args.target || "artifacts"
+        invoke_task("pl:jenkins:retrieve", target)
+      end
+    end
+  end
+end

data/ext/packaging/tasks/rpm_repos.rake

@@ -105,7 +105,7 @@ namespace :pl do
         # ignore this one because its an extra
         next if url == "#{base_url}srpm/"
 
-        dist,version,subdir,arch = url.split('/')[-4..-1]
+        dist,version,_subdir,arch = url.split('/')[-4..-1]
 
         # Create an array of lines that will become our yum config
         #
@@ -117,7 +117,7 @@ namespace :pl do
 
         # Write the new config to a file under our repo configs dir
         #
-        config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}-#{subdir}.repo")
+        config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}.repo")
         File.open(config_file, 'w') { |f| f.puts config }
       end
       puts "Wrote yum configuration files for #{@build.project} at #{@build.ref} to pkg/repo_configs/rpm"

data/ext/packaging/tasks/ship.rake

@@ -1,11 +1,12 @@
 namespace :pl do
   desc "Ship mocked rpms to #{@build.yum_host}"
   task :ship_rpms do
-    retry_on_fail(:times => 3) do
-      rsync_to('pkg/el', @build.yum_host, @build.yum_repo_path)
-    end
-    retry_on_fail(:times => 3) do
-      rsync_to('pkg/fedora', @build.yum_host, @build.yum_repo_path)
+    ["el", "fedora"].each do |dist|
+      retry_on_fail(:times => 3) do
+        pkgs = Dir["pkg/#{dist}/**/*.rpm"].map { |f| "'#{f.gsub("pkg/#{dist}/", "#{@build.yum_repo_path}/#{dist}/")}'"}
+        rsync_to("pkg/#{dist}", @build.yum_host, @build.yum_repo_path)
+        remote_set_immutable(@build.yum_host, pkgs)
+      end
     end
   end
 
@@ -19,7 +20,7 @@ namespace :pl do
     task :update_yum_repo do
       STDOUT.puts "Really run remote repo update on #{@build.yum_host}? [y,n]"
       if ask_yes_or_no
-        remote_ssh_cmd(@build.yum_host, '/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile mk_repo')
+        remote_ssh_cmd(@build.yum_host, 'rake -f /opt/repository/Rakefile mk_repo')
       end
     end
 
@@ -28,7 +29,7 @@ namespace :pl do
       STDOUT.puts "Really run remote freight command on #{@build.apt_host}? [y,n]"
       if ask_yes_or_no
         override = "OVERRIDE=1" if ENV['OVERRIDE']
-        remote_ssh_cmd(@build.apt_host, "/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile freight #{override}")
+        remote_ssh_cmd(@build.apt_host, "rake -f /opt/repository/Rakefile freight #{override}")
       end
     end
   end
@@ -60,10 +61,23 @@ namespace :pl do
     end
   end if @build.build_ips
 
-  desc "Ship built gem to rubygems"
-  task :ship_gem do
-    ship_gem("pkg/#{@build.project}-#{@build.gemversion}.gem")
-  end if @build.build_gem
+  # We want to ship a gem only for projects that build gems
+  if @build.build_gem
+    desc "Ship built gem to rubygems"
+    task :ship_gem do
+      # Even if a project builds a gem, if it uses the odd_even strategy, we only
+      # want to ship final gems because otherwise a development gem would be
+      # preferred over the last final gem
+      if @build.version_strategy != "odd_even" || is_final?
+        FileList["pkg/#{@build.project}-#{@build.gemversion}*.gem"].each do |f|
+          puts "Shipping gem #{f} to rubygems"
+          ship_gem(f)
+        end
+      else
+        STDERR.puts "Not shipping development gem using odd_even strategy for the sake of your users."
+      end
+    end
+  end
 
   desc "ship apple dmg to #{@build.yum_host}"
   task :ship_dmg => 'pl:fetch' do
@@ -89,6 +103,11 @@ namespace :pl do
       Rake::Task["pl:ship_dmg"].execute if @build.build_dmg
       Rake::Task["pl:ship_tar"].execute
       Rake::Task["pl:jenkins:ship"].invoke("shipped")
+      add_shipped_metrics(:pe_version => ENV['PE_VER'], :is_rc => (! is_final?)) if @build.benchmark
+      post_shipped_metrics if @build.benchmark
+    else
+      puts "Ship canceled"
+      exit
     end
   end
 
@@ -104,12 +123,32 @@ namespace :pl do
       invoke_task("pl:fetch")
       target = args.target || "artifacts"
       artifact_dir = "#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}"
+
+      # In order to get a snapshot of what this build looked like at the time
+      # of shipping, we also generate and ship the params file
+      #
+      @build.params_to_yaml('pkg')
+
+
+      # Sadly, the packaging repo cannot yet act on its own, without living
+      # inside of a packaging-repo compatible project. This means in order to
+      # use the packaging repo for shipping and signing (things that really
+      # don't require build automation, specifically) we still need the project
+      # clone itself.
+      git_bundle('HEAD', 'signing_bundle', 'pkg')
+
       retry_on_fail(:times => 3) do
         remote_ssh_cmd(@build.distribution_server, "mkdir -p #{artifact_dir}")
       end
       retry_on_fail(:times => 3) do
-        rsync_to("pkg/", @build.distribution_server, "#{artifact_dir}/ --exclude repo_configs")
+        ignore_existing = "--ignore-existing"
+        rsync_to("pkg/", @build.distribution_server, "#{artifact_dir}/ #{ignore_existing} --exclude repo_configs")
+      end
+      # If we just shipped a tagged version, we want to make it immutable
+      files = Dir.glob("pkg/**/*").select { |f| File.file?(f) }.map do |file|
+        "#{artifact_dir}/#{file.sub(/^pkg\//,'')}"
       end
+      remote_set_immutable(@build.distribution_server, files)
     end
 
     desc "Ship generated repository configs to the distribution server"