puppet 2.7.20 → 2.7.21

data/ext/packaging/tasks/retrieve.rake

@@ -0,0 +1,23 @@
+##
+# This task is intended to retrieve packages from the distribution server that
+# have been built by jenkins and placed in a specific location,
+# /opt/jenkins-builds/$PROJECT/$SHA where $PROJECT is the build project as
+# established in project_data.yaml and $SHA is the git sha/tag of the project that
+# was built into packages. The current day is assumed, but an environment
+# variable override exists to retrieve packages from another day. The sha/tag is
+# assumed to be the current project's HEAD, e.g.  to retrieve packages for a
+# release of 3.1.0, checkout 3.1.0 locally before retrieving.
+#
+
+namespace :pl do
+  namespace :jenkins do
+    desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
+    task :retrieve, :target do |t, args|
+      target = args.target || "artifacts"
+      invoke_task("pl:fetch")
+      mkdir_p 'pkg'
+      rsync_from("#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}/", @build.distribution_server, "pkg/")
+      puts "Packages staged in pkg"
+    end
+  end
+end

data/ext/packaging/tasks/rpm.rake

@@ -1,8 +1,12 @@
 def build_rpm(buildarg = "-bs")
   check_tool('rpmbuild')
   temp = get_temp
-  dist = find_dist_version
-  rpm_define = "#{dist} --define \"%_topdir  #{temp}\" "
+  if dist = el_version
+    if dist.to_i < 6
+      dist_string = "--define \"%dist .el#{dist}"
+    end
+  end
+  rpm_define = "#{dist_string} --define \"%_topdir  #{temp}\" "
   rpm_old_version = '--define "_source_filedigest_algorithm 1" --define "_binary_filedigest_algorithm 1" \
      --define "_binary_payload w9.gzdio" --define "_source_payload w9.gzdio" \
      --define "_default_patch_fuzz 2"'
@@ -11,13 +15,13 @@ def build_rpm(buildarg = "-bs")
   if buildarg == '-ba'
     mkdir_p 'pkg/rpm'
   end
-  if @sign_tar
+  if @build.sign_tar
     Rake::Task["pl:sign_tar"].invoke
-    cp_p "pkg/#{@name}-#{@version}.tar.gz.asc", "#{temp}/SOURCES"
+    cp_p "pkg/#{@build.project}-#{@build.version}.tar.gz.asc", "#{temp}/SOURCES"
   end
-  cp_p "pkg/#{@name}-#{@version}.tar.gz", "#{temp}/SOURCES"
-  erb "ext/redhat/#{@name}.spec.erb", "#{temp}/SPECS/#{@name}.spec"
-  sh "rpmbuild #{args} #{buildarg} --nodeps #{temp}/SPECS/#{@name}.spec"
+  cp_p "pkg/#{@build.project}-#{@build.version}.tar.gz", "#{temp}/SOURCES"
+  erb "ext/redhat/#{@build.project}.spec.erb", "#{temp}/SPECS/#{@build.project}.spec"
+  sh "rpmbuild #{args} #{buildarg} --nodeps #{temp}/SPECS/#{@build.project}.spec"
   mv FileList["#{temp}/SRPMS/*.rpm"], "pkg/srpm"
   if buildarg == '-ba'
     mv FileList["#{temp}/RPMS/*/*.rpm"], "pkg/rpm"
@@ -31,18 +35,6 @@ def build_rpm(buildarg = "-bs")
   end
 end
 
-def find_dist_version()
-   if File.exists?('/etc/fedora-release')
-      nil
-   elsif File.exists?('/etc/redhat-release')
-      dist = %x{rpm -q --qf \"%{VERSION}\" $(rpm -q --whatprovides /etc/redhat-release )}
-      unless dist.to_i > 5
-        return "--define \"%dist .el#{dist}\""
-      end
-      nil
-   end
-end
-
 namespace :package do
   desc "Create srpm from this git repository (unsigned)"
   task :srpm => :tar do

data/ext/packaging/tasks/rpm_repos.rake

@@ -0,0 +1,127 @@
+##
+#
+# A set of functionality for creating yum rpm repositories throughout the
+# standard pkg/ directory layout that the packaging repo creates. The standard
+# layout is:
+# pkg/{el,fedora}/{5,6,f16,f17,f18}/{products,devel,dependencies,extras}/{i386,x86_64,SRPMS}
+#
+# Because we'll likely be creating the repos on a server that is remote, e.g.
+# the distribution server, the logic here assumes we'll be doing everything via
+# ssh commands.
+#
+namespace :pl do
+  namespace :jenkins do
+    desc "Create yum repositories of built RPM packages for this SHA on the distribution server"
+    task :rpm_repos => "pl:fetch" do
+      # Formulate our command string, which will just find directories with rpms
+      # and create and update repositories.
+      #
+      artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
+
+      ##
+      # Test that the artifacts directory exists on the distribution server.
+      # This will give us some more helpful output.
+      #
+      cmd = 'echo "Checking for build artifacts. Will exit if not found." ; '
+      cmd << "[ -d #{artifact_directory}/artifacts ] || exit 0 ; "
+
+      ##
+      # Enter the directory containing the build artifacts and create repos.
+      #
+      cmd << "pushd #{artifact_directory} ; "
+      cmd << 'echo "Checking for running repo creation. Will wait if detected." ; '
+      cmd << "while [ -f .lock ] ; do sleep 1 ; echo -n '.' ; done ; "
+      cmd << 'echo "Setting lock" ; '
+      cmd << "touch .lock ; "
+      cmd << "rsync -avxl artifacts/ repos/ ; pushd repos ; "
+      cmd << "createrepo=$(which createrepo) ; "
+      cmd << 'for repodir in $(find ./ -name "*.rpm" | xargs -I {} dirname {}) ; do '
+      cmd << "pushd $repodir && $createrepo -d --update . && popd ; "
+      cmd << "done ; popd "
+
+      remote_ssh_cmd(@build.distribution_server, cmd)
+
+      # Always remove the lock file, even if we've failed
+      remote_ssh_cmd(@build.distribution_server, "rm -f #{artifact_directory}/.lock")
+
+      # Now that we've created our repositories, we can create the configs for
+      # them
+      Rake::Task["pl:jenkins:rpm_repo_configs"].invoke
+    end
+
+    # Generate yum configuration files that point to the repositories created
+    # on the distribution server with packages created from the current source
+    # repo commit. There is one for each dist/version that is packaged (e.g.
+    # el5, el6, etc). Files are created in pkg/repo_configs/rpm and are named
+    # pl-$project-$sha.conf, and can be placed in /etc/yum.repos.d to enable
+    # clients to install these packages.
+    #
+    desc "Create yum repository configs for package repos for this sha/tag on the distribution server"
+    task :rpm_repo_configs => "pl:fetch" do
+
+      # This is the standard path to all build artifacts on the distribution
+      # server for this commit
+      #
+      artifact_directory = File.join(@build.jenkins_repo_path, @build.project, @build.ref)
+      # First check if the artifacts directory exists
+      #
+      cmd = "[ -d #{artifact_directory} ] || exit 0 ; "
+      # Descend into the artifacts directory and test if we have any repos
+      #
+      cmd << "pushd #{artifact_directory} ; "
+      cmd << 'echo "Checking if rpm repos exists, will exit if not.." ; '
+      cmd << '[ -n "$(find repos -name "*.rpm")" ] || exit 0 ; '
+      cmd << "pushd repos ; "
+
+      cmd << 'for repo in $(find -name "repodata") ; do dirname $repo >> rpm_configs ; done'
+
+      remote_ssh_cmd(@build.distribution_server, cmd)
+
+      # There's a chance there were simply no rpms to make repos for. If so, we
+      # don't want to proceed.
+      %x{ssh -t #{@build.distribution_server} 'ls #{artifact_directory}/repos/rpm_configs'}
+      unless $?.success?
+        warn "No repos were found to generate configs from. Exiting.."
+        exit 0
+      end
+      mkdir_p "pkg"
+      rsync_from("#{artifact_directory}/repos/rpm_configs", @build.distribution_server, "pkg")
+
+      # Clean up the remote configs file
+      remote_ssh_cmd(@build.distribution_server, "rm #{artifact_directory}/repos/rpm_configs")
+
+      if File.exist?(File.join("pkg", "rpm_configs"))
+        mkdir_p File.join("pkg","repo_configs","rpm")
+
+        # Parse the rpm configs file to generate repository configs. Each line in
+        # the rpm_configs file corresponds with a repo directory on the
+        # distribution server.
+        #
+        lines = IO.readlines(File.join("pkg","rpm_configs")).map{ |l| l.chomp }.uniq
+        lines.each do |repo|
+          dist,version,subdir,arch = repo.split('/')[1..4]
+
+          # Skip any paths that don't have everything we're looking for, e.g.
+          # the top-level srpms directory that contains the original srpm from
+          # packaging
+          next if dist.nil? or version.nil? or subdir.nil? or arch.nil?
+
+          # Create an array of lines that will become our yum config
+          #
+          config = ["[pl-#{@build.project}-#{@build.ref}]"]
+          config << ["name=PL Repo for #{@build.project} at commit #{@build.ref}"]
+          config << ["baseurl=http://#{@build.builds_server}/#{@build.project}/#{@build.ref}/repos/#{dist}/#{version}/#{subdir}/#{arch}"]
+          config << ["enabled=1"]
+          config << ["gpgcheck=0"]
+
+          # Write the new config to a file under our repo configs dir
+          #
+          config_file = File.join("pkg", "repo_configs", "rpm", "pl-#{@build.project}-#{@build.ref}-#{dist}-#{version}-#{arch}-#{subdir}.repo")
+          File.open(config_file, 'w') { |f| f.puts config }
+        end
+        rm File.join("pkg","rpm_configs")
+        puts "Wrote yum configuration files for #{@build.project} at #{@build.ref} to pkg/repo_configs/rpm"
+      end
+    end
+  end
+end

data/ext/packaging/tasks/ship.rake

@@ -1,93 +1,106 @@
 namespace :pl do
-  desc "Ship mocked rpms to #{@yum_host}"
+  desc "Ship mocked rpms to #{@build.yum_host}"
   task :ship_rpms do
-    rsync_to('pkg/el', @yum_host, @yum_repo_path)
-    rsync_to('pkg/fedora', @yum_host, @yum_repo_path)
+    rsync_to('pkg/el', @build.yum_host, @build.yum_repo_path)
+    rsync_to('pkg/fedora', @build.yum_host, @build.yum_repo_path)
   end
 
-  desc "Update remote rpm repodata on #{@yum_host}"
-  task :remote_update_yum_repo do
-    STDOUT.puts "Really run remote repo update on #{@yum_host}? [y,n]"
-    if ask_yes_or_no
-      remote_ssh_cmd(@yum_host, '/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile mk_repo')
+  namespace :remote do
+    # These hacky bits execute a pre-existing rake task on the @build.apt_host
+    # The rake task takes packages in a specific directory and freights them
+    # to various target yum and apt repositories based on their specific type
+    # e.g., final vs devel vs PE vs FOSS packages
+
+    desc "Update remote rpm repodata on #{@build.yum_host}"
+    task :update_yum_repo do
+      STDOUT.puts "Really run remote repo update on #{@build.yum_host}? [y,n]"
+      if ask_yes_or_no
+        remote_ssh_cmd(@build.yum_host, '/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile mk_repo')
+      end
     end
-  end
 
-  desc "Ship cow-built debs to #{@apt_host}"
-  task :ship_debs do
-    rsync_to('pkg/deb/', @apt_host, @apt_repo_path)
+    desc "remote freight packages to repos on #{@build.apt_host}"
+    task :freight do
+      STDOUT.puts "Really run remote freight command on #{@build.apt_host}? [y,n]"
+      if ask_yes_or_no
+        override = "OVERRIDE=1" if ENV['OVERRIDE']
+        remote_ssh_cmd(@build.apt_host, "/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile freight #{override}")
+      end
+    end
   end
 
-  # These hacky bits execute a pre-existing rake task on the @apt_host that adds the debs
-  # shipped with the ship task to the apt repo and updates the repo metadata
-  desc "freight RCs to devel repos on #{@apt_host}"
-  task :remote_freight_devel do
-    STDOUT.puts "Really run remote freight RC command on #{@apt_host}? [y,n]"
-    if ask_yes_or_no
-      override = "OVERRIDE=1" if ENV['OVERRIDE']
-      # assume we're building in cows when we ship, since that's what the repo supports
-      # allow OVERRIDE as well for cases where we intend to ship final-style versions to devel repos and vice versa
-      remote_ssh_cmd(@apt_host, "/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile devel COW=1 #{override}")
-    end
+  desc "Ship cow-built debs to #{@build.apt_host}"
+  task :ship_debs do
+    rsync_to('pkg/deb/', @build.apt_host, @build.apt_repo_path)
   end
 
-  # These similar hacky bits execute the same pre-existing rake task on the @apt_host, but
-  # with a different argument
-  desc "remote freight final packages to PRODUCTION repos on #{@apt_host}"
-  task :remote_freight_final do
-    STDOUT.puts "Really run remote freight final command on #{@apt_host}? [y,n]"
-    if ask_yes_or_no
-      override = "OVERRIDE=1" if ENV['OVERRIDE']
-      remote_ssh_cmd(@apt_host, "/var/lib/gems/1.8/gems/rake-0.9.2.2/bin/rake -f /opt/repository/Rakefile community COW=1 #{override}")
-    end
+  namespace :remote do
   end
 
-  desc "Update remote ips repository on #{@ips_host}"
+  desc "Update remote ips repository on #{@build.ips_host}"
   task :update_ips_repo do
-    rsync_to('pkg/ips/pkgs/', @ips_host, @ips_store)
-    remote_ssh_cmd(@ips_host, "pkgrecv -s #{@ips_store}/pkgs/#{@name}@#{@ipsversion}.p5p -d #{@ips_repo} \\*")
-    remote_ssh_cmd(@ips_host, "pkgrepo refresh -s #{@ips_repo}")
-    remote_ssh_cmd(@ips_host, "/usr/sbin/svcadm restart svc:/application/pkg/server")
-  end if @build_ips
+    rsync_to('pkg/ips/pkgs/', @build.ips_host, @build.ips_store)
+    remote_ssh_cmd(@build.ips_host, "pkgrecv -s #{@build.ips_store}/pkgs/#{@build.project}@build.#{@build.ipsversion}.p5p -d #{@build.ips_repo} \\*")
+    remote_ssh_cmd(@build.ips_host, "pkgrepo refresh -s #{@build.ips_repo}")
+    remote_ssh_cmd(@build.ips_host, "/usr/sbin/svcadm restart svc:/application/pkg/server")
+  end if @build.build_ips
 
-  if File.exist?("#{ENV['HOME']}/.packaging/#{@builder_data_file}")
+  if File.exist?("#{ENV['HOME']}/.packaging")
     desc "Upload ips p5p packages to downloads"
-    task :ship_ips => [ 'pl:fetch', 'pl:load_extras' ] do
+    task :ship_ips => 'pl:fetch' do
       if Dir['pkg/ips/pkgs/**/*'].empty?
         STDOUT.puts "There aren't any p5p packages in pkg/ips/pkgs. Maybe something went wrong?"
       else
-        rsync_to('pkg/ips/pkgs/', @ips_package_host, @ips_path)
+        rsync_to('pkg/ips/pkgs/', @build.ips_package_host, @build.ips_path)
       end
-    end if @build_ips
+    end if @build.build_ips
   end
 
   desc "Ship built gem to rubygems"
   task :ship_gem do
-    ship_gem("pkg/#{@name}-#{@gemversion}.gem")
-  end if @build_gem
+    ship_gem("pkg/#{@build.project}-#{@build.gemversion}.gem")
+  end if @build.build_gem
 
-  if File.exist?("#{ENV['HOME']}/.packaging/#{@builder_data_file}")
-    desc "ship apple dmg to #{@yum_host}"
-    task :ship_dmg => ['pl:fetch', 'pl:load_extras'] do
-      rsync_to('pkg/apple/*.dmg', @yum_host, @dmg_path)
-    end if @build_dmg
+  if File.exist?("#{ENV['HOME']}/.packaging")
+    desc "ship apple dmg to #{@build.yum_host}"
+    task :ship_dmg => 'pl:fetch' do
+      rsync_to('pkg/apple/*.dmg', @build.yum_host, @build.dmg_path)
+    end if @build.build_dmg
 
-    desc "ship tarball and signature to #{@yum_host}"
-    task :ship_tar => ['pl:fetch', 'pl:load_extras'] do
-      rsync_to("pkg/#{@name}-#{@version}.tar.gz*", @yum_host, @tarball_path)
+    desc "ship tarball and signature to #{@build.yum_host}"
+    task :ship_tar => 'pl:fetch' do
+      rsync_to("pkg/#{@build.project}-#{@build.version}.tar.gz*", @build.yum_host, @build.tarball_path)
     end
 
     desc "UBER ship: ship all the things in pkg"
-    task :uber_ship => ['pl:fetch', 'pl:load_extras'] do
+    task :uber_ship => 'pl:fetch' do
       if confirm_ship(FileList["pkg/**/*"])
         ENV['ANSWER_OVERRIDE'] = 'yes'
-        Rake::Task["pl:ship_gem"].invoke if @build_gem
+        Rake::Task["pl:ship_gem"].invoke if @build.build_gem
         Rake::Task["pl:ship_rpms"].invoke
         Rake::Task["pl:ship_debs"].invoke
-        Rake::Task["pl:ship_dmg"].execute if @build_dmg
+        Rake::Task["pl:ship_dmg"].execute if @build.build_dmg
         Rake::Task["pl:ship_tar"].execute
+        Rake::Task["pl:jenkins:ship"].invoke("shipped")
       end
     end
   end
+
+  # It is odd to namespace this ship task under :jenkins, but this task is
+  # intended to be a component of the jenkins-based build workflow even if it
+  # doesn't interact with jenkins directly. The :target argument is so that we
+  # can invoke this task with a subdirectory of the standard distribution
+  # server path. That way we can separate out built artifacts from
+  # signed/actually shipped artifacts e.g. $path/shipped/ or $path/artifacts.
+  namespace :jenkins do
+    desc "Ship pkg directory contents to distribution server"
+    task :ship, :target do |t, args|
+      invoke_task("pl:fetch")
+      target = args.target || "artifacts"
+      artifact_dir = "#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}/#{target}"
+      remote_ssh_cmd(@build.distribution_server, "mkdir -p #{artifact_dir}")
+      rsync_to("pkg/", @build.distribution_server, "#{artifact_dir}/ --exclude repo_configs")
+    end
+  end
 end
 

data/ext/packaging/tasks/sign.rake

@@ -1,18 +1,23 @@
 def sign_el5(rpm)
-  %x{rpm --define '%_gpg_name #{@gpg_name}' --define '%__gpg_sign_cmd %{__gpg} gpg --force-v3-sigs --digest-algo=sha1 --batch --no-verbose --no-armor --passphrase-fd 3 --no-secmem-warning -u %{_gpg_name} -sbo %{__signature_filename} %{__plaintext_filename}' --addsign #{rpm} > /dev/null}
+  # Try this up to 5 times, to allow for incorrect passwords
+  retry_on_fail(:times => 5) do
+    sh "rpm --define '%_gpg_name #{@build.gpg_name}' --define '%__gpg_sign_cmd %{__gpg} gpg --force-v3-sigs --digest-algo=sha1 --batch --no-verbose --no-armor --passphrase-fd 3 --no-secmem-warning -u %{_gpg_name} -sbo %{__signature_filename} %{__plaintext_filename}' --addsign #{rpm} > /dev/null"
+  end
 end
 
 def sign_modern(rpm)
-  %x{rpm --define '%_gpg_name #{@gpg_name}' --addsign #{rpm} > /dev/null}
+  retry_on_fail(:times => 5) do
+    sh "rpm --define '%_gpg_name #{@build.gpg_name}' --addsign #{rpm} > /dev/null"
+  end
 end
 
 def rpm_has_sig(rpm)
-  %x{rpm -Kv #{rpm} | grep "#{@gpg_key.downcase}" &> /dev/null}
+  %x{rpm -Kv #{rpm} | grep "#{@build.gpg_key.downcase}" &> /dev/null}
   $?.success?
 end
 
 def sign_deb_changes(file)
-  %x{debsign --re-sign -k#{@gpg_key} #{file}}
+  %x{debsign --re-sign -k#{@build.gpg_key} #{file}}
 end
 
 # requires atleast a self signed prvate key and certificate pair
@@ -21,18 +26,18 @@ end
 # technically this can be any ips-compliant package identifier, e.g. application/facter
 # repo_uri is the path to the repo currently containing the package
 def sign_ips(fmri, repo_uri)
-  %x{pkgsign -s #{repo_uri}  -k #{@privatekey_pem} -c #{@certificate_pem} -i #{@ips_inter_cert} #{fmri}}
+  %x{pkgsign -s #{repo_uri}  -k #{@build.privatekey_pem} -c #{@build.certificate_pem} -i #{@build.ips_inter_cert} #{fmri}}
 end
 
 namespace :pl do
   desc "Sign the tarball, defaults to PL key, pass GPG_KEY to override or edit build_defaults"
   task :sign_tar do
-    unless File.exist? "pkg/#{@name}-#{@version}.tar.gz"
+    if File.exist? "pkg/#{@build.project}-#{@build.version}.tar.gz"
+      load_keychain if has_tool('keychain')
+      gpg_sign_file "pkg/#{@build.project}-#{@build.version}.tar.gz"
+    else
       STDERR.puts "No tarball exists. Try rake package:tar?"
-      exit 1
     end
-    load_keychain if has_tool('keychain')
-    gpg_sign_file "pkg/#{@name}-#{@version}.tar.gz"
   end
 
   desc "Sign mocked rpms, Defaults to PL Key, pass KEY to override"
@@ -56,7 +61,7 @@ namespace :pl do
     fmri      = args.fmri
     puts "Signing ips packages..."
     sign_ips(fmri, repo_uri)
-  end if @build_ips
+  end if @build.build_ips
 
   desc "Check if all rpms are signed"
   task :check_rpm_sigs do
@@ -81,5 +86,28 @@ namespace :pl do
     sign_deb_changes("pkg/deb/*/*.changes") unless Dir["pkg/deb/*/*.changes"].empty?
     sign_deb_changes("pkg/deb/*.changes") unless Dir["pkg/deb/*.changes"].empty?
   end
+
+  ##
+  # This crazy piece of work establishes a remote repo on the distribution
+  # server, ships our packages out to it, signs them, and brings them back.
+  #
+  namespace :jenkins do
+    desc "Sign all locally staged packages on #{@build.distribution_server}"
+    task :sign_all => "pl:fetch" do
+      if Dir["pkg/*"].empty?
+        warn "There were files found in pkg/. Maybe you wanted to build/retrieve something first?"
+        exit 1
+      end
+      sign_tasks   = ["pl:sign_tar", "pl:sign_rpms", "pl:sign_deb_changes"]
+      remote_repo  = remote_bootstrap(@build.distribution_server, 'HEAD')
+      build_params = remote_buildparams(@build.distribution_server, @build)
+      rsync_to('pkg', @build.distribution_server, remote_repo)
+      remote_ssh_cmd(@build.distribution_server, "cd #{remote_repo} ; rake #{sign_tasks.join(' ')} PARAMS_FILE=#{build_params}")
+      rsync_from("#{remote_repo}/pkg/", @build.distribution_server, "pkg/")
+      remote_ssh_cmd(@build.distribution_server, "rm -rf #{remote_repo}")
+      remote_ssh_cmd(@build.distribution_server, "rm #{build_params}")
+      puts "Signed packages staged in 'pkg/ directory"
+    end
+  end
 end