packaging 0.101.0 → 0.105.0

data/spec/lib/packaging/sign_spec.rb

@@ -15,13 +15,6 @@ Header V4 RSA/SHA256 Signature, key ID ef8d349f: NOKEY
 Header SHA1 digest: OK (3cb7e9861e8bc09783a1b6c8d88243a3c16daa81)
 V4 RSA/SHA256 Signature, key ID ef8d349f: NOKEY
 MD5 digest: OK (d5f06ba2a9053de532326d0659ec0d11)
-DOC
-      }
-      let(:el5_signed_response) { <<-DOC
-Header V3 RSA/SHA1 signature: NOKEY, key ID ef8d349f
-Header SHA1 digest: OK (12ea7bd578097a3aecc5deb8ada6aca6147d68e3)
-V3 RSA/SHA1 signature: NOKEY, key ID ef8d349f
-MD5 digest: OK (27353c6153068a3c9902fcb4ad5b8b92)
 DOC
       }
       let(:sles12_signed_response) { <<-DOC
@@ -40,10 +33,6 @@ DOC
         allow(Pkg::Sign::Rpm).to receive(:`).and_return(el7_signed_response)
         expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
       end
-      it 'returns true if rpm has been signed (el5)' do
-        allow(Pkg::Sign::Rpm).to receive(:`).and_return(el5_signed_response)
-        expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
-      end
       it 'returns true if rpm has been signed (sles12)' do
         allow(Pkg::Sign::Rpm).to receive(:`).and_return(sles12_signed_response)
         expect(Pkg::Sign::Rpm.has_sig?(rpm)).to be true
@@ -68,7 +57,6 @@ DOC
         "#{rpm_directory}/aix/7.1/PC1/ppc/puppet-agent-5.5.3-1.aix7.1.ppc.rpm",
       ] }
       let(:v3_rpms) { [
-        "#{rpm_directory}/el/5/PC1/i386/puppet-agent-5.5.3-1.el5.i386.rpm",
         "#{rpm_directory}/sles/11/PC1/x86_64/puppet-agent-5.5.3-1.sles11.x86_64.rpm",
       ] }
       let(:v4_rpms) { [

data/spec/lib/packaging/util/ship_spec.rb

@@ -51,7 +51,6 @@ describe '#Pkg::Util::Ship' do
     pkg/sles/12/puppet6/ppc64le/puppet-agent-6.19.0-1.sles12.ppc64le.rpm
     pkg/sles/12/puppet6/x86_64/puppet-agent-6.19.0-1.sles12.x86_64.rpm
     pkg/sles/15/puppet6/x86_64/puppet-agent-6.19.0-1.sles15.x86_64.rpm
-    pkg/apple/10.14/puppet6/x86_64/puppet-agent-6.19.0-1.osx10.14.dmg
     pkg/apple/10.15/puppet6/x86_64/puppet-agent-6.19.0-1.osx10.15.dmg
     pkg/fedora/32/puppet6/x86_64/puppet-agent-6.19.0-1.fc32.x86_64.rpm
     pkg/windows/puppet-agent-6.19.0-x64.msi
@@ -71,7 +70,6 @@ describe '#Pkg::Util::Ship' do
     pkg/puppet6/sles/12/ppc64le/puppet-agent-6.19.0-1.sles12.ppc64le.rpm
     pkg/puppet6/sles/12/x86_64/puppet-agent-6.19.0-1.sles12.x86_64.rpm
     pkg/puppet6/sles/15/x86_64/puppet-agent-6.19.0-1.sles15.x86_64.rpm
-    pkg/mac/puppet6/10.14/x86_64/puppet-agent-6.19.0-1.osx10.14.dmg
     pkg/mac/puppet6/10.15/x86_64/puppet-agent-6.19.0-1.osx10.15.dmg
     pkg/puppet6/fedora/32/x86_64/puppet-agent-6.19.0-1.fc32.x86_64.rpm
     pkg/windows/puppet6/puppet-agent-6.19.0-x64.msi

data/tasks/deb_repos.rake

@@ -10,6 +10,7 @@
 #
 namespace :pl do
   namespace :jenkins do
+    # The equivalent to invoking this task is calling Pkg::Util::Repo.deb_repos
     desc "Create apt repositories of build DEB packages for this SHA on the distributions erver"
     task :deb_repos => "pl:fetch" do
       Pkg::Deb::Repo.create_repos

data/tasks/fetch.rake

@@ -26,6 +26,7 @@ end
 # It uses curl to download the files, and places them in a temporary
 # directory, e.g. /tmp/somedirectory/{project,team}/Pkg::Config.builder_data_file
 namespace :pl do
+  # The equivalent to invoking this task is calling Pkg::Util::File.fetch
   desc "retrieve build-data configurations to override/extend local build_defaults"
   task :fetch do
     # Remove .packaging directory from old-style extras loading

data/tasks/jenkins.rake

@@ -286,7 +286,7 @@ namespace :pl do
         ship_nightly_msi
       )
       tasks.map { |t| "pl:#{t}" }.each do |t|
-        puts "Running #{t} . . ."
+        puts "Running #{t}:"
         Rake::Task[t].invoke
       end
     end

data/tasks/load_extras.rake

@@ -6,6 +6,7 @@
 # generic tasks with data not generally useful outside the
 # PL Release team
 namespace :pl do
+  # The equivalent to invoking this task is calling Pkg::Util::File.load_extras(temp_directory)
   task :load_extras, :tempdir do |t, args|
     unless ENV['PARAMS_FILE'] && ENV['PARAMS_FILE'] != ''
       tempdir = args.tempdir

data/tasks/rpm_repos.rake

@@ -11,6 +11,7 @@
 #
 namespace :pl do
   namespace :jenkins do
+    # The equivalent to invoking this task is calling Pkg::Util::Repo.rpm_repos
     desc "Create yum repositories of built RPM packages for this SHA on the distribution server"
     task :rpm_repos => "pl:fetch" do
       Pkg::Rpm::Repo.create_remote_repos

data/tasks/ship.rake

@@ -1,44 +1,77 @@
 namespace :pl do
   namespace :remote do
-    # These hacky bits execute a pre-existing rake task on the Pkg::Config.apt_host
-    # The rake task takes packages in a specific directory and freights them
-    # to various target yum and apt repositories based on their specific type
-    # e.g., final vs devel vs PE vs FOSS packages
+    # Repo updates are the ways we convince the various repos to regenerate any repo-based
+    # metadata.
+    #
+    # It is broken up into various pieces and types to try to avoid too much redundant
+    # behavior.
 
     desc "Update '#{Pkg::Config.repo_name}' yum repository on '#{Pkg::Config.yum_staging_server}'"
     task update_yum_repo: 'pl:fetch' do
       command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository/Rakefile mk_repo'
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => Pkg::Paths.yum_repo_name, :repo_path => Pkg::Config.yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.yum_staging_server,
+        command,
+        {
+          repo_name: Pkg::Paths.yum_repo_name,
+          repo_path: Pkg::Config.yum_repo_path,
+          repo_host: Pkg::Config.yum_staging_server
+        }
+      )
     end
 
     desc "Update all final yum repositories on '#{Pkg::Config.yum_staging_server}'"
     task update_all_final_yum_repos: 'pl:fetch' do
       command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository/Rakefile mk_repo'
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => '', :repo_path => Pkg::Config.yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.yum_staging_server,
+        command,
+        {
+          repo_name: '',
+          repo_path: Pkg::Config.yum_repo_path,
+          repo_host: Pkg::Config.yum_staging_server
+        }
+      )
     end
 
     desc "Update '#{Pkg::Config.nonfinal_repo_name}' nightly yum repository on '#{Pkg::Config.yum_staging_server}'"
     task update_nightlies_yum_repo: 'pl:fetch' do
       command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository-nightlies/Rakefile mk_repo'
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => Pkg::Config.nonfinal_repo_name, :repo_path => Pkg::Config.nonfinal_yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.yum_staging_server,
+        command,
+        {
+          repo_name: Pkg::Config.nonfinal_repo_name,
+          repo_path: Pkg::Config.nonfinal_yum_repo_path,
+          repo_host: Pkg::Config.yum_staging_server
+        }
+      )
     end
 
     desc "Update all nightly yum repositories on '#{Pkg::Config.yum_staging_server}'"
     task update_all_nightlies_yum_repos: 'pl:fetch' do
       command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository-nightlies/Rakefile mk_repo'
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => '', :repo_path => Pkg::Config.nonfinal_yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.yum_staging_server,
+        command,
+        {
+          repo_name: '',
+          repo_path: Pkg::Config.nonfinal_yum_repo_path,
+          repo_host: Pkg::Config.yum_staging_server
+        }
+      )
     end
 
     task freight: :update_apt_repo
@@ -46,17 +79,35 @@ namespace :pl do
     desc "Update remote apt repository on '#{Pkg::Config.apt_signing_server}'"
     task update_apt_repo: 'pl:fetch' do
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.apt_signing_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.apt_signing_server, Pkg::Config.apt_repo_command, { :repo_name => Pkg::Paths.apt_repo_name, :repo_path => Pkg::Config.apt_repo_path, :repo_host => Pkg::Config.apt_host, :repo_url => Pkg::Config.apt_repo_url })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.apt_signing_server,
+        Pkg::Config.apt_repo_command,
+        {
+          repo_name: Pkg::Paths.apt_repo_name,
+          repo_path: Pkg::Config.apt_repo_path,
+          repo_host: Pkg::Config.apt_host,
+          repo_url: Pkg::Config.apt_repo_url
+        }
+      )
     end
 
     desc "Update nightlies apt repository on '#{Pkg::Config.apt_signing_server}'"
     task update_nightlies_apt_repo: 'pl:fetch' do
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.apt_signing_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.apt_signing_server, Pkg::Config.nonfinal_apt_repo_command, { :repo_name => Pkg::Config.nonfinal_repo_name, :repo_path => Pkg::Config.nonfinal_apt_repo_path, :repo_host => Pkg::Config.apt_host, :repo_url => Pkg::Config.apt_repo_url })
-      end
+      next unless Pkg::Util.ask_yes_or_no
+
+      Pkg::Repo.update_repo(
+        Pkg::Config.apt_signing_server,
+        Pkg::Config.nonfinal_apt_repo_command,
+        {
+          repo_name: Pkg::Config.nonfinal_repo_name,
+          repo_path: Pkg::Config.nonfinal_apt_repo_path,
+          repo_host: Pkg::Config.apt_host,
+          repo_url: Pkg::Config.apt_repo_url
+        }
+      )
     end
 
     desc "Update apt and yum repos"
@@ -74,32 +125,31 @@ namespace :pl do
     desc "Update remote ips repository on #{Pkg::Config.ips_host}"
     task :update_ips_repo  => 'pl:fetch' do
       if Dir['pkg/ips/pkgs/**/*'].empty? && Dir['pkg/solaris/11/**/*'].empty?
-        $stdout.puts "There aren't any p5p packages in pkg/ips/pkgs or pkg/solaris/11. Maybe something went wrong?"
-      else
+        $stdout.puts "Error: there aren't any p5p packages in pkg/ips/pkgs or pkg/solaris/11."
+        next
+      end
 
-        if !Dir['pkg/ips/pkgs/**/*'].empty?
-          source_dir = 'pkg/ips/pkgs/'
-        else
-          source_dir = 'pkg/solaris/11/'
-        end
+      source_dir = 'pkg/solaris/11/'
+      source_dir = 'pkg/ips/pkgs/' unless Dir['pkg/ips/pkgs/**/*'].empty?
 
-        tmpdir, _ = Pkg::Util::Net.remote_execute(
-                  Pkg::Config.ips_host,
-                  'mktemp -d -p /var/tmp',
-                  { capture_output: true }
-                )
-        tmpdir.chomp!
+      tmpdir, _ = Pkg::Util::Net.remote_execute(
+                Pkg::Config.ips_host,
+                'mktemp -d -p /var/tmp',
+                { capture_output: true }
+              )
+      tmpdir.chomp!
 
-        Pkg::Util::Net.rsync_to(source_dir, Pkg::Config.ips_host, tmpdir)
+      Pkg::Util::Net.rsync_to(source_dir, Pkg::Config.ips_host, tmpdir)
 
-        remote_cmd = %(for pkg in #{tmpdir}/*.p5p; do
-      sudo pkgrecv -s $pkg -d #{Pkg::Config.ips_path} '*';
+      remote_cmd = %(for pkg in #{tmpdir}/*.p5p; do
+        sudo pkgrecv -s $pkg -d #{Pkg::Config.ips_path} '*';
       done)
 
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, remote_cmd)
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
-      end
+      Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, remote_cmd)
+      Pkg::Util::Net.remote_execute(Pkg::Config.ips_host,
+        "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.ips_host,
+        "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
     end
 
     desc "Move dmg repos from #{Pkg::Config.dmg_staging_server} to #{Pkg::Config.dmg_host}"
@@ -235,6 +285,7 @@ namespace :pl do
 
     desc "Sync signed apt repos from #{Pkg::Config.apt_signing_server} to Google Cloud Platform"
     task :sync_apt_repo_to_gcp => 'pl:fetch' do
+      ssh = Pkg::Util::Tool.check_tool('ssh')
       target_site = 'apt.repos.puppetlabs.com'
       sync_command_puppet_6 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet6"
       sync_command_puppet_7 = "#{GCP_REPO_SYNC} apt.repos.puppet.com puppet7"
@@ -243,8 +294,11 @@ namespace :pl do
       puts
 
       Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_6)
-        Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, sync_command_puppet_7)
+        %x(#{ssh} #{Pkg::Config.apt_signing_server} '/bin/bash -l -c "#{sync_command_puppet_6}"')
+      end
+
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        %x(#{ssh} #{Pkg::Config.apt_signing_server} '/bin/bash -l -c "#{sync_command_puppet_7}"')
       end
     end
     # Keep 'deploy' for backward compatibility
@@ -292,6 +346,13 @@ namespace :pl do
     end
   end
 
+  ##
+  ## Here's where we start 'shipping' (old terminology) or 'staging' (current terminology)
+  ## by copying local 'pkg' directories to the staging server.
+  ##
+  ## Note, that for debs, we conflate 'staging server' with 'signing server' because we
+  ## must stage in th place where we sign.
+  ##
   desc "Ship mocked rpms to #{Pkg::Config.yum_staging_server}"
   task ship_rpms: 'pl:fetch' do
     Pkg::Util::Ship.ship_rpms('pkg', Pkg::Config.yum_repo_path)
@@ -302,6 +363,7 @@ namespace :pl do
     Pkg::Util::Ship.ship_rpms('pkg', Pkg::Config.nonfinal_yum_repo_path, nonfinal: true)
   end
 
+  ## This is the old-style deb shipping
   desc "Ship cow-built debs to #{Pkg::Config.apt_signing_server}"
   task ship_debs: 'pl:fetch' do
     Pkg::Util::Ship.ship_debs('pkg', Pkg::Config.apt_repo_staging_path, chattr: false)
@@ -309,7 +371,20 @@ namespace :pl do
 
   desc "Ship nightly debs to #{Pkg::Config.apt_signing_server}"
   task ship_nightly_debs: 'pl:fetch' do
-    Pkg::Util::Ship.ship_debs('pkg', Pkg::Config.nonfinal_apt_repo_staging_path, chattr: false, nonfinal: true)
+    Pkg::Util::Ship.ship_debs(
+      'pkg', Pkg::Config.nonfinal_apt_repo_staging_path, chattr: false, nonfinal: true)
+  end
+
+  ## This is the new-style apt stager
+  desc "Stage debs to #{Pkg::Config.apt_signing_server}"
+  task stage_stable_debs: 'pl:fetch' do
+    Pkg::Util::AptStagingServer.send_packages('pkg', 'stable')
+  end
+  task stage_debs: :stage_stable_debs
+
+  desc "Stage nightly debs to #{Pkg::Config.apt_signing_server}"
+  task stage_nightly_debs: 'pl:fetch' do
+    Pkg::Util::AptStagingServer.send_packages('pkg', 'nightly')
   end
 
   desc 'Ship built gem to rubygems.org, internal Gem mirror, and public file server'
@@ -325,12 +400,13 @@ namespace :pl do
           puts 'This will ship to an internal gem mirror, a public file server, and rubygems.org'
           puts "Do you want to start shipping the rubygem '#{gem_file}'?"
           next unless Pkg::Util.ask_yes_or_no
+
           Rake::Task['pl:ship_gem_to_rubygems'].execute(file: gem_file)
         end
 
         Rake::Task['pl:ship_gem_to_downloads'].invoke
       else
-        $stderr.puts 'Not shipping development gem using odd_even strategy for the sake of your users.'
+        warn 'Not shipping development gem using odd_even strategy for the sake of your users.'
       end
     end
   end
@@ -342,6 +418,7 @@ namespace :pl do
     if Pkg::Config.build_gem
       fail 'Value `Pkg::Config.gem_host` not defined, skipping nightly ship' unless Pkg::Config.gem_host
       fail 'Value `Pkg::Config.nonfinal_gem_path` not defined, skipping nightly ship' unless Pkg::Config.nonfinal_gem_path
+
       FileList['pkg/*.gem'].each do |gem_file|
         Pkg::Gem.ship_to_internal_mirror(gem_file)
       end
@@ -451,22 +528,25 @@ namespace :pl do
 
   desc 'UBER ship: ship all the things in pkg'
   task uber_ship: 'pl:fetch' do
-    if Pkg::Util.confirm_ship(FileList['pkg/**/*'])
-      Rake::Task['pl:ship_rpms'].invoke
-      Rake::Task['pl:ship_debs'].invoke
-      Rake::Task['pl:ship_dmg'].invoke
-      Rake::Task['pl:ship_swix'].invoke
-      Rake::Task['pl:ship_nuget'].invoke
-      Rake::Task['pl:ship_tar'].invoke
-      Rake::Task['pl:ship_svr4'].invoke
-      Rake::Task['pl:ship_p5p'].invoke
-      Rake::Task['pl:ship_msi'].invoke
-      add_shipped_metrics(pe_version: ENV['PE_VER'], is_rc: !Pkg::Util::Version.final?) if Pkg::Config.benchmark
-      post_shipped_metrics if Pkg::Config.benchmark
-    else
+    unless Pkg::Util.confirm_ship(FileList['pkg/**/*'])
       puts 'Ship canceled'
       exit
     end
+
+    Rake::Task['pl:ship_rpms'].invoke
+    Rake::Task['pl:ship_debs'].invoke
+    Rake::Task['pl:ship_dmg'].invoke
+    Rake::Task['pl:ship_swix'].invoke
+    Rake::Task['pl:ship_nuget'].invoke
+    Rake::Task['pl:ship_tar'].invoke
+    Rake::Task['pl:ship_svr4'].invoke
+    Rake::Task['pl:ship_p5p'].invoke
+    Rake::Task['pl:ship_msi'].invoke
+
+    if Pkg::Config.benchmark
+      add_shipped_metrics(pe_version: ENV['PE_VER'], is_rc: !Pkg::Util::Version.final?)
+      post_shipped_metrics
+    end
   end
 
   desc 'Create the rolling repo links'
@@ -530,7 +610,7 @@ namespace :pl do
           { extra_options: '-oBatchMode=yes' }
         )
       end
-    rescue
+    rescue StandardError
       errs << "Unlocking the OSX keychain failed! Check the password in your .bashrc on #{Pkg::Config.osx_signing_server}"
     end
 
@@ -565,70 +645,62 @@ namespace :pl do
   # server path. That way we can separate out built artifacts from
   # signed/actually shipped artifacts e.g. $path/shipped/ or $path/artifacts.
   namespace :jenkins do
+    # The equivalent to invoking this task is calling Pkg::Util::Ship.ship_to_artifactory(local_directory, target)
     desc 'ship pkg directory contents to artifactory'
     task :ship_to_artifactory, :local_dir do |_t, args|
       Pkg::Util::RakeUtils.invoke_task('pl:fetch')
       unless Pkg::Config.project
-        fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
+        fail "Error: 'project' must be set in build_defaults.yaml or " \
+             "in the 'PROJECT_OVERRIDE' environment variable."
       end
+
       artifactory = Pkg::ManageArtifactory.new(Pkg::Config.project, Pkg::Config.ref)
 
       local_dir = args.local_dir || 'pkg'
-      artifacts = Dir.glob("#{local_dir}/**/*").reject { |e| File.directory? e }
-      artifacts.sort! do |a, b|
-        if File.extname(a) =~ /(md5|sha\d+)/ && File.extname(b) !~ /(md5|sha\d+)/
-          1
-        elsif File.extname(b) =~ /(md5|sha\d+)/ && File.extname(a) !~ /(md5|sha\d+)/
-          -1
-        else
-          a <=> b
-        end
-      end
-      artifacts.each do |artifact|
-        if File.extname(artifact) == ".yaml" || File.extname(artifact) == ".json"
-          artifactory.deploy_package(artifact)
-        elsif artifactory.package_exists_on_artifactory?(artifact)
-          warn "Attempt to upload '#{artifact}' failed. Package already exists!"
-        else
+      Dir.glob("#{local_dir}/**/*").reject { |e| File.directory? e }.each do |artifact|
+        # Always deploy yamls and jsons
+        if artifact.end_with?('.yaml', '.json')
           artifactory.deploy_package(artifact)
+          next
         end
+
+        # Don't deploy if the package already exists
+        if artifactory.package_exists_on_artifactory?(artifact)
+          warn "Attempt to upload '#{artifact}' failed. Package already exists."
+          next
+        end
+
+        artifactory.deploy_package(artifact)
       end
     end
 
-    desc 'Ship pkg directory contents to distribution server'
+    # The equivalent to invoking this task is calling Pkg::Util::Ship.ship(local_directory, target)
+    desc 'Ship "pkg" directory contents to distribution server'
     task :ship, :target, :local_dir do |_t, args|
       Pkg::Util::RakeUtils.invoke_task('pl:fetch')
       unless Pkg::Config.project
-        fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
+        fail "Error: 'project' must be set in build_defaults.yaml or " \
+             "in the 'PROJECT_OVERRIDE' environment variable."
       end
+
       target = args.target || 'artifacts'
       local_dir = args.local_dir || 'pkg'
-      project_basedir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
-      artifact_dir = "#{project_basedir}/#{target}"
+      project_basedir = File.join(
+        Pkg::Config.jenkins_repo_path, Pkg::Config.project, Pkg::Config.ref
+      )
+      artifact_dir = File.join(project_basedir, target)
 
       # For EZBake builds, we also want to include the ezbake.manifest file to
       # get a snapshot of this build and all dependencies. We eventually will
       # create a yaml version of this file, but until that point we want to
       # make the original ezbake.manifest available
-      #
-      ezbake_manifest = File.join('ext', 'ezbake.manifest')
-      if File.exist?(ezbake_manifest)
-        cp(ezbake_manifest, File.join(local_dir, "#{Pkg::Config.ref}.ezbake.manifest"))
-      end
-      ezbake_yaml = File.join("ext", "ezbake.manifest.yaml")
-      if File.exists?(ezbake_yaml)
-        cp(ezbake_yaml, File.join(local_dir, "#{Pkg::Config.ref}.ezbake.manifest.yaml"))
-      end
+      Pkg::Util::EZbake.add_manifest(local_dir)
 
       # Inside build_metadata*.json files there is additional metadata containing
       # information such as git ref and dependencies that are needed at build
       # time. If these files exist, copy them downstream.
       # Typically these files are named 'ext/build_metadata.<project>.<platform>.json'
-      build_metadata_json_files = Dir.glob('ext/build_metadata*.json')
-      build_metadata_json_files.each do |source_file|
-        target_file = File.join(local_dir, "#{Pkg::Config.ref}.#{File.basename(source_file)}")
-        cp(source_file, target_file)
-      end
+      Pkg::Util::BuildMetadata.add_misc_json_files(local_dir)
 
       # Sadly, the packaging repo cannot yet act on its own, without living
       # inside of a packaging-repo compatible project. This means in order to
@@ -664,54 +736,11 @@ namespace :pl do
       # and if the source package exists before linking. Searching for the
       # packages has been restricted specifically to just the pkg/windows dir
       # on purpose, as this is where we currently have all windows packages
-      # building to. Once we move the Metadata about the output location in
-      # to one source of truth we can refactor this to use that to search
-      #                                           -Sean P. M. 08/12/16
-
-      {
-        'windows' => ['x86', 'x64'],
-        'windowsfips' => ['x64']
-      }.each_pair do |platform, archs|
-        packages = Dir["#{local_dir}/#{platform}/*"]
-
-        archs.each do |arch|
-          package_version = Pkg::Util::Git.describe.tr('-', '.')
-          package_filename = File.join(local_dir, platform, "#{Pkg::Config.project}-#{package_version}-#{arch}.msi")
-          link_filename = File.join(local_dir, platform, "#{Pkg::Config.project}-#{arch}.msi")
-
-          next unless !packages.include?(link_filename) && packages.include?(package_filename)
-          # Dear future code spelunkers:
-          # Using symlinks instead of hard links causes failures when we try
-          # to set these files to be immutable. Also be wary of whether the
-          # linking utility you're using expects the source path to be relative
-          # to the link target or pwd.
-          #
-          FileUtils.ln(package_filename, link_filename)
-        end
-      end
-
-      Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
-        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
-        Pkg::Util::Net.rsync_to("#{local_dir}/", Pkg::Config.distribution_server, "#{artifact_dir}/", extra_flags: ['--ignore-existing', '--exclude repo_configs'])
-      end
-
-      # In order to get a snapshot of what this build looked like at the time
-      # of shipping, we also generate and ship the params file
-      #
-      Pkg::Config.config_to_yaml(local_dir)
-      Pkg::Util::Execution.retry_on_fail(:times => 3) do
-        Pkg::Util::Net.rsync_to("#{local_dir}/#{Pkg::Config.ref}.yaml", Pkg::Config.distribution_server, "#{artifact_dir}/", extra_flags: ["--exclude repo_configs"])
-      end
-
-      # If we just shipped a tagged version, we want to make it immutable
-      files = Dir.glob("#{local_dir}/**/*").select { |f| File.file?(f) and !f.include? "#{Pkg::Config.ref}.yaml" }.map do |file|
-        "#{artifact_dir}/#{file.sub(/^#{local_dir}\//, '')}"
-      end
+      # building to.
+      Pkg::Util::Windows.add_msi_links(local_dir)
 
-      Pkg::Util::Net.remote_set_ownership(Pkg::Config.distribution_server, 'root', 'release', files)
-      Pkg::Util::Net.remote_set_permissions(Pkg::Config.distribution_server, '0664', files)
-      Pkg::Util::Net.remote_set_immutable(Pkg::Config.distribution_server, files)
+      # Send packages to the distribution server.
+      Pkg::Util::DistributionServer.send_packages(local_dir, artifact_dir)
     end
 
     desc 'Ship generated repository configs to the distribution server'

data/tasks/sign.rake

@@ -107,8 +107,8 @@ namespace :pl do
   ##
   # This crazy piece of work establishes a remote repo on the signing
   # server, ships our packages out to it, signs them, and brings them back.
-  #
   namespace :jenkins do
+    # The equivalent to invoking this task is calling Pkg::Util::Sign.sign_all(root_directory)
     desc "Sign all locally staged packages on #{Pkg::Config.signing_server}"
     task :sign_all, :root_dir do |_t, args|
       Pkg::Util::RakeUtils.invoke_task('pl:fetch')