packaging 0.88.77 → 0.99.0

data/tasks/ship.rake

@@ -5,39 +5,27 @@ namespace :pl do
     # to various target yum and apt repositories based on their specific type
     # e.g., final vs devel vs PE vs FOSS packages
 
-    desc "Update '#{Pkg::Config.repo_name}' yum repository on '#{Pkg::Config.yum_staging_server}'"
+    desc "Update remote yum repository on '#{Pkg::Config.yum_staging_server}'"
     task update_yum_repo: 'pl:fetch' do
-      command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository/Rakefile mk_repo'
-      $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => Pkg::Paths.yum_repo_name, :repo_path => Pkg::Config.yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
-    end
-
-    desc "Update all final yum repositories on '#{Pkg::Config.yum_staging_server}'"
-    task update_all_final_yum_repos: 'pl:fetch' do
-      command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository/Rakefile mk_repo'
-      $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => '', :repo_path => Pkg::Config.yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
-      end
-    end
-
-    desc "Update '#{Pkg::Config.nonfinal_repo_name}' nightly yum repository on '#{Pkg::Config.yum_staging_server}'"
-    task update_nightlies_yum_repo: 'pl:fetch' do
-      command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository-nightlies/Rakefile mk_repo'
-      $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => Pkg::Config.nonfinal_repo_name, :repo_path => Pkg::Config.nonfinal_yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
+      if Pkg::Util::Version.final?
+        path = Pkg::Config.yum_repo_path
+      else
+        path = Pkg::Config.nonfinal_yum_repo_path || Pkg::Config.yum_repo_path
       end
-    end
+      yum_whitelist = {
+        __REPO_NAME__: Pkg::Paths.repo_name,
+        __REPO_PATH__: path,
+        __REPO_HOST__: Pkg::Config.yum_staging_server,
+        __GPG_KEY__: Pkg::Util::Gpg.key
+      }
 
-    desc "Update all nightly yum repositories on '#{Pkg::Config.yum_staging_server}'"
-    task update_all_nightlies_yum_repos: 'pl:fetch' do
-      command = Pkg::Config.yum_repo_command || 'rake -f /opt/repository-nightlies/Rakefile mk_repo'
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.yum_staging_server}'? [y,n]"
       if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.yum_staging_server, command, { :repo_name => '', :repo_path => Pkg::Config.nonfinal_yum_repo_path, :repo_host => Pkg::Config.yum_staging_server })
+        if Pkg::Config.yum_repo_command
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_staging_server, Pkg::Util::Misc.search_and_replace(Pkg::Config.yum_repo_command, yum_whitelist))
+        else
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_staging_server, 'rake -f /opt/repository/Rakefile mk_repo')
+        end
       end
     end
 
@@ -45,17 +33,35 @@ namespace :pl do
 
     desc "Update remote apt repository on '#{Pkg::Config.apt_signing_server}'"
     task update_apt_repo: 'pl:fetch' do
-      $stdout.puts "Really run remote repo update on '#{Pkg::Config.apt_signing_server}'? [y,n]"
-      if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.apt_signing_server, Pkg::Config.apt_repo_command, { :repo_name => Pkg::Paths.apt_repo_name, :repo_path => Pkg::Config.apt_repo_path, :repo_host => Pkg::Config.apt_host, :repo_url => Pkg::Config.apt_repo_url })
-      end
-    end
+      if Pkg::Util::Version.final?
+        path = Pkg::Config.apt_repo_path
+        cmd = Pkg::Config.apt_repo_command
+      else
+        path = Pkg::Config.nonfinal_apt_repo_path || Pkg::Config.apt_repo_path
+        cmd = Pkg::Config.nonfinal_apt_repo_command || Pkg::Config.apt_repo_command
+      end
+      apt_whitelist = {
+        __REPO_NAME__: Pkg::Paths.repo_name,
+        __REPO_PATH__: path,
+        __REPO_URL__: Pkg::Config.apt_repo_url,
+        __REPO_HOST__: Pkg::Config.apt_host,
+        __APT_PLATFORMS__: Pkg::Config.apt_releases.join(' '),
+        __GPG_KEY__: Pkg::Util::Gpg.key
+      }
 
-    desc "Update nightlies apt repository on '#{Pkg::Config.apt_signing_server}'"
-    task update_nightlies_apt_repo: 'pl:fetch' do
       $stdout.puts "Really run remote repo update on '#{Pkg::Config.apt_signing_server}'? [y,n]"
       if Pkg::Util.ask_yes_or_no
-        Pkg::Repo.update_repo(Pkg::Config.apt_signing_server, Pkg::Config.nonfinal_apt_repo_command, { :repo_name => Pkg::Config.nonfinal_repo_name, :repo_path => Pkg::Config.nonfinal_apt_repo_path, :repo_host => Pkg::Config.apt_host, :repo_url => Pkg::Config.apt_repo_url })
+        if cmd
+          Pkg::Util::Net.remote_ssh_cmd(
+            Pkg::Config.apt_signing_server,
+            Pkg::Util::Misc.search_and_replace(
+              cmd,
+              apt_whitelist
+            )
+          )
+        else
+          warn %(Pkg::Config#apt_repo_command returned something unexpected, so no attempt will be made to update remote repos)
+        end
       end
     end
 
@@ -65,12 +71,6 @@ namespace :pl do
       Rake::Task['pl:remote:update_yum_repo'].invoke
     end
 
-    desc "Update nightlies apt and yum repos"
-    task :update_nightly_repos => "pl:fetch" do
-      Rake::Task['pl:remote:update_nightlies_apt_repo'].invoke
-      Rake::Task['pl:remote:update_nightlies_yum_repo'].invoke
-    end
-
     desc "Update remote ips repository on #{Pkg::Config.ips_host}"
     task :update_ips_repo  => 'pl:fetch' do
       if Dir['pkg/ips/pkgs/**/*'].empty? && Dir['pkg/solaris/11/**/*'].empty?
@@ -83,11 +83,7 @@ namespace :pl do
           source_dir = 'pkg/solaris/11/'
         end
 
-        tmpdir, _ = Pkg::Util::Net.remote_execute(
-                  Pkg::Config.ips_host,
-                  'mktemp -d -p /var/tmp',
-                  { capture_output: true }
-                )
+        tmpdir, _ = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, 'mktemp -d -p /var/tmp', true)
         tmpdir.chomp!
 
         Pkg::Util::Net.rsync_to(source_dir, Pkg::Config.ips_host, tmpdir)
@@ -96,9 +92,9 @@ namespace :pl do
       sudo pkgrecv -s $pkg -d #{Pkg::Config.ips_path} '*';
       done)
 
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, remote_cmd)
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
-        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, remote_cmd)
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
       end
     end
 
@@ -108,7 +104,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(times: 3) do
           cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.dmg_path, target_host: Pkg::Config.dmg_host, extra_flags: ['--update'])
-          Pkg::Util::Net.remote_execute(Pkg::Config.dmg_staging_server, cmd)
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.dmg_staging_server, cmd)
         end
       end
     end
@@ -119,7 +115,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(times: 3) do
           cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.swix_path, target_host: Pkg::Config.swix_host, extra_flags: ['--update'])
-          Pkg::Util::Net.remote_execute(Pkg::Config.swix_staging_server, cmd)
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.swix_staging_server, cmd)
         end
       end
     end
@@ -134,7 +130,7 @@ namespace :pl do
         else
           Pkg::Util::Execution.retry_on_fail(times: 3) do
             cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.tarball_path, target_host: Pkg::Config.tar_host, extra_flags: ['--update'])
-            Pkg::Util::Net.remote_execute(Pkg::Config.tar_staging_server, cmd)
+            Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.tar_staging_server, cmd)
           end
         end
       end
@@ -150,7 +146,7 @@ namespace :pl do
         else
           Pkg::Util::Execution.retry_on_fail(times: 3) do
             cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.msi_path, target_host: Pkg::Config.msi_host, extra_flags: ['--update'])
-            Pkg::Util::Net.remote_execute(Pkg::Config.msi_staging_server, cmd)
+            Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.msi_staging_server, cmd)
           end
         end
       end
@@ -178,7 +174,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh apt.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, command)
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_signing_server, command)
         end
       end
     end
@@ -204,7 +200,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh yum.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.yum_staging_server, command)
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_staging_server, command)
         end
       end
     end
@@ -215,7 +211,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh downloads.puppetlabs.com'
-          Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
+          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
         end
       end
     end
@@ -232,7 +228,7 @@ namespace :pl do
       puts "Deploying nightly builds from #{Pkg::Config.staging_server} to AWS S3..."
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         command = 'sudo /usr/local/bin/s3_repo_sync.sh nightlies.puppet.com'
-        Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
       end
     end
 
@@ -244,51 +240,52 @@ namespace :pl do
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           Pkg::Config.rsync_servers.each do |rsync_server|
             ['apt', 'yum'].each do |repo|
-              # Don't --delete so that folks using archived packages can continue to do so
-              command = "sudo su - rsync --command 'rsync --verbose -a --exclude '*.html' /opt/repo-s3-stage/repositories/#{repo}.puppetlabs.com/ rsync@#{rsync_server}:/opt/repository/#{repo}'"
-              Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
+              command = "sudo su - rsync --command 'rsync --verbose -a --exclude '*.html' --delete /opt/repo-s3-stage/repositories/#{repo}.puppetlabs.com/ rsync@#{rsync_server}:/opt/repository/#{repo}'"
+              Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
             end
           end
         end
       end
     end
-
-    desc "Remotely link nightly shipped gems to latest versions on #{Pkg::Config.gem_host}"
-    task link_nightly_shipped_gems_to_latest: 'pl:fetch' do
-      Pkg::Config.gemversion = Pkg::Util::Version.extended_dot_version
-
-      remote_path = Pkg::Config.nonfinal_gem_path
-      gems = FileList['pkg/*.gem'].map! { |path| path.gsub!('pkg/', '') }
-      command = %(cd #{remote_path}; )
-
-      command += gems.map! do |gem_name|
-        %(sudo ln -sf #{gem_name} #{gem_name.gsub(Pkg::Config.gemversion, 'latest')})
-      end.join(';')
-
-      command += %(; sync)
-
-      Pkg::Util::Net.remote_execute(Pkg::Config.gem_host, command)
-    end
   end
 
   desc "Ship mocked rpms to #{Pkg::Config.yum_staging_server}"
   task ship_rpms: 'pl:fetch' do
-    Pkg::Util::Ship.ship_rpms('pkg', Pkg::Config.yum_repo_path)
-  end
+    if Pkg::Util::Version.final?
+      path = Pkg::Config.yum_repo_path
+    else
+      path = Pkg::Config.nonfinal_yum_repo_path || Pkg::Config.yum_repo_path
+    end
+    Pkg::Util::Ship.ship_pkgs(['pkg/**/*.rpm', 'pkg/**/*.srpm'], Pkg::Config.yum_staging_server, path)
 
-  desc "Ship nightly rpms to #{Pkg::Config.yum_staging_server}"
-  task ship_nightly_rpms: 'pl:fetch' do
-    Pkg::Util::Ship.ship_rpms('pkg', Pkg::Config.nonfinal_yum_repo_path, nonfinal: true)
+    # I really don't care which one we grab, it just has to be some supported
+    # version and architecture from the `el` hash. So here we're just grabbing
+    # the first one, parsing out some info, and breaking out of the loop. Not
+    # elegant, I know, but effective.
+    Pkg::Platforms::PLATFORM_INFO['el'].each do |key, value|
+      generic_platform_tag = "el-#{key}-#{value[:architectures][0]}"
+      Pkg::Util::Ship.create_rolling_repo_link(generic_platform_tag, Pkg::Config.yum_staging_server, path)
+      break
+    end
   end
 
   desc "Ship cow-built debs to #{Pkg::Config.apt_signing_server}"
   task ship_debs: 'pl:fetch' do
-    Pkg::Util::Ship.ship_debs('pkg', Pkg::Config.apt_repo_staging_path, chattr: false)
-  end
+    if Pkg::Util::Version.final?
+      staging_path = Pkg::Config.apt_repo_staging_path
+    else
+      staging_path = Pkg::Config.nonfinal_apt_repo_staging_path || Pkg::Config.apt_repo_staging_path
+    end
+    Pkg::Util::Ship.ship_pkgs(['pkg/**/*.debian.tar.gz', 'pkg/**/*.orig.tar.gz', 'pkg/**/*.dsc', 'pkg/**/*.deb', 'pkg/**/*.changes'], Pkg::Config.apt_signing_server, staging_path, chattr: false)
 
-  desc "Ship nightly debs to #{Pkg::Config.apt_signing_server}"
-  task ship_nightly_debs: 'pl:fetch' do
-    Pkg::Util::Ship.ship_debs('pkg', Pkg::Config.nonfinal_apt_repo_staging_path, chattr: false, nonfinal: true)
+    # We need to iterate through all the supported platforms here because of
+    # how deb repos are set up. Each codename will have its own link from the
+    # current versioned repo (i.e., puppet5) to the rolling repo. The one thing
+    # we don't care about is architecture, so we just grab the first supported
+    # architecture for the codename we're working with at the moment.
+    Pkg::Platforms.codenames.each do |codename|
+      Pkg::Util::Ship.create_rolling_repo_link(Pkg::Platforms.codename_to_tags(codename)[0], Pkg::Config.apt_signing_server, staging_path)
+    end
   end
 
   desc 'Ship built gem to rubygems.org, internal Gem mirror, and public file server'
@@ -305,6 +302,7 @@ namespace :pl do
           puts "Do you want to start shipping the rubygem '#{gem_file}'?"
           next unless Pkg::Util.ask_yes_or_no
           Rake::Task['pl:ship_gem_to_rubygems'].execute(file: gem_file)
+          Rake::Task['pl:ship_gem_to_internal_mirror'].execute(file: gem_file)
         end
 
         Rake::Task['pl:ship_gem_to_downloads'].invoke
@@ -314,22 +312,6 @@ namespace :pl do
     end
   end
 
-  desc 'Ship built gem to internal Gem mirror and public nightlies file server'
-  task ship_nightly_gem: 'pl:fetch' do
-    # We want to ship a Gem only for projects that build gems, so
-    # all of the Gem shipping tasks are wrapped in an `if`.
-    if Pkg::Config.build_gem
-      fail 'Value `Pkg::Config.gem_host` not defined, skipping nightly ship' unless Pkg::Config.gem_host
-      fail 'Value `Pkg::Config.nonfinal_gem_path` not defined, skipping nightly ship' unless Pkg::Config.nonfinal_gem_path
-      FileList['pkg/*.gem'].each do |gem_file|
-        Pkg::Gem.ship_to_internal_mirror(gem_file)
-      end
-      Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Ship.ship_gem('pkg', Pkg::Config.nonfinal_gem_path, platform_independent: true)
-      end
-    end
-  end
-
   desc 'Ship built gem to rubygems.org'
   task :ship_gem_to_rubygems, [:file] => 'pl:fetch' do |_t, args|
     puts "Do you want to ship #{args[:file]} to rubygems.org?"
@@ -341,22 +323,50 @@ namespace :pl do
     end
   end
 
-  desc "Ship built gems to public Downloads server (#{Pkg::Config.gem_host})"
-  task :ship_gem_to_downloads => 'pl:fetch' do
-    if Pkg::Config.gem_host && Pkg::Config.gem_path
+  desc "Ship built gems to internal Gem server (#{Pkg::Config.internal_gem_host})"
+  task :ship_gem_to_internal_mirror, [:file] => 'pl:fetch' do |_t, args|
+    unless Pkg::Config.internal_gem_host
+      warn 'Value `Pkg::Config.internal_gem_host` not defined; skipping internal ship'
+    end
+
+    puts "Do you want to ship #{args[:file]} to the internal stickler server(#{Pkg::Config.internal_stickler_host})?"
+    if Pkg::Util.ask_yes_or_no
+      puts "Shipping gem #{args[:file]} to internal Gem server (#{Pkg::Config.internal_stickler_host})"
       Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Ship.ship_gem('pkg', Pkg::Config.gem_path, platform_independent: true)
+        Pkg::Gem.ship_to_stickler(args[:file])
       end
-    else
+    end
+
+    puts "Do you want to ship #{args[:file]} to the internal nexus server(#{Pkg::Config.internal_nexus_host})?"
+    if Pkg::Util.ask_yes_or_no
+      puts "Shipping gem #{args[:file]} to internal Gem server (#{Pkg::Config.internal_nexus_host})"
+      Pkg::Util::Execution.retry_on_fail(times: 3) do
+        Pkg::Gem.ship_to_nexus(args[:file])
+      end
+    end
+  end
+
+  desc "Ship built gems to public Downloads server (#{Pkg::Config.gem_host})"
+  task :ship_gem_to_downloads => 'pl:fetch' do
+    unless Pkg::Config.gem_host
       warn 'Value `Pkg::Config.gem_host` not defined; skipping shipping to public Download server'
     end
+
+    Pkg::Util::Execution.retry_on_fail(times: 3) do
+      Pkg::Util::Ship.ship_pkgs(['pkg/*.gem*'], Pkg::Config.gem_host, Pkg::Config.gem_path, platform_independent: true)
+    end
   end
 
   desc "Ship svr4 packages to #{Pkg::Config.svr4_host}"
   task :ship_svr4 do
     Pkg::Util::Execution.retry_on_fail(:times => 3) do
       if File.directory?("pkg/solaris/10")
-        Pkg::Util::Ship.ship_svr4('pkg', Pkg::Config.svr4_path)
+        if Pkg::Util::Version.final?
+          path = Pkg::Config.svr4_path
+        else
+          path = Pkg::Config.nonfinal_svr4_path || Pkg::Config.svr4_path
+        end
+        Pkg::Util::Ship.ship_pkgs(['pkg/**/*.pkg.gz'], Pkg::Config.svr4_host, path)
       end
     end
   end
@@ -365,39 +375,87 @@ namespace :pl do
   task :ship_p5p do
     Pkg::Util::Execution.retry_on_fail(:times => 3) do
       if File.directory?("pkg/solaris/11")
-        Pkg::Util::Ship.ship_p5p('pkg', Pkg::Config.p5p_path)
+        if Pkg::Util::Version.final?
+          path = Pkg::Config.p5p_path
+        else
+          path = Pkg::Config.nonfinal_p5p_path || Pkg::Config.p5p_path
+        end
+        Pkg::Util::Ship.ship_pkgs(['pkg/**/*.p5p'], Pkg::Config.p5p_host, path)
       end
     end
   end
 
   desc "ship apple dmg to #{Pkg::Config.dmg_staging_server}"
   task ship_dmg: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'dmg')
-    Pkg::Util::Ship.ship_dmg('pkg', path)
-  end
+    # TODO: realistically, this shouldn't be here. This block needs to be
+    # removed, but only when we can successfully modify all instances of
+    # this to be set to '/opt/downloads'. In the meantime, we need to write
+    # this terrible workaround to ensure backward compatibility.
+    #
+    # I'm so sorry
+    # ~MAS 2017-08-14
+    if Pkg::Config.dmg_path == "/opt/downloads/mac"
+      path = "/opt/downloads"
+    else
+      path = Pkg::Config.dmg_path
+    end
+    path = Pkg::Config.nonfinal_dmg_path if Pkg::Config.nonfinal_dmg_path && !Pkg::Util::Version.final?
 
-  desc "ship nightly apple dmgs to #{Pkg::Config.dmg_staging_server}"
-  task ship_nightly_dmg: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'dmg', nonfinal: true)
-    Pkg::Util::Ship.ship_dmg('pkg', path, nonfinal: true)
+    Pkg::Util::Ship.ship_pkgs(['pkg/**/*.dmg'], Pkg::Config.dmg_staging_server, path)
+
+    # I really don't care which one we grab, it just has to be some supported
+    # version and architecture from the `osx` hash. So here we're just grabbing
+    # the first one, parsing out some info, and breaking out of the loop. Not
+    # elegant, I know, but effective.
+    Pkg::Platforms::PLATFORM_INFO['osx'].each do |key, value|
+      generic_platform_tag = "osx-#{key}-#{value[:architectures][0]}"
+      Pkg::Util::Ship.create_rolling_repo_link(generic_platform_tag, Pkg::Config.dmg_staging_server, path)
+      break
+    end
+
+    Pkg::Platforms.platform_tags_for_package_format('dmg').each do |platform_tag|
+      # TODO remove the PC1 links when we no longer need to maintain them
+      _, version, arch = Pkg::Platforms.parse_platform_tag(platform_tag)
+      Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', "/opt/downloads/mac/#{version}/PC1/#{arch}", 'dmg')
+
+      # Create the latest symlink for the current supported repo
+      Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(platform_tag, path), 'dmg')
+    end
   end
 
   desc "ship Arista EOS swix packages and signatures to #{Pkg::Config.swix_staging_server}"
   task ship_swix: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'swix')
-    Pkg::Util::Ship.ship_swix('pkg', path)
-  end
+    # TODO: realistically, this shouldn't be here. This block needs to be
+    # removed, but only when we can successfully modify all instances of
+    # this to be set to '/opt/downloads'. In the meantime, we need to write
+    # this terrible workaround to ensure backward compatibility.
+    #
+    # I'm so sorry
+    # ~MAS 2017-08-14
+    if Pkg::Config.swix_path == "/opt/downloads/eos"
+      path = "/opt/downloads"
+    else
+      path = Pkg::Config.swix_path
+    end
+    path = Pkg::Config.nonfinal_swix_path if Pkg::Config.nonfinal_swix_path && !Pkg::Util::Version.final?
+
+    Pkg::Util::Ship.ship_pkgs(['pkg/**/*.swix*'], Pkg::Config.swix_staging_server, path)
 
-  desc "ship nightly Arista EOS swix packages and signatures to #{Pkg::Config.swix_staging_server}"
-  task ship_nightly_swix: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'swix', nonfinal: true)
-    Pkg::Util::Ship.ship_swix('pkg', path, nonfinal: true)
+    # I really don't care which one we grab, it just has to be some supported
+    # version and architecture from the `eos` hash. So here we're just grabbing
+    # the first one, parsing out some info, and breaking out of the loop. Not
+    # elegant, I know, but effective.
+    Pkg::Platforms::PLATFORM_INFO['eos'].each do |key, value|
+      generic_platform_tag = "eos-#{key}-#{value[:architectures][0]}"
+      Pkg::Util::Ship.create_rolling_repo_link(generic_platform_tag, Pkg::Config.swix_staging_server, path)
+      break
+    end
   end
 
   desc "ship tarball and signature to #{Pkg::Config.tar_staging_server}"
   task ship_tar: 'pl:fetch' do
     if Pkg::Config.build_tar
-      Pkg::Util::Ship.ship_tar('pkg', Pkg::Config.tarball_path, excludes: ['signing_bundle', 'packaging-bundle'], platform_independent: true)
+      Pkg::Util::Ship.ship_pkgs(['pkg/*.tar.gz*'], Pkg::Config.tar_staging_server, Pkg::Config.tarball_path, excludes: ['signing_bundle', 'packaging-bundle'], platform_independent: true)
     end
   end
 
@@ -413,19 +471,41 @@ namespace :pl do
 
   desc "Ship MSI packages to #{Pkg::Config.msi_staging_server}"
   task ship_msi: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'msi')
-    Pkg::Util::Ship.ship_msi('pkg', path, excludes: ["#{Pkg::Config.project}-x(86|64).msi"])
-  end
+    # TODO: realistically, this shouldn't be here. This block needs to be
+    # removed, but only when we can successfully modify all instances of
+    # this to be set to '/opt/downloads'. In the meantime, we need to write
+    # this terrible workaround to ensure backward compatibility.
+    #
+    # I'm so sorry
+    # ~MAS 2017-08-14
+    if Pkg::Config.msi_path == "/opt/downloads/windows"
+      path = "/opt/downloads"
+    else
+      path = Pkg::Config.msi_path
+    end
+    path = Pkg::Config.nonfinal_msi_path if Pkg::Config.nonfinal_msi_path && !Pkg::Util::Version.final?
 
-  desc "Ship nightly MSI packages to #{Pkg::Config.msi_staging_server}"
-  task ship_nightly_msi: 'pl:fetch' do
-    path = Pkg::Paths.remote_repo_base(package_format: 'msi', nonfinal: true)
-    Pkg::Util::Ship.ship_msi('pkg', path, excludes: ["#{Pkg::Config.project}-x(86|64).msi"], nonfinal: true)
-  end
+    Pkg::Util::Ship.ship_pkgs(['pkg/**/*.msi'], Pkg::Config.msi_staging_server, path, excludes: ["#{Pkg::Config.project}-x(86|64).msi"])
 
-  desc "Add #{Pkg::Config.project} version #{Pkg::Config.ref} to release-metrics"
-  task :update_release_metrics => "pl:fetch" do
-    Pkg::Metrics.update_release_metrics
+    # I really don't care which one we grab, it just has to be some supported
+    # version and architecture from the `windows` hash. So here we're just grabbing
+    # the first one, parsing out some info, and breaking out of the loop. Not
+    # elegant, I know, but effective.
+    Pkg::Platforms::PLATFORM_INFO['windows'].each do |key, value|
+      generic_platform_tag = "windows-#{key}-#{value[:architectures][0]}"
+      Pkg::Util::Ship.create_rolling_repo_link(generic_platform_tag, Pkg::Config.msi_staging_server, path)
+
+      # Create the symlinks for the latest supported repo
+      Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(generic_platform_tag, path), 'msi', arch: 'x64')
+      Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', Pkg::Paths.artifacts_path(generic_platform_tag, path), 'msi', arch: 'x86')
+      break
+    end
+
+    # We provide symlinks to the latest package in a given directory. This
+    # allows users to upgrade more easily to the latest version that we release
+    # TODO remove the links to PC1 when we no longer ship to that repo
+    Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', '/opt/downloads/windows', 'msi', arch: 'x64')
+    Pkg::Util::Net.remote_create_latest_symlink('puppet-agent', '/opt/downloads/windows', 'msi', arch: 'x86')
   end
 
   desc 'UBER ship: ship all the things in pkg'
@@ -448,16 +528,6 @@ namespace :pl do
     end
   end
 
-  desc 'Create the rolling repo links'
-  task create_repo_links: 'pl:fetch' do
-    Pkg::Util::Ship.create_rolling_repo_links
-  end
-
-  desc 'Create rolling repo links for nightlies'
-  task create_nightly_repo_links: 'pl:fetch' do
-    Pkg::Util::Ship.create_rolling_repo_links(true)
-  end
-
   desc 'Test out the ship requirements'
   task ship_check: 'pl:fetch' do
     errs = []
@@ -503,17 +573,29 @@ namespace :pl do
     # Check for ability to sign OSX. Should just need to be able to unlock keychain
     begin
       unless ssh_errs.include?(Pkg::Config.osx_signing_server)
-        Pkg::Util::Net.remote_execute(
-          Pkg::Config.osx_signing_server,
-          %(/usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}"),
-          { extra_options: '-oBatchMode=yes' }
-        )
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.osx_signing_server, %(/usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}"), false, '-oBatchMode=yes')
       end
     rescue
       errs << "Unlocking the OSX keychain failed! Check the password in your .bashrc on #{Pkg::Config.osx_signing_server}"
     end
 
     if Pkg::Config.build_gem
+      # Do we have stickler and nexus?
+      if Pkg::Util::Misc.check_gem('stickler')
+        `stickler list --server #{Pkg::Config.internal_stickler_host} > /dev/null 2>&1`
+        unless $CHILD_STATUS.zero?
+          errs << "Listing gems at the stickler server #{Pkg::Config.internal_stickler_host} failed!"
+        end
+      else
+        errs << 'gem stickler not found'
+      end
+
+      errs << 'gem nexus not found' unless Pkg::Util::Misc.check_gem('nexus')
+      `gem list --source #{Pkg::Config.internal_nexus_host} > /dev/null 2>&1`
+      unless $CHILD_STATUS.zero?
+        errs << "Listing gems at the nexus server #{Pkg::Config.internal_nexus_host} failed!"
+      end
+
       # Do we have rubygems access set up
       if Pkg::Util::File.file_exists?("#{ENV['HOME']}/.gem/credentials")
         # Do we have permissions to publish this gem on rubygems
@@ -534,7 +616,6 @@ namespace :pl do
         puts " * #{err}"
       end
     end
-
   end
 
   # It is odd to namespace this ship task under :jenkins, but this task is
@@ -547,29 +628,17 @@ namespace :pl do
     desc 'ship pkg directory contents to artifactory'
     task :ship_to_artifactory, :local_dir do |_t, args|
       Pkg::Util::RakeUtils.invoke_task('pl:fetch')
-      unless Pkg::Config.project
-        fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
-      end
       artifactory = Pkg::ManageArtifactory.new(Pkg::Config.project, Pkg::Config.ref)
 
       local_dir = args.local_dir || 'pkg'
       Dir.glob("#{local_dir}/**/*").reject { |e| File.directory? e }.each do |artifact|
-        if File.extname(artifact) == ".yaml" || File.extname(artifact) == ".json"
-          artifactory.deploy_package(artifact)
-        elsif artifactory.package_exists_on_artifactory?(artifact)
-          warn "Attempt to upload '#{artifact}' failed. Package already exists!"
-        else
-          artifactory.deploy_package(artifact)
-        end
+        artifactory.deploy_package(artifact)
       end
     end
 
     desc 'Ship pkg directory contents to distribution server'
     task :ship, :target, :local_dir do |_t, args|
       Pkg::Util::RakeUtils.invoke_task('pl:fetch')
-      unless Pkg::Config.project
-        fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
-      end
       target = args.target || 'artifacts'
       local_dir = args.local_dir || 'pkg'
       project_basedir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
@@ -589,14 +658,12 @@ namespace :pl do
         cp(ezbake_yaml, File.join(local_dir, "#{Pkg::Config.ref}.ezbake.manifest.yaml"))
       end
 
-      # Inside build_metadata*.json files there is additional metadata containing
+      # We are starting to collect additional metadata which contains
       # information such as git ref and dependencies that are needed at build
-      # time. If these files exist, copy them downstream.
-      # Typically these files are named 'ext/build_metadata.<project>.<platform>.json'
-      build_metadata_json_files = Dir.glob('ext/build_metadata*.json')
-      build_metadata_json_files.each do |source_file|
-        target_file = File.join(local_dir, "#{Pkg::Config.ref}.#{File.basename(source_file)}")
-        cp(source_file, target_file)
+      # time. If this file exists we will make it available for downstream.
+      build_data_json = File.join("ext", "build_metadata.json")
+      if File.exists?(build_data_json)
+        cp(build_data_json, File.join(local_dir, "#{Pkg::Config.ref}.build_metadata.json"))
       end
 
       # Sadly, the packaging repo cannot yet act on its own, without living
@@ -636,32 +703,25 @@ namespace :pl do
       # building to. Once we move the Metadata about the output location in
       # to one source of truth we can refactor this to use that to search
       #                                           -Sean P. M. 08/12/16
-
-      {
-        'windows' => ['x86', 'x64'],
-        'windowsfips' => ['x64']
-      }.each_pair do |platform, archs|
-        packages = Dir["#{local_dir}/#{platform}/*"]
-
-        archs.each do |arch|
-          package_version = Pkg::Util::Git.describe.tr('-', '.')
-          package_filename = File.join(local_dir, platform, "#{Pkg::Config.project}-#{package_version}-#{arch}.msi")
-          link_filename = File.join(local_dir, platform, "#{Pkg::Config.project}-#{arch}.msi")
-
-          next unless !packages.include?(link_filename) && packages.include?(package_filename)
-          # Dear future code spelunkers:
-          # Using symlinks instead of hard links causes failures when we try
-          # to set these files to be immutable. Also be wary of whether the
-          # linking utility you're using expects the source path to be relative
-          # to the link target or pwd.
-          #
-          FileUtils.ln(package_filename, link_filename)
-        end
+      packages = Dir["#{local_dir}/windows/*"]
+      ['x86', 'x64'].each do |arch|
+        package_version = Pkg::Util::Git.describe.tr('-', '.')
+        package_filename = File.join(local_dir, 'windows', "#{Pkg::Config.project}-#{package_version}-#{arch}.msi")
+        link_filename = File.join(local_dir, 'windows', "#{Pkg::Config.project}-#{arch}.msi")
+
+        next unless !packages.include?(link_filename) && packages.include?(package_filename)
+        # Dear future code spelunkers:
+        # Using symlinks instead of hard links causes failures when we try
+        # to set these files to be immutable. Also be wary of whether the
+        # linking utility you're using expects the source path to be relative
+        # to the link target or pwd.
+        #
+        FileUtils.ln(package_filename, link_filename)
       end
 
       Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
-        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
         Pkg::Util::Net.rsync_to("#{local_dir}/", Pkg::Config.distribution_server, "#{artifact_dir}/", extra_flags: ['--ignore-existing', '--exclude repo_configs'])
       end