packaging 0.88.77 → 0.99.0

data/tasks/gem.rake

@@ -1,159 +1,146 @@
-require 'rubygems/package_task'
-
-def glob_gem_files
-  gem_files = []
-  gem_excludes_file_list = []
-  gem_excludes_raw = Pkg::Config.gem_excludes.nil? ? [] : Pkg::Config.gem_excludes.split(' ')
-  gem_excludes_raw << 'ext/packaging'
-  gem_excludes_raw << 'pkg'
-  gem_excludes_raw.each do |exclude|
-    if File.directory?(exclude)
-      gem_excludes_file_list += FileList["#{exclude}/**/*"]
-    else
-      gem_excludes_file_list << exclude
+if Pkg::Config.build_gem
+  require 'rubygems/package_task'
+
+  def glob_gem_files
+    gem_files = []
+    gem_excludes_file_list = []
+    gem_excludes_raw = Pkg::Config.gem_excludes.nil? ? [] : Pkg::Config.gem_excludes.split(' ')
+    gem_excludes_raw << 'ext/packaging'
+    gem_excludes_raw << 'pkg'
+    gem_excludes_raw.each do |exclude|
+      if File.directory?(exclude)
+        gem_excludes_file_list += FileList["#{exclude}/**/*"]
+      else
+        gem_excludes_file_list << exclude
+      end
     end
-  end
-  files = FileList[Pkg::Config.gem_files.split(' ')]
-  files.each do |file|
-    if File.directory?(file)
-      gem_files += FileList["#{file}/**/*"]
-    else
-      gem_files << file
+    files = FileList[Pkg::Config.gem_files.split(' ')]
+    files.each do |file|
+      if File.directory?(file)
+        gem_files += FileList["#{file}/**/*"]
+      else
+        gem_files << file
+      end
     end
+    gem_files -= gem_excludes_file_list
   end
-  gem_files -= gem_excludes_file_list
-end
-
-def add_gem_dependency(opts = {})
-  spec = opts[:spec]
-  version = opts[:version]
-  type = opts[:type].to_s
-  gem = opts[:gem].to_s
-  if opts[:version].nil? or opts[:version].empty?
-    spec.send("add_#{type}_dependency".to_sym, gem)
-  else
-    spec.send("add_#{type}_dependency".to_sym, gem, version)
-  end
-  spec
-end
 
-def create_default_gem_spec
-  spec = Gem::Specification.new do |s|
-    s.name = Pkg::Config.project                                     unless Pkg::Config.project.nil?
-    s.name = Pkg::Config.gem_name                                    unless Pkg::Config.gem_name.nil?
-    s.version = Pkg::Config.gemversion                               unless Pkg::Config.gemversion.nil?
-    s.author = Pkg::Config.author                                    unless Pkg::Config.author.nil?
-    s.email = Pkg::Config.email                                      unless Pkg::Config.email.nil?
-    s.homepage = Pkg::Config.homepage                                unless Pkg::Config.homepage.nil?
-    s.summary = Pkg::Config.summary                                  unless Pkg::Config.summary.nil?
-    s.summary = Pkg::Config.gem_summary                              unless Pkg::Config.gem_summary.nil?
-    s.description = Pkg::Config.description                          unless Pkg::Config.description.nil?
-    s.description = Pkg::Config.gem_description                      unless Pkg::Config.gem_description.nil?
-    s.files = glob_gem_files                                         unless glob_gem_files.nil?
-    s.executables = Pkg::Config.gem_executables                      unless Pkg::Config.gem_executables.nil?
-    s.require_path = Pkg::Config.gem_require_path                    unless Pkg::Config.gem_require_path.nil?
-    s.required_ruby_version = Pkg::Config.gem_required_ruby_version  unless Pkg::Config.gem_required_ruby_version.nil?
-    s.required_rubygems_version = Pkg::Config.gem_required_rubygems_version unless Pkg::Config.gem_required_rubygems_version.nil?
-    s.test_files = FileList[Pkg::Config.gem_test_files.split(' ')]   unless Pkg::Config.gem_test_files.nil?
-    s.rubyforge_project = Pkg::Config.gem_forge_project              unless Pkg::Config.gem_forge_project.nil?
-    Pkg::Config.gem_rdoc_options.each do |option|
-      s.rdoc_options << option
-    end unless Pkg::Config.gem_rdoc_options.nil?
+  def add_gem_dependency(opts = {})
+    spec = opts[:spec]
+    version = opts[:version]
+    type = opts[:type].to_s
+    gem = opts[:gem].to_s
+    if opts[:version].nil? or opts[:version].empty?
+      spec.send("add_#{type}_dependency".to_sym, gem)
+    else
+      spec.send("add_#{type}_dependency".to_sym, gem, version)
+    end
+    spec
   end
 
-  Pkg::Config.gem_runtime_dependencies.each do |gem, version|
-    spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => :runtime)
-  end unless Pkg::Config.gem_runtime_dependencies.nil?
-
-  Pkg::Config.gem_development_dependencies.each do |gem, version|
-    spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => :development)
-  end unless Pkg::Config.gem_development_dependencies.nil?
-  spec
-end
-
-def create_gem(spec, gembuilddir)
-  gem_files = glob_gem_files + FileList[(Pkg::Config.gem_test_files || '').split(' ')]
-  workdir = File.join(Pkg::Util::File.mktemp)
+  def create_default_gem_spec
+    spec = Gem::Specification.new do |s|
+      s.name = Pkg::Config.project                                     unless Pkg::Config.project.nil?
+      s.name = Pkg::Config.gem_name                                    unless Pkg::Config.gem_name.nil?
+      s.version = Pkg::Config.gemversion                               unless Pkg::Config.gemversion.nil?
+      s.author = Pkg::Config.author                                    unless Pkg::Config.author.nil?
+      s.email = Pkg::Config.email                                      unless Pkg::Config.email.nil?
+      s.homepage = Pkg::Config.homepage                                unless Pkg::Config.homepage.nil?
+      s.summary = Pkg::Config.summary                                  unless Pkg::Config.summary.nil?
+      s.summary = Pkg::Config.gem_summary                              unless Pkg::Config.gem_summary.nil?
+      s.description = Pkg::Config.description                          unless Pkg::Config.description.nil?
+      s.description = Pkg::Config.gem_description                      unless Pkg::Config.gem_description.nil?
+      s.files = glob_gem_files                                         unless glob_gem_files.nil?
+      s.executables = Pkg::Config.gem_executables                      unless Pkg::Config.gem_executables.nil?
+      s.require_path = Pkg::Config.gem_require_path                    unless Pkg::Config.gem_require_path.nil?
+      s.required_ruby_version = Pkg::Config.gem_required_ruby_version  unless Pkg::Config.gem_required_ruby_version.nil?
+      s.required_rubygems_version = Pkg::Config.gem_required_rubygems_version unless Pkg::Config.gem_required_rubygems_version.nil?
+      s.test_files = FileList[Pkg::Config.gem_test_files.split(' ')]   unless Pkg::Config.gem_test_files.nil?
+      s.rubyforge_project = Pkg::Config.gem_forge_project              unless Pkg::Config.gem_forge_project.nil?
+      Pkg::Config.gem_rdoc_options.each do |option|
+        s.rdoc_options << option
+      end unless Pkg::Config.gem_rdoc_options.nil?
+    end
 
-  bench = Benchmark.realtime do
-    Pkg::Util::File.install_files_into_dir(gem_files, workdir)
+    Pkg::Config.gem_runtime_dependencies.each do |gem, version|
+      spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => :runtime)
+    end unless Pkg::Config.gem_runtime_dependencies.nil?
 
-    cd workdir do
-      gem_task = Gem::PackageTask.new(spec)
-      gem_task.define
-      Rake::Task[:gem].reenable
-      Rake::Task[:gem].invoke
-      rm_rf File.join("pkg", gembuilddir)
-      mv Dir.glob("pkg/#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}*.gem"), File.join(Pkg::Config.project_root, "pkg")
-    end
+    Pkg::Config.gem_development_dependencies.each do |gem, version|
+      spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => :development)
+    end unless Pkg::Config.gem_development_dependencies.nil?
+    spec
   end
 
-  rm_rf workdir
-  puts "Finished building in: #{bench}"
-end
+  def create_gem(spec, gembuilddir)
+    gem_files = glob_gem_files + FileList[(Pkg::Config.gem_test_files || '').split(' ')]
+    workdir = File.join(Pkg::Util::File.mktemp)
 
-def create_default_gem
-  spec = create_default_gem_spec
-  create_gem(spec, "#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}")
-end
-
-def unknown_gems_platform?(platform)
-  return true if platform.os == "unknown"
-  false
-end
+    bench = Benchmark.realtime do
+      Pkg::Util::File.install_files_into_dir(gem_files, workdir)
 
-def create_platform_specific_gems
-  Pkg::Config.gem_platform_dependencies.each do |platform, dependency_hash|
-    spec = create_default_gem_spec
-    pf = Gem::Platform.new(platform)
-    fail "
-      Platform: '#{platform}' is not recognized by rubygems.
-      This is probably an erroneous 'gem_platform_dependencies' entry!" if unknown_gems_platform?(pf)
-    spec.platform = pf
-    dependency_hash.each do |type, gems|
-      t = case type
-      when "gem_runtime_dependencies"
-        "runtime"
-      when "gem_development_dependencies"
-        "development"
-      else
-        fail "Platform specific gem dependency type must be 'gem_runtime_dependencies' or 'gem_development_dependencies', not '#{type}'"
-      end
-      gems.each do |gem, version|
-        spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => t)
+      cd workdir do
+        gem_task = Gem::PackageTask.new(spec)
+        gem_task.define
+        Rake::Task[:gem].reenable
+        Rake::Task[:gem].invoke
+        rm_rf File.join("pkg", gembuilddir)
+        mv Dir.glob("pkg/#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}*.gem"), File.join(Pkg::Config.project_root, "pkg")
       end
     end
-    create_gem(spec, "#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}-#{platform}")
+
+    rm_rf workdir
+    puts "Finished building in: #{bench}"
   end
-end
 
-def package_gem
-  unless Pkg::Config.build_gem
-    puts "The #{Pkg::Config.project} project does not build any gems! Passing through ..."
-    return
+  def create_default_gem
+    spec = create_default_gem_spec
+    create_gem(spec, "#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}")
   end
 
-  mkdir_p File.join(Pkg::Config.project_root, "pkg")
-  create_default_gem
-  if Pkg::Config.gem_platform_dependencies
-    create_platform_specific_gems
+  def unknown_gems_platform?(platform)
+    return true if platform.os == "unknown"
+    false
   end
-end
 
-namespace :package do
-  desc "Build a gem - All gems if platform specific"
-  task :gem => ["clean"] do
-    package_gem
+  def create_platform_specific_gems
+    Pkg::Config.gem_platform_dependencies.each do |platform, dependency_hash|
+      spec = create_default_gem_spec
+      pf = Gem::Platform.new(platform)
+      fail "
+        Platform: '#{platform}' is not recognized by rubygems.
+        This is probably an erroneous 'gem_platform_dependencies' entry!" if unknown_gems_platform?(pf)
+      spec.platform = pf
+      dependency_hash.each do |type, gems|
+        t = case type
+        when "gem_runtime_dependencies"
+          "runtime"
+        when "gem_development_dependencies"
+          "development"
+        else
+          fail "Platform specific gem dependency type must be 'gem_runtime_dependencies' or 'gem_development_dependencies', not '#{type}'"
+        end
+        gems.each do |gem, version|
+          spec = add_gem_dependency(:spec => spec, :gem => gem, :version => version, :type => t)
+        end
+      end
+      create_gem(spec, "#{Pkg::Config.gem_name}-#{Pkg::Config.gemversion}-#{platform}")
+    end
   end
 
-  task :nightly_gem => ["clean"] do
-    Pkg::Config.gemversion = Pkg::Util::Version.extended_dot_version
-    package_gem
+  namespace :package do
+    desc "Build a gem - All gems if platform specific"
+    task :gem => ["clean"] do
+      mkdir_p File.join(Pkg::Config.project_root, "pkg")
+      create_default_gem
+      if Pkg::Config.gem_platform_dependencies
+        create_platform_specific_gems
+      end
+    end
   end
-end
 
-# An alias task to simplify our remote logic in jenkins.rake
-namespace :pl do
-  task :gem => "package:gem"
-  task :nightly_gem => "package:nightly_gem"
+  # An alias task to simplify our remote logic in jenkins.rake
+  namespace :pl do
+    task :gem => "package:gem"
+  end
 end

data/tasks/jenkins.rake

@@ -52,14 +52,14 @@
 #
 #     pushd git_repo
 #
-#     ### Install the packaging gem via Bundler
-#     bundle install
+#     ### Clone the packaging repo
+#     rake package:bootstrap
 #
 #     ### Perform the build
-#     bundle exec rake pl:build_from_params PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
+#     rake pl:build_from_params PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
 #
 #     ### Send the results
-#     bundle exec rake pl:jenkins:ship["artifacts"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
+#     rake pl:jenkins:ship["artifacts"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
 #
 #   popd
 # popd
@@ -75,7 +75,7 @@
 # if [ -n "$DOWNSTREAM_JOB" ] ; then
 #   pushd project
 #     pushd git_repo
-#       bundle exec rake pl:jenkins:post["$DOWNSTREAM_JOB"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
+#       rake pl:jenkins:post["$DOWNSTREAM_JOB"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
 #     popd
 #   popd
 # fi
@@ -257,33 +257,7 @@ namespace :pl do
         ship_debs
         ship_dmg
         ship_swix
-        ship_tar
         ship_msi
-        ship_gem
-      )
-      tasks.map { |t| "pl:#{t}" }.each do |t|
-        puts "Running #{t} . . ."
-        Rake::Task[t].invoke
-      end
-      # mark the build as successfully shipped
-      Rake::Task["pl:jenkins:ship"].invoke("shipped")
-      # add the release to release-metrics
-      begin
-        Rake::Task["pl:update_release_metrics"].invoke
-      rescue => e
-        fail "Error updating release-metrics:\n#{e}\nYou will need to add this release manually."
-      end
-    end
-
-    task :stage_nightlies => "pl:fetch" do
-      tasks = %w(
-        jenkins:retrieve
-        jenkins:sign_all
-        ship_nightly_rpms
-        ship_nightly_debs
-        ship_nightly_dmg
-        ship_nightly_swix
-        ship_nightly_msi
       )
       tasks.map { |t| "pl:#{t}" }.each do |t|
         puts "Running #{t} . . ."
@@ -292,8 +266,8 @@ namespace :pl do
     end
 
     task :ship_nightlies => "pl:fetch" do
-      Rake::Task['pl:jenkins:stage_nightlies'].invoke
-      Rake::Task['pl:remote:update_nightly_repos'].invoke
+      Rake::Task['pl:jenkins:uber_ship_lite'].invoke
+      Rake::Task['pl:remote:update_foss_repos'].invoke
       Rake::Task['pl:remote:deploy_nightlies_to_s3'].invoke
     end
 
@@ -304,24 +278,6 @@ namespace :pl do
       Rake::Task['pl:remote:deploy_to_rsync_server'].invoke
     end
 
-    task :stage_release_packages => "pl:fetch" do
-      Rake::Task['pl:jenkins:uber_ship_lite'].invoke
-      # Deb packages only appear in the freight directory until repo updates.
-      # We must run that before creating symlinks so we can link from packages
-      # in the apt repository.
-      Rake::Task['pl:remote:update_apt_repo'].invoke
-      Pkg::Util::Ship.update_release_package_symlinks('pkg')
-    end
-
-    task :stage_nightly_release_packages => "pl:fetch" do
-      Rake::Task['pl:jenkins:stage_nightlies'].invoke
-      # Deb packages only appear in the freight directory until repo updates.
-      # We must run that before creating symlinks so we can link from packages
-      # in the apt repository.
-      Rake::Task['pl:remote:update_nightlies_apt_repo'].invoke
-      Pkg::Util::Ship.update_release_package_symlinks('pkg', true)
-    end
-
     desc "Retrieve packages built by jenkins, sign, and ship all!"
     task :uber_ship => "pl:fetch" do
       uber_tasks = %w(

data/tasks/nightly_repos.rake

@@ -18,18 +18,13 @@ namespace :pl do
       Pkg::Util::File.empty_dir?("repos") and fail "There were no repos found in repos/. Maybe something in the pipeline failed?"
       signing_bundle = ENV['SIGNING_BUNDLE']
 
-      remote_repo   = Pkg::Util::Net.remote_unpack_git_bundle(signing_server, 'HEAD', nil, signing_bundle)
+      remote_repo   = Pkg::Util::Net.remote_bootstrap(signing_server, 'HEAD', nil, signing_bundle)
       build_params  = Pkg::Util::Net.remote_buildparams(signing_server, Pkg::Config)
       Pkg::Util::Net.rsync_to('repos', signing_server, remote_repo)
-      rake_command = <<-DOC
-cd #{remote_repo} ;
-#{Pkg::Util::Net.remote_bundle_install_command}
-bundle exec rake pl:jenkins:sign_repos GPG_KEY=#{Pkg::Util::Gpg.key} PARAMS_FILE=#{build_params}
-DOC
-      Pkg::Util::Net.remote_execute(signing_server, rake_command)
+      Pkg::Util::Net.remote_ssh_cmd(signing_server, "cd #{remote_repo} ; rake pl:jenkins:sign_repos GPG_KEY=#{Pkg::Util::Gpg.key} PARAMS_FILE=#{build_params}")
       Pkg::Util::Net.rsync_from("#{remote_repo}/repos/", signing_server, target)
-      Pkg::Util::Net.remote_execute(signing_server, "rm -rf #{remote_repo}")
-      Pkg::Util::Net.remote_execute(signing_server, "rm #{build_params}")
+      Pkg::Util::Net.remote_ssh_cmd(signing_server, "rm -rf #{remote_repo}")
+      Pkg::Util::Net.remote_ssh_cmd(signing_server, "rm #{build_params}")
       puts "Signed packages staged in '#{target}' directory"
     end
 
@@ -38,9 +33,9 @@ DOC
       Pkg::Rpm::Repo.create_local_repos('repos')
       Pkg::Rpm::Repo.sign_repos('repos')
       Pkg::Deb::Repo.sign_repos('repos', 'Apt repository for signed builds')
-      Pkg::Sign::Dmg.sign('repos') unless Dir['repos/apple/**/*.dmg'].empty?
-      Pkg::Sign::Ips.sign('repos') unless Dir['repos/solaris/11/**/*.p5p'].empty?
-      Pkg::Sign::Msi.sign('repos') unless Dir['repos/windows/**/*.msi'].empty?
+      Pkg::OSX.sign('repos') unless Dir['repos/apple/**/*.dmg'].empty?
+      Pkg::IPS.sign('repos') unless Dir['repos/solaris/11/**/*.p5p'].empty?
+      Pkg::MSI.sign('repos') unless Dir['repos/windows/**/*.msi'].empty?
     end
 
     task :ship_signed_repos, [:target_prefix] => "pl:fetch" do |t, args|
@@ -48,7 +43,7 @@ DOC
       target_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target_prefix}_repos"
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         # Ship the now signed repos to the distribution server
-        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{target_dir}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{target_dir}")
         Pkg::Util::Net.rsync_to("#{target_prefix}_repos/", Pkg::Config.distribution_server, target_dir)
       end
     end
@@ -68,6 +63,60 @@ DOC
       Pkg::Repo.create_all_repo_archives(name_of_archive, versioning)
     end
 
+    # This is pretty similar to the 'pack_signed_repo' task. The difference here is that instead
+    # of creating a tarball for each repo passed, it adds each repo to a single archive, creating
+    # one 'all' tarball with all of the repos. This is useful for cutomers who have a PE master with
+    # no internet access. They can unpack the puppet-agent-all tarball into the location that
+    # pe_repo expects and use simplified agent install without needing internet access, or having to
+    # manually download each agent that they need to feed to pe_repo.
+    # This task should be invoked after prepare_signed_repos, so that there are repos to pack up.
+    task :pack_all_signed_repos, [:path_to_repo, :name_of_archive, :versioning] => ["pl:fetch"] do |t, args|
+      # path_to_repo should be relative to ./pkg
+      name_of_archive = args.name_of_archive or fail ":name_of_archive is a required argument for #{t}"
+      versioning = args.versioning or fail ":versioning is a required argument for #{t}"
+      tar = Pkg::Util::Tool.check_tool('tar')
+
+      Dir.chdir("pkg") do
+        if versioning == 'ref'
+          local_target = File.join(Pkg::Config.project, Pkg::Config.ref, "repos")
+        elsif versioning == 'version'
+          local_target = File.join(Pkg::Config.project, Pkg::Util::Version.dot_version, "repos")
+        end
+
+        Dir.chdir(local_target) do
+          if !Pkg::Util::File.exist?("#{name_of_archive}.tar.gz")
+            warn "Skipping #{name_of_archive} because it (#{name_of_archive}.tar.gz) has no files"
+          else
+            if File.exist?("#{Pkg::Config.project}-all.tar")
+              tar_cmd = "--update"
+            else
+              tar_cmd = "--create"
+            end
+            Pkg::Util::Execution.ex("#{tar} --owner=0 --group=0 #{tar_cmd} --file #{Pkg::Config.project}-all.tar #{name_of_archive}.tar.gz")
+          end
+        end
+      end
+    end
+
+    # tar does not support adding or updating files in a compressed archive, so
+    # we have a task to compress the "all" tarball from the 'pack_all_signed_repos'
+    # task
+    task :compress_the_all_tarball, [:versioning] => ["pl:fetch"] do |t, args|
+      versioning = args.versioning or fail ":versioning is a required argument for #{t}"
+      gzip = Pkg::Util::Tool.check_tool('gzip')
+      Dir.chdir("pkg") do
+        if versioning == 'ref'
+          local_target = File.join(Pkg::Config.project, Pkg::Config.ref)
+        elsif versioning == 'version'
+          local_target = File.join(Pkg::Config.project, Pkg::Util::Version.dot_version)
+        end
+        Dir.chdir(local_target) do
+          Pkg::Util::Execution.ex("#{gzip} --fast #{File.join("repos", "#{Pkg::Config.project}-all.tar")}")
+        end
+      end
+    end
+
+
     task :prepare_signed_repos, [:target_host, :target_prefix, :versioning] => ["clean", "pl:fetch"] do |t, args|
       target_host = args.target_host or fail ":target_host is a required argument to #{t}"
       target_prefix = args.target_prefix or fail ":target_prefix is a required argument for #{t}"
@@ -147,7 +196,7 @@ DOC
 
       if args.foss_only && Pkg::Config.foss_platforms && !Pkg::Config.foss_platforms.empty?
         Pkg::Config.foss_platforms.each do |platform|
-          include_paths << Pkg::Paths.repo_path(platform, legacy: true, nonfinal: true)
+          include_paths << Pkg::Paths.repo_path(platform, legacy: true)
           if Pkg::Paths.repo_config_path(platform)
             include_paths << Pkg::Paths.repo_config_path(platform)
           end
@@ -244,11 +293,11 @@ DOC
       local_pa_latest = "#{pa_source}-latest"
       local_pe_latest = "#{pe_target}-latest"
 
-      Pkg::Util::Net.remote_execute(target_host, "mkdir -p '#{pe_target}'")
-      Pkg::Util::Net.remote_execute(target_host, "mkdir -p '#{local_pe_latest}'")
-      Pkg::Util::Net.remote_execute(target_host, "cp -r #{local_pa_latest}/* #{local_pe_latest}")
-      Pkg::Util::Net.remote_execute(target_host, "sed -i 's|/#{File.basename(local_pa_latest)}|/#{pe_version}/#{File.basename(local_pa_latest)}|' #{local_pe_latest}/repo_configs/*/*")
-      Pkg::Util::Net.remote_execute(target_host, "ln -sf '#{local_pa}' '#{local_pe}'")
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "mkdir -p '#{pe_target}'")
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "mkdir -p '#{local_pe_latest}'")
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "cp -r #{local_pa_latest}/* #{local_pe_latest}")
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "sed -i 's|/#{File.basename(local_pa_latest)}|/#{pe_version}/#{File.basename(local_pa_latest)}|' #{local_pe_latest}/repo_configs/*/*")
+      Pkg::Util::Net.remote_ssh_cmd(target_host, "ln -sf '#{local_pa}' '#{local_pe}'")
     end
 
     task :nightly_repos => ["pl:fetch"] do
@@ -279,7 +328,7 @@ DOC
 
       latest_filename = File.join(latest_filepath, "LATEST")
       File.open(latest_filename, 'w') { |file| file.write(version) }
-      Pkg::Util::Net.s3sync_to(latest_filename, target_bucket, Pkg::Config.project, ["--acl-public", "--follow-symlinks"])
+      Pkg::Util::Net.s3sync_to(latest_filepath, target_bucket, Pkg::Config.project, ["--acl-public", "--follow-symlinks"])
       FileUtils.rm_rf latest_filepath
     end
   end

data/tasks/pe_ship.rake

@@ -14,9 +14,9 @@ if Pkg::Config.build_pe
       end
       if Pkg::Config.team == 'release'
 
-        # If this is not a feature branch or release branch, we need to link the
-        # shipped packages into the feature repos and update their metadata.
-        unless Pkg::Config.pe_feature_branch || Pkg::Config.pe_release_branch
+        # If this is not a feature branch, we need to link the shipped packages into the feature repos,
+        # then update their metadata as well.
+        unless Pkg::Config.pe_feature_branch
           puts "Linking RPMs to feature repo"
           Pkg::Util::RakeUtils.invoke_task("pe:remote:link_shipped_rpms_to_feature_repo")
           Pkg::Util::RakeUtils.invoke_task("pe:remote:update_yum_repo")
@@ -65,7 +65,7 @@ if Pkg::Config.build_pe
           puts "Shipping PE debs to apt repo 'incoming' dir on #{Pkg::Config.apt_host}"
           Pkg::Util::Execution.retry_on_fail(:times => 3) do
             Dir["pkg/pe/deb/#{dist}/*.deb"].each do |deb|
-              Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, "mkdir -p '#{target_path}/#{dist}'")
+              Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, "mkdir -p '#{target_path}/#{dist}'")
               Pkg::Util::Net.rsync_to(deb, Pkg::Config.apt_host, "#{target_path}/#{dist}/#{File.basename(deb)}")
             end
           end
@@ -126,9 +126,8 @@ if Pkg::Config.build_pe
           files += Dir["pkg/pe/deb/#{dist}/*"].select { |f| f !~ /^.*\.deb$/ }.map { |f| "#{base_path}/#{dist}-source/#{File.basename(f)}" }
         end
       end
-      # If this is not a feature branch or release branch, we need to link the
-      # shipped packages into the feature repos
-      unless Pkg::Config.pe_feature_branch || Pkg::Config.pe_release_branch
+      # If this is not a feature branch, we need to link the shipped packages into the feature repos
+      unless Pkg::Config.pe_feature_branch
         puts "Linking DEBs to feature repo"
         Pkg::Util::RakeUtils.invoke_task("pe:remote:link_shipped_debs_to_feature_repo")
       end
@@ -149,7 +148,7 @@ if Pkg::Config.build_pe
         command += %(done; )
         command += %(sync)
 
-        Pkg::Util::Net.remote_execute(Pkg::Config.yum_host, command)
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_host, command)
       end
 
       desc "Remotely add shipped packages to apt repo on #{Pkg::Config.apt_host}"
@@ -171,11 +170,7 @@ if Pkg::Config.build_pe
             aptly publish repo -gpg-key=\"8BBEB79B\" #{Pkg::Config::pe_version}-#{dist} #{Pkg::Config::pe_version}
           fi
         eos
-        stdout, stderr = Pkg::Util::Net.remote_execute(
-                  Pkg::Config.apt_host,
-                  cmd,
-                  { capture_output: true }
-                )
+        stdout, stderr = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, cmd, true)
 
         output = stdout.to_s + stderr.to_s
 
@@ -189,7 +184,7 @@ if Pkg::Config.build_pe
         puts
 
         puts "Cleaning up apt repo 'incoming' dir on #{Pkg::Config.apt_host}"
-        Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, "rm -r #{incoming_dir}")
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, "rm -r #{incoming_dir}")
 
       end
 
@@ -205,7 +200,7 @@ if Pkg::Config.build_pe
         command += %(done; )
         command += %(sync)
 
-        Pkg::Util::Net.remote_execute(Pkg::Config.yum_host, command)
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_host, command)
       end
 
       desc "Remotely link shipped deb packages into feature repo on #{Pkg::Config.apt_host}"
@@ -219,7 +214,7 @@ if Pkg::Config.build_pe
         command += %(done; )
         command += %(sync)
 
-        Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, command)
+        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, command)
       end
     end
   end

data/tasks/retrieve.rake

@@ -14,20 +14,16 @@ namespace :pl do
   namespace :jenkins do
     desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
     task :retrieve, [:remote_target, :local_target] => 'pl:fetch' do |t, args|
-      unless Pkg::Config.project
-        fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
-      end
       remote_target = args.remote_target || "artifacts"
       local_target = args.local_target || "pkg"
       mkdir_p local_target
-      build_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{remote_target}"
-      build_path = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{remote_target}"
+      build_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
+      build_path = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
       if Pkg::Config.foss_only
         Pkg::Retrieve.foss_only_retrieve(build_url, local_target)
       else
-        Pkg::Retrieve.retrieve_all(build_url, build_path, local_target)
+        Pkg::Retrieve.retrieve_all(build_url, build_path, remote_target, local_target)
       end
-      fail "Uh oh, looks like we didn't find anything in #{local_target} when attempting to retrieve from #{build_url}!" if Dir["#{local_target}/*"].empty?
       puts "Packages staged in #{local_target}"
     end
   end
@@ -38,14 +34,11 @@ if Pkg::Config.build_pe
     namespace :jenkins do
       desc "Retrieve packages from the distribution server\. Check out commit to retrieve"
       task :retrieve, [:remote_target, :local_target] => 'pl:fetch' do |t, args|
-        unless Pkg::Config.project
-          fail "You must set the 'project' in build_defaults.yaml or with the 'PROJECT_OVERRIDE' environment variable."
-        end
         remote_target = args.remote_target || "artifacts"
         local_target = args.local_target || "pkg"
-        build_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{remote_target}"
-        build_path = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{remote_target}"
-        Pkg::Retrieve.retrieve_all(build_url, build_path, local_target)
+        build_url = "http://#{Pkg::Config.builds_server}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
+        build_path = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}"
+        Pkg::Retrieve.retrieve_all(build_url, build_path, remote_target, local_target)
       end
     end
   end