packaging 0.99.75 → 0.99.79

data/tasks/pe_ship.rake

@@ -65,7 +65,7 @@ if Pkg::Config.build_pe
           puts "Shipping PE debs to apt repo 'incoming' dir on #{Pkg::Config.apt_host}"
           Pkg::Util::Execution.retry_on_fail(:times => 3) do
             Dir["pkg/pe/deb/#{dist}/*.deb"].each do |deb|
-              Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, "mkdir -p '#{target_path}/#{dist}'")
+              Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, "mkdir -p '#{target_path}/#{dist}'")
               Pkg::Util::Net.rsync_to(deb, Pkg::Config.apt_host, "#{target_path}/#{dist}/#{File.basename(deb)}")
             end
           end
@@ -145,23 +145,11 @@ if Pkg::Config.build_pe
         # single line. By breaking it into a series of concatenated strings, we can maintain
         # a semblance of formatting and structure (nevermind readability).
         command  = %(for dir in #{repo_base_path}/{#{rpm_family_and_version.join(",")}}-*; do)
-
-        # For (RE-12463), make this less quiet than it has been.
-        # We're trying to find out why its aborting. Preserve the original
-        # so it can be restored.
-        command += %(  sudo createrepo --checksum=sha --checkts --update --delta-workers=0 --database --update $dir; )
-
-        ## Original
-        # command += %(  sudo createrepo --checksum=sha --checkts --update --delta-workers=0 --quiet --database --update $dir; )
-        ##
+        command += %(  sudo createrepo --checksum=sha --checkts --update --delta-workers=0 --quiet --database --update $dir; )
         command += %(done; )
         command += %(sync)
 
-        # Again for (RE-12463). Make this more verbose temporarily
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_host, command, false, '', true, true)
-
-        ## Original one below
-        # Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_host, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.yum_host, command)
       end
 
       desc "Remotely add shipped packages to apt repo on #{Pkg::Config.apt_host}"
@@ -183,7 +171,11 @@ if Pkg::Config.build_pe
             aptly publish repo -gpg-key=\"8BBEB79B\" #{Pkg::Config::pe_version}-#{dist} #{Pkg::Config::pe_version}
           fi
         eos
-        stdout, stderr = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, cmd, true)
+        stdout, stderr = Pkg::Util::Net.remote_execute(
+                  Pkg::Config.apt_host,
+                  cmd,
+                  { capture_output: true }
+                )
 
         output = stdout.to_s + stderr.to_s
 
@@ -197,7 +189,7 @@ if Pkg::Config.build_pe
         puts
 
         puts "Cleaning up apt repo 'incoming' dir on #{Pkg::Config.apt_host}"
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, "rm -r #{incoming_dir}")
+        Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, "rm -r #{incoming_dir}")
 
       end
 
@@ -213,7 +205,7 @@ if Pkg::Config.build_pe
         command += %(done; )
         command += %(sync)
 
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_host, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.yum_host, command)
       end
 
       desc "Remotely link shipped deb packages into feature repo on #{Pkg::Config.apt_host}"
@@ -227,7 +219,7 @@ if Pkg::Config.build_pe
         command += %(done; )
         command += %(sync)
 
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_host, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.apt_host, command)
       end
     end
   end

data/tasks/ship.rake

@@ -83,7 +83,11 @@ namespace :pl do
           source_dir = 'pkg/solaris/11/'
         end
 
-        tmpdir, _ = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, 'mktemp -d -p /var/tmp', true)
+        tmpdir, _ = Pkg::Util::Net.remote_execute(
+                  Pkg::Config.ips_host,
+                  'mktemp -d -p /var/tmp',
+                  { capture_output: true }
+                )
         tmpdir.chomp!
 
         Pkg::Util::Net.rsync_to(source_dir, Pkg::Config.ips_host, tmpdir)
@@ -92,9 +96,9 @@ namespace :pl do
       sudo pkgrecv -s $pkg -d #{Pkg::Config.ips_path} '*';
       done)
 
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, remote_cmd)
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.ips_host, "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
+        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, remote_cmd)
+        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo pkgrepo refresh -s #{Pkg::Config.ips_path}")
+        Pkg::Util::Net.remote_execute(Pkg::Config.ips_host, "sudo /usr/sbin/svcadm restart svc:/application/pkg/server:#{Pkg::Config.ips_repo || 'default'}")
       end
     end
 
@@ -104,7 +108,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(times: 3) do
           cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.dmg_path, target_host: Pkg::Config.dmg_host, extra_flags: ['--update'])
-          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.dmg_staging_server, cmd)
+          Pkg::Util::Net.remote_execute(Pkg::Config.dmg_staging_server, cmd)
         end
       end
     end
@@ -115,7 +119,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(times: 3) do
           cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.swix_path, target_host: Pkg::Config.swix_host, extra_flags: ['--update'])
-          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.swix_staging_server, cmd)
+          Pkg::Util::Net.remote_execute(Pkg::Config.swix_staging_server, cmd)
         end
       end
     end
@@ -130,7 +134,7 @@ namespace :pl do
         else
           Pkg::Util::Execution.retry_on_fail(times: 3) do
             cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.tarball_path, target_host: Pkg::Config.tar_host, extra_flags: ['--update'])
-            Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.tar_staging_server, cmd)
+            Pkg::Util::Net.remote_execute(Pkg::Config.tar_staging_server, cmd)
           end
         end
       end
@@ -146,7 +150,7 @@ namespace :pl do
         else
           Pkg::Util::Execution.retry_on_fail(times: 3) do
             cmd = Pkg::Util::Net.rsync_cmd(Pkg::Config.msi_path, target_host: Pkg::Config.msi_host, extra_flags: ['--update'])
-            Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.msi_staging_server, cmd)
+            Pkg::Util::Net.remote_execute(Pkg::Config.msi_staging_server, cmd)
           end
         end
       end
@@ -174,7 +178,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh apt.puppetlabs.com'
-          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.apt_signing_server, command)
+          Pkg::Util::Net.remote_execute(Pkg::Config.apt_signing_server, command)
         end
       end
     end
@@ -200,7 +204,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh yum.puppetlabs.com'
-          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.yum_staging_server, command)
+          Pkg::Util::Net.remote_execute(Pkg::Config.yum_staging_server, command)
         end
       end
     end
@@ -211,7 +215,7 @@ namespace :pl do
       if Pkg::Util.ask_yes_or_no
         Pkg::Util::Execution.retry_on_fail(:times => 3) do
           command = 'sudo /usr/local/bin/s3_repo_sync.sh downloads.puppetlabs.com'
-          Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+          Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
         end
       end
     end
@@ -228,7 +232,7 @@ namespace :pl do
       puts "Deploying nightly builds from #{Pkg::Config.staging_server} to AWS S3..."
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         command = 'sudo /usr/local/bin/s3_repo_sync.sh nightlies.puppet.com'
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
       end
     end
 
@@ -242,7 +246,7 @@ namespace :pl do
             ['apt', 'yum'].each do |repo|
               # Don't --delete so that folks using archived packages can continue to do so
               command = "sudo su - rsync --command 'rsync --verbose -a --exclude '*.html' /opt/repo-s3-stage/repositories/#{repo}.puppetlabs.com/ rsync@#{rsync_server}:/opt/repository/#{repo}'"
-              Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+              Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
             end
           end
         end
@@ -263,26 +267,7 @@ namespace :pl do
 
       command += %(; sync)
 
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.gem_host, command)
-    end
-
-    # PA-3356: temporary task to link puppet 7 nightly gem
-    # TODO: PA-3358 - remove when puppet 7 is officialy out
-    desc "Remotely link puppet 7 nightly shipped gems to latest versions on #{Pkg::Config.gem_host}"
-    task link_puppet_7_nightly_shipped_gems_to_latest: 'pl:fetch' do
-      Pkg::Config.gemversion = Pkg::Util::Version.extended_dot_version.gsub(/6\.\d+\.\d+/, '7.0.0')
-
-      remote_path = Pkg::Config.nonfinal_gem_path
-      gems = FileList['pkg/*.gem'].map! { |path| path.gsub!('pkg/', '') }
-      command = %(cd #{remote_path}; )
-
-      command += gems.map! do |gem_name|
-        %(sudo ln -sf #{gem_name} #{gem_name.gsub(Pkg::Config.gemversion, 'latest')})
-      end.join(';')
-
-      command += %(; sync)
-
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.gem_host, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.gem_host, command)
     end
   end
 
@@ -463,6 +448,16 @@ namespace :pl do
     end
   end
 
+  desc 'Create the rolling repo links'
+  task create_repo_links: 'pl:fetch' do
+    Pkg::Util::Ship.create_rolling_repo_links
+  end
+
+  desc 'Create rolling repo links for nightlies'
+  task create_nightly_repo_links: 'pl:fetch' do
+    Pkg::Util::Ship.create_rolling_repo_links(true)
+  end
+
   desc 'Test out the ship requirements'
   task ship_check: 'pl:fetch' do
     errs = []
@@ -508,7 +503,11 @@ namespace :pl do
     # Check for ability to sign OSX. Should just need to be able to unlock keychain
     begin
       unless ssh_errs.include?(Pkg::Config.osx_signing_server)
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.osx_signing_server, %(/usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}"), false, '-oBatchMode=yes')
+        Pkg::Util::Net.remote_execute(
+          Pkg::Config.osx_signing_server,
+          %(/usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}"),
+          { extra_options: '-oBatchMode=yes' }
+        )
       end
     rescue
       errs << "Unlocking the OSX keychain failed! Check the password in your .bashrc on #{Pkg::Config.osx_signing_server}"
@@ -536,15 +535,6 @@ namespace :pl do
       end
     end
 
-    desc 'Create the rolling repo links'
-    task create_repo_links: 'pl:fetch' do
-      Pkg::Util::Ship.create_rolling_repo_links
-    end
-
-    desc 'Create rolling repo links for nightlies'
-    task create_nightly_repo_links: 'pl:fetch' do
-      Pkg::Util::Ship.create_rolling_repo_links(true)
-    end
   end
 
   # It is odd to namespace this ship task under :jenkins, but this task is
@@ -563,7 +553,17 @@ namespace :pl do
       artifactory = Pkg::ManageArtifactory.new(Pkg::Config.project, Pkg::Config.ref)
 
       local_dir = args.local_dir || 'pkg'
-      Dir.glob("#{local_dir}/**/*").reject { |e| File.directory? e }.each do |artifact|
+      artifacts = Dir.glob("#{local_dir}/**/*").reject { |e| File.directory? e }
+      artifacts.sort! do |a, b|
+        if File.extname(a) =~ /(md5|sha\d+)/ && File.extname(b) !~ /(md5|sha\d+)/
+          1
+        elsif File.extname(b) =~ /(md5|sha\d+)/ && File.extname(a) !~ /(md5|sha\d+)/
+          -1
+        else
+          a <=> b
+        end
+      end
+      artifacts.each do |artifact|
         if File.extname(artifact) == ".yaml" || File.extname(artifact) == ".json"
           artifactory.deploy_package(artifact)
         elsif artifactory.package_exists_on_artifactory?(artifact)
@@ -670,8 +670,8 @@ namespace :pl do
       end
 
       Pkg::Util::Execution.retry_on_fail(times: 3) do
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir --mode=775 -p #{project_basedir}")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{artifact_dir}")
         Pkg::Util::Net.rsync_to("#{local_dir}/", Pkg::Config.distribution_server, "#{artifact_dir}/", extra_flags: ['--ignore-existing', '--exclude repo_configs'])
       end
 

data/tasks/sign.rake

@@ -144,10 +144,10 @@ cd #{remote_repo} ;
 #{Pkg::Util::Net.remote_bundle_install_command}
 bundle exec rake #{sign_tasks.map { |task| task + "[#{root_dir}]" }.join(" ")} PARAMS_FILE=#{build_params}
 DOC
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.signing_server, rake_command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, rake_command)
       Pkg::Util::Net.rsync_from("#{remote_repo}/#{root_dir}/", Pkg::Config.signing_server, "#{root_dir}/")
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.signing_server, "rm -rf #{remote_repo}")
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.signing_server, "rm #{build_params}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm -rf #{remote_repo}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.signing_server, "rm #{build_params}")
       puts "Signed packages staged in #{root_dir}/ directory"
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: packaging
 version: !ruby/object:Gem::Version
-  version: 0.99.75
+  version: 0.99.79
 platform: ruby
 authors:
 - Puppet Labs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-08 00:00:00.000000000 Z
+date: 2021-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -124,6 +124,7 @@ files:
 - lib/packaging/artifactory/extensions.rb
 - lib/packaging/config.rb
 - lib/packaging/config/params.rb
+- lib/packaging/config/validations.rb
 - lib/packaging/deb.rb
 - lib/packaging/deb/repo.rb
 - lib/packaging/gem.rb
@@ -260,27 +261,27 @@ specification_version: 4
 summary: Puppet Labs' packaging automation
 test_files:
 - spec/lib/packaging_spec.rb
+- spec/lib/packaging/tar_spec.rb
+- spec/lib/packaging/retrieve_spec.rb
+- spec/lib/packaging/deb_spec.rb
 - spec/lib/packaging/rpm/repo_spec.rb
-- spec/lib/packaging/paths_spec.rb
+- spec/lib/packaging/config_spec.rb
 - spec/lib/packaging/deb/repo_spec.rb
+- spec/lib/packaging/platforms_spec.rb
+- spec/lib/packaging/gem_spec.rb
+- spec/lib/packaging/paths_spec.rb
 - spec/lib/packaging/util/gpg_spec.rb
-- spec/lib/packaging/util/os_spec.rb
-- spec/lib/packaging/util/file_spec.rb
 - spec/lib/packaging/util/ship_spec.rb
-- spec/lib/packaging/util/git_spec.rb
-- spec/lib/packaging/util/execution_spec.rb
-- spec/lib/packaging/util/misc_spec.rb
 - spec/lib/packaging/util/rake_utils_spec.rb
+- spec/lib/packaging/util/execution_spec.rb
 - spec/lib/packaging/util/version_spec.rb
-- spec/lib/packaging/util/git_tag_spec.rb
+- spec/lib/packaging/util/misc_spec.rb
 - spec/lib/packaging/util/jenkins_spec.rb
+- spec/lib/packaging/util/git_tag_spec.rb
 - spec/lib/packaging/util/net_spec.rb
-- spec/lib/packaging/deb_spec.rb
-- spec/lib/packaging/config_spec.rb
-- spec/lib/packaging/artifactory_spec.rb
-- spec/lib/packaging/gem_spec.rb
-- spec/lib/packaging/tar_spec.rb
+- spec/lib/packaging/util/git_spec.rb
+- spec/lib/packaging/util/os_spec.rb
+- spec/lib/packaging/util/file_spec.rb
 - spec/lib/packaging/sign_spec.rb
 - spec/lib/packaging/repo_spec.rb
-- spec/lib/packaging/platforms_spec.rb
-- spec/lib/packaging/retrieve_spec.rb
+- spec/lib/packaging/artifactory_spec.rb