packaging 0.99.75 → 0.99.79

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4eba143d4627b1c6ce1eb694894a79987fb4c9eb169a8de3dfe756c037f2cfd
-  data.tar.gz: fc772076ec4a28f57e2a99afb51a29ecba0d20b6b0eea0d32d16f43575f8c8d4
+  metadata.gz: ffbdc37d8e8208db94c8f3ac323ca856d173099795d58c3406b28c06ed04b4a9
+  data.tar.gz: 6a261865330a3b86e8795cc3ad586552b744c41e8ce277ce5b7b1595af12bdb5
 SHA512:
-  metadata.gz: 5e6b6a56070e12ea898725f030d2cf95b6e04f6ac329f6dab38a20ab9aa765a3a7667f6967373dbde8cf8686403784ad9ca4e49bc83923e260a7c9517e457329
-  data.tar.gz: 5a3cadefed6983b5bcacf0f2abb117b8f2c5aa829dc8567a041b4e4b679d1489d8a2badb80a1c7c1b0a51cbda94ad07d8f00d9378dcaca9ab02b4cd831aeab2f
+  metadata.gz: 512c7f42a929636c35b01bc50da52dc2e7f9c6387405cf0c4928115aa4b9bfb70e19510da8112fe121022938a89c16c89ba76c3f9142b9b867d1c9a3b28401e4
+  data.tar.gz: b55a3806f0f0026be2819f336ffdc5fb6d7641c9f06b6337e7b5e3e81c11766cd95ca2779b1de5dfd31d52fc1193d603b345c5077d6afc4e9d326bf0d486e999

data/README.md

@@ -404,7 +404,7 @@ deb_build_mirrors:
 # Who is packaging. Turns up in various packaging artifacts
 packager: 'puppetlabs'
 # GPG key ID of the signer
-gpg_key: '7F438280EF8D349F'
+gpg_key: '4528B6CD9E61EF26'
 # Whether to require tarball signing as a prerequisite of other package building
 sign_tar: false
 # a space separated list of mock configs. These are the rpm distributions to package for. If a noarch package, only one arch of each is needed.

data/lib/packaging/archive.rb

@@ -33,7 +33,7 @@ module Pkg::Archive
       sudo chmod g+w -R #{Pkg::Config.yum_archive_path}
       mv #{full_directory} #{archive_path}
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Move directories from freight path (aka repo staging path) to archive staging paths
@@ -60,7 +60,7 @@ module Pkg::Archive
         mv $pool_directory /opt/tmp-apt
       done
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Move downloads directories to archive staging path
@@ -85,7 +85,7 @@ module Pkg::Archive
       sudo chmod g+w -R #{Pkg::Config.downloads_archive_path}
       mv #{full_directory} #{archive_path}
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Delete empty directories from repo paths on weth
@@ -104,7 +104,7 @@ module Pkg::Archive
           fi
         done
       CMD
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 
@@ -112,7 +112,7 @@ module Pkg::Archive
   def remove_dead_symlinks
     base_paths.each do |path|
       command = "find #{path} -xtype l -delete"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 
@@ -120,7 +120,7 @@ module Pkg::Archive
   def delete_staged_archives
     archive_paths.each do |archive_path|
       command = "sudo rm -rf #{File.join(archive_path, '*')}"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 end

data/lib/packaging/artifactory.rb

@@ -224,8 +224,6 @@ module Pkg
         deploy_properties(platform_tag, File.basename(package)),
         headers
       )
-    rescue
-      raise "Attempt to upload '#{package}' to #{File.join(@artifactory_uri, data[:full_artifactory_path])} failed"
     end
 
     # @param pkg [String] The package to download YAML for

data/lib/packaging/config.rb

@@ -6,6 +6,7 @@ module Pkg
   #
   class Config
     require 'packaging/config/params.rb'
+    require 'packaging/config/validations.rb'
     require 'yaml'
 
     class << self
@@ -80,7 +81,11 @@ module Pkg
         dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
         cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
               "find ./artifacts/ -mindepth 2 -type f; fi"
-        artifacts, _ = Pkg::Util::Net.remote_ssh_cmd(self.builds_server, cmd, true)
+        artifacts, _ = Pkg::Util::Net.remote_execute(
+                     self.builds_server,
+                     cmd,
+                     { capture_output: true }
+                   )
 
         artifacts = artifacts.split("\n")
         data = {}
@@ -389,6 +394,31 @@ module Pkg
         end
       end
 
+      ##
+      #   Ask for validation of BUILD_PARAMS
+      #
+      #   Issued as warnings initially but the intent is to turn this into
+      #   a failure.
+      #
+      def perform_validations
+        error_count = 0
+        Pkg::Params::VALIDATIONS.each do |v|
+          variable_name = v[:var]
+          variable_value = self.instance_variable_get("@#{v[:var]}")
+          validations = v[:validations]
+          validations.each do |validation|
+            unless Pkg::ConfigValidations.send(validation, variable_value)
+              warn "Warning: variable \"#{variable_name}\" failed validation \"#{validation}\""
+              error_count += 1
+            end
+          end
+        end
+
+        if error_count != 0
+          warn "Warning: #{error_count} validation failure(s)."
+        end
+      end
+
       def string_to_array(str)
         delimiters = /[,\s;]/
         return str if str.respond_to?('each')

data/lib/packaging/config/params.rb

@@ -65,6 +65,7 @@ module Pkg::Params
                   :gem_files,
                   :gem_forge_project,
                   :gem_host,
+                  :gem_license,
                   :gem_name,
                   :gem_path,
                   :gem_platform_dependencies,
@@ -363,6 +364,7 @@ module Pkg::Params
                     { :oldvar => :yum_host,               :newvar => :tar_host },
                  ]
 
+
   # These are variables that we have deprecated. If they are encountered in a
   # project's config, we issue deprecations for them.
   #
@@ -373,4 +375,14 @@ module Pkg::Params
                   { :var => :gpg_name, :message => "
     DEPRECATED, 29-Jul-2014: 'gpg_name' has been replaced with 'gpg_key'.
                    Please update this field in your build_defaults.yaml" }]
+
+  # Provide an open-ended template for validating BUILD_PARAMS.
+  #
+  # Each validatation contains the variable name as ':var' and a list of validations it
+  # must pass from the Pkg::Params::Validations class.
+  #
+  VALIDATIONS = [
+    { :var => :project, :validations => [:not_empty?] }
+  ]
+
 end

data/lib/packaging/config/validations.rb

@@ -0,0 +1,13 @@
+module Pkg
+  class ConfigValidations
+
+    class << self
+
+      # As a validation, this one is kindof lame but is intended as a seed pattern for possibly
+      # more robust ones.
+      def not_empty?(value)
+        value.to_s.empty? ? false : true
+      end
+    end
+  end
+end

data/lib/packaging/deb/repo.rb

@@ -125,7 +125,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
       command = repo_creation_command(File.join(artifact_directory, 'repos'), artifact_paths)
 
       begin
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
         # Now that we've created our package repositories, we can generate repo
         # configurations for use with downstream jobs, acceptance clients, etc.
         Pkg::Deb::Repo.generate_repo_configs
@@ -134,7 +134,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
         Pkg::Deb::Repo.ship_repo_configs
       ensure
         # Always remove the lock file, even if we've failed
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
       end
     end
 
@@ -146,7 +146,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
 
       Pkg::Util::RakeUtils.invoke_task("pl:fetch")
       repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/deb"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         Pkg::Util::Net.rsync_to("pkg/#{target}/deb/", Pkg::Config.distribution_server, repo_dir)
       end
@@ -252,11 +252,11 @@ SignWith: #{Pkg::Config.gpg_key}"
       rsync_command = repo_deployment_command(apt_path, destination_staging_path, destination_server, dryrun)
       cp_command = repo_deployment_command(destination_staging_path, apt_path, nil, dryrun)
 
-      Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
+      Pkg::Util::Net.remote_execute(origin_server, rsync_command)
       if dryrun
         puts "[DRYRUN] not executing #{cp_command} on #{destination_server}"
       else
-        Pkg::Util::Net.remote_ssh_cmd(destination_server, cp_command)
+        Pkg::Util::Net.remote_execute(destination_server, cp_command)
       end
     end
 

data/lib/packaging/paths.rb

@@ -322,7 +322,8 @@ module Pkg::Paths
     if %w(puppet7 puppet7-nightly
           puppet6 puppet6-nightly
           puppet5 puppet5-nightly
-          puppet).include? repo_name
+          puppet  puppet-nightly
+          puppet-tools).include? repo_name
       return File.join(remote_repo_path, 'pool', code_name, repo_name, project[0], project)
     end
 

data/lib/packaging/platforms.rb

@@ -12,13 +12,6 @@ module Pkg
     # Each element in this hash
     PLATFORM_INFO = {
       'aix' => {
-        '6.1' => {
-          architectures: ['power'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          repo: false,
-        },
         '7.1' => {
           architectures: ['power'],
           source_architecture: 'SRPMS',
@@ -28,25 +21,6 @@ module Pkg
         },
       },
 
-      'cisco-wrlinux' => {
-        '5' => {
-          architectures: ['x86_64'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v4',
-          repo: true,
-        },
-        '7' => {
-          architectures: ['x86_64'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v4',
-          repo: true,
-        },
-      },
-
       'debian' => {
         '8' => {
           codename: 'jessie',
@@ -72,6 +46,14 @@ module Pkg
           source_package_formats: DEBIAN_SOURCE_FORMATS,
           repo: true,
         },
+        '11' => {
+          codename: 'bullseye',
+          architectures: ['amd64'],
+          source_architecture: 'source',
+          package_format: 'deb',
+          source_package_formats: DEBIAN_SOURCE_FORMATS,
+          repo: true,
+        },
       },
 
       'el' => {
@@ -100,7 +82,7 @@ module Pkg
           repo: true,
         },
         '8' => {
-          architectures: ['x86_64', 'aarch64'],
+          architectures: ['x86_64', 'ppc64le', 'aarch64'],
           source_architecture: 'SRPMS',
           package_format: 'rpm',
           source_package_formats: ['src.rpm'],
@@ -109,14 +91,6 @@ module Pkg
         }
       },
 
-      'eos' => {
-        '4' => {
-          architectures: ['i386'],
-          package_format: 'swix',
-          repo: false,
-        },
-      },
-
       'fedora' => {
         '30' => {
           architectures: ['x86_64'],
@@ -142,6 +116,14 @@ module Pkg
           signature_format: 'v4',
           repo: true,
         },
+        '34' => {
+          architectures: ['x86_64'],
+          source_architecture: 'SRPMS',
+          package_format: 'rpm',
+          source_package_formats: ['src.rpm'],
+          signature_format: 'v4',
+          repo: true,
+        },
       },
 
       'osx' => {
@@ -160,6 +142,11 @@ module Pkg
           package_format: 'dmg',
           repo: false,
         },
+        '11' => {
+          architectures: ['x86_64', 'arm64'],
+          package_format: 'dmg',
+          repo: false,
+        },
       },
 
       'redhatfips' => {

data/lib/packaging/repo.rb

@@ -110,7 +110,11 @@ module Pkg::Repo
       cmd = "[ -d #{artifact_directory} ] || exit 1 ; "
       cmd << "pushd #{artifact_directory} > /dev/null && "
       cmd << "find . -name '*.#{pkg_ext}' -print0 | xargs --no-run-if-empty -0 -I {} dirname {} "
-      stdout, stderr = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd, true)
+      stdout, _ = Pkg::Util::Net.remote_execute(
+                Pkg::Config.distribution_server,
+                cmd,
+                { capture_output: true }
+              )
       return stdout.split
     rescue => e
       fail "Error: Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
@@ -120,7 +124,7 @@ module Pkg::Repo
       cmd = "[ -d #{artifact_parent_directory}/artifacts ] || exit 1 ; "
       cmd << "pushd #{artifact_parent_directory} > /dev/null && "
       cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd)
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, cmd)
     rescue => e
       fail "Error: Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
     end
@@ -143,7 +147,9 @@ module Pkg::Repo
         __APT_PLATFORMS__: Pkg::Config.apt_releases.join(' '),
         __GPG_KEY__: Pkg::Util::Gpg.key
       }
-      Pkg::Util::Net.remote_ssh_cmd(remote_host, Pkg::Util::Misc.search_and_replace(command, whitelist))
+      Pkg::Util::Net.remote_execute(
+        remote_host,
+        Pkg::Util::Misc.search_and_replace(command, whitelist))
     end
   end
 end

data/lib/packaging/rpm/repo.rb

@@ -16,7 +16,7 @@ module Pkg::Rpm::Repo
 
       Pkg::Util::RakeUtils.invoke_task("pl:fetch")
       repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/rpm"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         Pkg::Util::Net.rsync_to("pkg/#{target}/rpm/", Pkg::Config.distribution_server, repo_dir)
       end
@@ -215,7 +215,7 @@ module Pkg::Rpm::Repo
       command = Pkg::Rpm::Repo.repo_creation_command(File.join(artifact_directory, directory), artifact_paths)
 
       begin
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
         # Now that we've created our package repositories, we can generate repo
         # configurations for use with downstream jobs, acceptance clients, etc.
         Pkg::Rpm::Repo.generate_repo_configs
@@ -224,7 +224,7 @@ module Pkg::Rpm::Repo
         Pkg::Rpm::Repo.ship_repo_configs
       ensure
         # Always remove the lock file, even if we've failed
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
       end
     end
 
@@ -248,7 +248,7 @@ module Pkg::Rpm::Repo
     def deploy_repos(yum_path, origin_server, destination_server, dryrun = false)
       rsync_command = repo_deployment_command(yum_path, yum_path, destination_server, dryrun)
 
-      Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
+      Pkg::Util::Net.remote_execute(origin_server, rsync_command)
     end
   end
 end

data/lib/packaging/sign/dmg.rb

@@ -1,41 +1,79 @@
 module Pkg::Sign::Dmg
   module_function
 
-  def sign(target_dir = 'pkg')
-    use_identity = "-i #{Pkg::Config.osx_signing_ssh_key}" unless Pkg::Config.osx_signing_ssh_key.nil?
-
-    if Pkg::Config.osx_signing_server =~ /@/
-      host_string = "#{Pkg::Config.osx_signing_server}"
-    else
-      host_string = "#{ENV['USER']}@#{Pkg::Config.osx_signing_server}"
+  def sign(pkg_directory = 'pkg')
+    use_identity = ''
+    unless Pkg::Config.osx_signing_ssh_key.nil?
+      use_identity = "-i #{Pkg::Config.osx_signing_ssh_key}"
     end
+
+    host_string = "#{ENV['USER']}@#{Pkg::Config.osx_signing_server}"
+    host_string = "#{Pkg::Config.osx_signing_server}" if Pkg::Config.osx_signing_server =~ /@/
+
     ssh_host_string = "#{use_identity} #{host_string}"
     rsync_host_string = "-e 'ssh #{use_identity}' #{host_string}"
+    archs =  Dir.glob("#{pkg_directory}/{apple,mac,osx}/**/{x86_64,arm64}").map { |el| el.split('/').last }
+
+    if archs.empty?
+      $stderr.puts "Error: no architectures found in #{pkg_directory}/{apple,mac,osx}"
+      exit 1
+    end
+
+    archs.each do |arch|
+      remote_working_directory = "/tmp/#{Pkg::Util.rand_string}/#{arch}"
+      dmg_mount_point = File.join(remote_working_directory, "mount")
+      signed_items_directory    = File.join(remote_working_directory, "signed")
+
+      dmgs = Dir.glob("#{pkg_directory}/{apple,mac,osx}/**/#{arch}/*.dmg")
+      if dmgs.empty?
+        $stderr.puts "Error: no dmgs found in #{pkg_directory}/{apple,mac,osx} for #{arch} architecture."
+        exit 1
+      end
+
+      dmg_basenames = dmgs.map { |d| File.basename(d, '.dmg') }.join(' ')
+
+      sign_package_command = %W[
+        for dmg in #{dmg_basenames}; do
+          /usr/bin/hdiutil attach #{remote_working_directory}/$dmg.dmg
+            -mountpoint #{dmg_mount_point} -nobrowse -quiet ;
 
-    work_dir  = "/tmp/#{Pkg::Util.rand_string}"
-    mount     = File.join(work_dir, "mount")
-    signed    = File.join(work_dir, "signed")
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, "mkdir -p #{mount} #{signed}")
-    dmgs = Dir.glob("#{target_dir}/apple/**/*.dmg")
-    Pkg::Util::Net.rsync_to(dmgs.join(" "), rsync_host_string, work_dir)
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, %Q[for dmg in #{dmgs.map { |d| File.basename(d, ".dmg") }.join(" ")}; do
-      /usr/bin/hdiutil attach #{work_dir}/$dmg.dmg -mountpoint #{mount} -nobrowse -quiet ;
-      /usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}" ;
-        for pkg in $(ls #{mount}/*.pkg | xargs -n 1 basename); do
-          if /usr/sbin/pkgutil --check-signature #{mount}/$pkg ; then
-            echo "$pkg is already signed, skipping . . ." ;
-            cp #{mount}/$pkg #{signed}/$pkg ;
-          else
-            /usr/bin/productsign --keychain "#{Pkg::Config.osx_signing_keychain}" --sign "#{Pkg::Config.osx_signing_cert}" #{mount}/$pkg #{signed}/$pkg ;
-          fi
+          /usr/bin/security -q unlock-keychain
+            -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}" ;
+
+          for pkg in #{dmg_mount_point}/*.pkg; do
+            pkg_basename=$(basename $pkg) ;
+            if /usr/sbin/pkgutil --check-signature $pkg ; then
+              echo "Warning: $pkg is already signed, skipping" ;
+              cp $pkg #{signed_items_directory}/$pkg_basename ;
+              continue ;
+            fi ;
+
+            /usr/bin/productsign --keychain "#{Pkg::Config.osx_signing_keychain}"
+              --sign "#{Pkg::Config.osx_signing_cert}"
+              $pkg #{signed_items_directory}/$pkg_basename ;
+          done ;
+
+          /usr/bin/hdiutil detach #{dmg_mount_point} -quiet ;
+          /bin/rm #{remote_working_directory}/$dmg.dmg ;
+          /usr/bin/hdiutil create -volname $dmg
+            -srcfolder #{signed_items_directory}/ #{remote_working_directory}/$dmg.dmg ;
+          /bin/rm #{signed_items_directory}/* ;
         done
-      /usr/bin/hdiutil detach #{mount} -quiet ;
-      /bin/rm #{work_dir}/$dmg.dmg ;
-      /usr/bin/hdiutil create -volname $dmg -srcfolder #{signed}/ #{work_dir}/$dmg.dmg ;
-      /bin/rm #{signed}/* ; done])
-    dmgs.each do | dmg |
-      Pkg::Util::Net.rsync_from("#{work_dir}/#{File.basename(dmg)}", rsync_host_string, File.dirname(dmg))
+      ].join(' ')
+
+      Pkg::Util::Net.remote_execute(ssh_host_string,
+                                    "mkdir -p #{dmg_mount_point} #{signed_items_directory}")
+
+      Pkg::Util::Net.rsync_to(dmgs.join(' '), rsync_host_string, remote_working_directory)
+
+      Pkg::Util::Net.remote_execute(ssh_host_string, sign_package_command)
+
+      dmgs.each do |dmg|
+        Pkg::Util::Net.rsync_from(
+          "#{remote_working_directory}/#{File.basename(dmg)}", rsync_host_string, File.dirname(dmg))
+      end
+
+      Pkg::Util::Net.remote_execute(ssh_host_string, "rm -rf '#{remote_working_directory}'")
     end
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, "if [ -d '#{work_dir}' ]; then rm -rf '#{work_dir}'; fi")
   end
 end