packaging 0.99.72 → 0.99.77

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d7bf70560a370066a2f9ab94c35c38093b5884e7baeabc9efe0926de62e262f
-  data.tar.gz: c6bf09ab5b588fa500b3db7408372194e2b7a1ac154ea98e1481085166deb28b
+  metadata.gz: 365629e0e3cba195de934ccd1cee9ecff5c7ed684ac79e4befc883420037b191
+  data.tar.gz: 2498029caf73135b3ee7c5a2704e310ae6398c4eb7716c3687db487fc827dc0d
 SHA512:
-  metadata.gz: 91aa048b12dfe8fd25867d9d5522db55559f2273284cafb3089362f0f14ee09ae536c3d039a04d63ebb95d0c1238a6467f60e27d8a1b41b512234ba609cb58ff
-  data.tar.gz: b26d4ff051f2b7ff0cddfdb1a8f3237377d237a3bc59511cbef6785ccb566eac8917d100108acbe41488d2748e8439860f26038069fd8f818d72c7474e4468e8
+  metadata.gz: 4a73420699b6ce732c52c527468c5b9cef405c0565aab9b33c3e592109ddee4aa965f162b4d3950a7900584994dba2994cc07602d75e36b0a53bf8dbbfcd23ec
+  data.tar.gz: f871e9711752e70ac48956663402fa3f0cdf2693ae08eff8aa0ac59384a0081e1dd27fdbac01a3a17964de4a135a9ded98d381808cace9d9dd62c6ac37d20a8d

data/README.md

@@ -404,7 +404,7 @@ deb_build_mirrors:
 # Who is packaging. Turns up in various packaging artifacts
 packager: 'puppetlabs'
 # GPG key ID of the signer
-gpg_key: '7F438280EF8D349F'
+gpg_key: '4528B6CD9E61EF26'
 # Whether to require tarball signing as a prerequisite of other package building
 sign_tar: false
 # a space separated list of mock configs. These are the rpm distributions to package for. If a noarch package, only one arch of each is needed.

data/lib/packaging/archive.rb

@@ -33,7 +33,7 @@ module Pkg::Archive
       sudo chmod g+w -R #{Pkg::Config.yum_archive_path}
       mv #{full_directory} #{archive_path}
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Move directories from freight path (aka repo staging path) to archive staging paths
@@ -60,7 +60,7 @@ module Pkg::Archive
         mv $pool_directory /opt/tmp-apt
       done
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Move downloads directories to archive staging path
@@ -85,7 +85,7 @@ module Pkg::Archive
       sudo chmod g+w -R #{Pkg::Config.downloads_archive_path}
       mv #{full_directory} #{archive_path}
     CMD
-    Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+    Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
   end
 
   # Delete empty directories from repo paths on weth
@@ -104,7 +104,7 @@ module Pkg::Archive
           fi
         done
       CMD
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 
@@ -112,7 +112,7 @@ module Pkg::Archive
   def remove_dead_symlinks
     base_paths.each do |path|
       command = "find #{path} -xtype l -delete"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 
@@ -120,7 +120,7 @@ module Pkg::Archive
   def delete_staged_archives
     archive_paths.each do |archive_path|
       command = "sudo rm -rf #{File.join(archive_path, '*')}"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, command)
+      Pkg::Util::Net.remote_execute(Pkg::Config.staging_server, command)
     end
   end
 end

data/lib/packaging/config.rb

@@ -6,6 +6,7 @@ module Pkg
   #
   class Config
     require 'packaging/config/params.rb'
+    require 'packaging/config/validations.rb'
     require 'yaml'
 
     class << self
@@ -80,7 +81,11 @@ module Pkg
         dir = "/opt/jenkins-builds/#{self.project}/#{self.ref}"
         cmd = "if [ -s \"#{dir}/artifacts\" ]; then cd #{dir};"\
               "find ./artifacts/ -mindepth 2 -type f; fi"
-        artifacts, _ = Pkg::Util::Net.remote_ssh_cmd(self.builds_server, cmd, true)
+        artifacts, _ = Pkg::Util::Net.remote_execute(
+                     self.builds_server,
+                     cmd,
+                     { capture_output: true }
+                   )
 
         artifacts = artifacts.split("\n")
         data = {}
@@ -389,6 +394,31 @@ module Pkg
         end
       end
 
+      ##
+      #   Ask for validation of BUILD_PARAMS
+      #
+      #   Issued as warnings initially but the intent is to turn this into
+      #   a failure.
+      #
+      def perform_validations
+        error_count = 0
+        Pkg::Params::VALIDATIONS.each do |v|
+          variable_name = v[:var]
+          variable_value = self.instance_variable_get("@#{v[:var]}")
+          validations = v[:validations]
+          validations.each do |validation|
+            unless Pkg::ConfigValidations.send(validation, variable_value)
+              warn "Warning: variable \"#{variable_name}\" failed validation \"#{validation}\""
+              error_count += 1
+            end
+          end
+        end
+
+        if error_count != 0
+          warn "Warning: #{error_count} validation failure(s)."
+        end
+      end
+
       def string_to_array(str)
         delimiters = /[,\s;]/
         return str if str.respond_to?('each')

data/lib/packaging/config/params.rb

@@ -363,6 +363,7 @@ module Pkg::Params
                     { :oldvar => :yum_host,               :newvar => :tar_host },
                  ]
 
+
   # These are variables that we have deprecated. If they are encountered in a
   # project's config, we issue deprecations for them.
   #
@@ -373,4 +374,14 @@ module Pkg::Params
                   { :var => :gpg_name, :message => "
     DEPRECATED, 29-Jul-2014: 'gpg_name' has been replaced with 'gpg_key'.
                    Please update this field in your build_defaults.yaml" }]
+
+  # Provide an open-ended template for validating BUILD_PARAMS.
+  #
+  # Each validatation contains the variable name as ':var' and a list of validations it
+  # must pass from the Pkg::Params::Validations class.
+  #
+  VALIDATIONS = [
+    { :var => :project, :validations => [:not_empty?] }
+  ]
+
 end

data/lib/packaging/config/validations.rb

@@ -0,0 +1,13 @@
+module Pkg
+  class ConfigValidations
+
+    class << self
+
+      # As a validation, this one is kindof lame but is intended as a seed pattern for possibly
+      # more robust ones.
+      def not_empty?(value)
+        value.to_s.empty? ? false : true
+      end
+    end
+  end
+end

data/lib/packaging/deb/repo.rb

@@ -125,7 +125,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
       command = repo_creation_command(File.join(artifact_directory, 'repos'), artifact_paths)
 
       begin
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
         # Now that we've created our package repositories, we can generate repo
         # configurations for use with downstream jobs, acceptance clients, etc.
         Pkg::Deb::Repo.generate_repo_configs
@@ -134,7 +134,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
         Pkg::Deb::Repo.ship_repo_configs
       ensure
         # Always remove the lock file, even if we've failed
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
       end
     end
 
@@ -146,7 +146,7 @@ Description: Apt repository for acceptance testing" >> conf/distributions ; )
 
       Pkg::Util::RakeUtils.invoke_task("pl:fetch")
       repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/deb"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         Pkg::Util::Net.rsync_to("pkg/#{target}/deb/", Pkg::Config.distribution_server, repo_dir)
       end
@@ -252,11 +252,11 @@ SignWith: #{Pkg::Config.gpg_key}"
       rsync_command = repo_deployment_command(apt_path, destination_staging_path, destination_server, dryrun)
       cp_command = repo_deployment_command(destination_staging_path, apt_path, nil, dryrun)
 
-      Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
+      Pkg::Util::Net.remote_execute(origin_server, rsync_command)
       if dryrun
         puts "[DRYRUN] not executing #{cp_command} on #{destination_server}"
       else
-        Pkg::Util::Net.remote_ssh_cmd(destination_server, cp_command)
+        Pkg::Util::Net.remote_execute(destination_server, cp_command)
       end
     end
 

data/lib/packaging/paths.rb

@@ -16,9 +16,9 @@ module Pkg::Paths
     if source_formats.find { |fmt| path =~ /#{fmt}$/ }
       return Pkg::Platforms.get_attribute_for_platform_version(platform, version, :source_architecture)
     end
-    arches.find { |a| path.include?(a) } || arches[0]
+    arches.find { |a| path.include?(package_arch(platform, a)) } || arches[0]
   rescue
-    arches.find { |a| path.include?(a) } || arches[0]
+    arches.find { |a| path.include?(package_arch(platform, a)) } || arches[0]
   end
 
   # Given a path to an artifact, divine the appropriate platform tag associated
@@ -117,7 +117,7 @@ module Pkg::Paths
 
       # In puppet7 and beyond, we moved the repo_name to the top to allow each
       # puppet major release to have its own apt repo.
-      if %w(puppet7 puppet7-nightly).include? repo_name
+      if %w(FUTURE-puppet7 FUTURE-puppet7-nightly).include? repo_name
         return File.join(prefix, apt_repo_name(is_nonfinal), debian_code_name)
       end
 
@@ -313,13 +313,17 @@ module Pkg::Paths
     # In puppet7 and beyond, we moved the puppet major version to near the top to allow each
     # puppet major release to have its own apt repo, for example:
     # /opt/repository/apt/puppet7/pool/bionic/p/puppet-agent
-    if %w(puppet7 puppet7-nightly).include? repo_name
+    if %w(FUTURE-puppet7 FUTURE-puppet7-nightly).include? repo_name
       return File.join(remote_repo_path, repo_name, 'pool', code_name, project[0], project)
     end
 
     # For repos prior to puppet7, the puppet version was part of the repository
     # For example: /opt/repository/apt/pool/bionic/puppet6/p/puppet-agent
-    if %w(puppet puppet5 puppet6 puppet-nightly puppet5-nightly puppet6-nightly).include? repo_name
+    if %w(puppet7 puppet7-nightly
+          puppet6 puppet6-nightly
+          puppet5 puppet5-nightly
+          puppet  puppet-nightly
+          puppet-tools).include? repo_name
       return File.join(remote_repo_path, 'pool', code_name, repo_name, project[0], project)
     end
 
@@ -358,4 +362,15 @@ module Pkg::Paths
     return base_component if component_qualifier == 'repos'
     return full_component
   end
+
+  #for ubuntu-20.04-aarch64, debian package architecture is arm64
+  def package_arch(platform, arch)
+    if platform == 'ubuntu' && arch == 'aarch64'
+      return 'arm64'
+    end
+    arch
+  end
+
+  private :package_arch
+
 end

data/lib/packaging/platforms.rb

@@ -12,13 +12,6 @@ module Pkg
     # Each element in this hash
     PLATFORM_INFO = {
       'aix' => {
-        '6.1' => {
-          architectures: ['power'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          repo: false,
-        },
         '7.1' => {
           architectures: ['power'],
           source_architecture: 'SRPMS',
@@ -28,25 +21,6 @@ module Pkg
         },
       },
 
-      'cisco-wrlinux' => {
-        '5' => {
-          architectures: ['x86_64'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v4',
-          repo: true,
-        },
-        '7' => {
-          architectures: ['x86_64'],
-          source_architecture: 'SRPMS',
-          package_format: 'rpm',
-          source_package_formats: ['src.rpm'],
-          signature_format: 'v4',
-          repo: true,
-        },
-      },
-
       'debian' => {
         '8' => {
           codename: 'jessie',
@@ -100,7 +74,7 @@ module Pkg
           repo: true,
         },
         '8' => {
-          architectures: ['x86_64', 'aarch64'],
+          architectures: ['x86_64', 'ppc64le', 'aarch64'],
           source_architecture: 'SRPMS',
           package_format: 'rpm',
           source_package_formats: ['src.rpm'],
@@ -109,14 +83,6 @@ module Pkg
         }
       },
 
-      'eos' => {
-        '4' => {
-          architectures: ['i386'],
-          package_format: 'swix',
-          repo: false,
-        },
-      },
-
       'fedora' => {
         '30' => {
           architectures: ['x86_64'],
@@ -142,6 +108,14 @@ module Pkg
           signature_format: 'v4',
           repo: true,
         },
+        '34' => {
+          architectures: ['x86_64'],
+          source_architecture: 'SRPMS',
+          package_format: 'rpm',
+          source_package_formats: ['src.rpm'],
+          signature_format: 'v4',
+          repo: true,
+        },
       },
 
       'osx' => {
@@ -160,6 +134,11 @@ module Pkg
           package_format: 'dmg',
           repo: false,
         },
+        '11' => {
+          architectures: ['x86_64'],
+          package_format: 'dmg',
+          repo: false,
+        },
       },
 
       'redhatfips' => {
@@ -248,7 +227,7 @@ module Pkg
         },
         '20.04' => {
           codename: 'focal',
-          architectures: ['amd64'],
+          architectures: ['amd64', 'aarch64'],
           source_architecture: 'source',
           package_format: 'deb',
           source_package_formats: DEBIAN_SOURCE_FORMATS,

data/lib/packaging/repo.rb

@@ -110,7 +110,11 @@ module Pkg::Repo
       cmd = "[ -d #{artifact_directory} ] || exit 1 ; "
       cmd << "pushd #{artifact_directory} > /dev/null && "
       cmd << "find . -name '*.#{pkg_ext}' -print0 | xargs --no-run-if-empty -0 -I {} dirname {} "
-      stdout, stderr = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd, true)
+      stdout, _ = Pkg::Util::Net.remote_execute(
+                Pkg::Config.distribution_server,
+                cmd,
+                { capture_output: true }
+              )
       return stdout.split
     rescue => e
       fail "Error: Could not retrieve directories that contain #{pkg_ext} packages in #{Pkg::Config.distribution_server}:#{artifact_directory}"
@@ -120,7 +124,7 @@ module Pkg::Repo
       cmd = "[ -d #{artifact_parent_directory}/artifacts ] || exit 1 ; "
       cmd << "pushd #{artifact_parent_directory} > /dev/null && "
       cmd << 'rsync --archive --verbose --one-file-system --ignore-existing artifacts/ repos/ '
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, cmd)
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, cmd)
     rescue => e
       fail "Error: Could not populate repos directory in #{Pkg::Config.distribution_server}:#{artifact_parent_directory}"
     end
@@ -143,7 +147,9 @@ module Pkg::Repo
         __APT_PLATFORMS__: Pkg::Config.apt_releases.join(' '),
         __GPG_KEY__: Pkg::Util::Gpg.key
       }
-      Pkg::Util::Net.remote_ssh_cmd(remote_host, Pkg::Util::Misc.search_and_replace(command, whitelist))
+      Pkg::Util::Net.remote_execute(
+        remote_host,
+        Pkg::Util::Misc.search_and_replace(command, whitelist))
     end
   end
 end

data/lib/packaging/rpm/repo.rb

@@ -16,7 +16,7 @@ module Pkg::Rpm::Repo
 
       Pkg::Util::RakeUtils.invoke_task("pl:fetch")
       repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/#{target}/rpm"
-      Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
       Pkg::Util::Execution.retry_on_fail(:times => 3) do
         Pkg::Util::Net.rsync_to("pkg/#{target}/rpm/", Pkg::Config.distribution_server, repo_dir)
       end
@@ -215,7 +215,7 @@ module Pkg::Rpm::Repo
       command = Pkg::Rpm::Repo.repo_creation_command(File.join(artifact_directory, directory), artifact_paths)
 
       begin
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, command)
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, command)
         # Now that we've created our package repositories, we can generate repo
         # configurations for use with downstream jobs, acceptance clients, etc.
         Pkg::Rpm::Repo.generate_repo_configs
@@ -224,7 +224,7 @@ module Pkg::Rpm::Repo
         Pkg::Rpm::Repo.ship_repo_configs
       ensure
         # Always remove the lock file, even if we've failed
-        Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
+        Pkg::Util::Net.remote_execute(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock")
       end
     end
 
@@ -248,7 +248,7 @@ module Pkg::Rpm::Repo
     def deploy_repos(yum_path, origin_server, destination_server, dryrun = false)
       rsync_command = repo_deployment_command(yum_path, yum_path, destination_server, dryrun)
 
-      Pkg::Util::Net.remote_ssh_cmd(origin_server, rsync_command)
+      Pkg::Util::Net.remote_execute(origin_server, rsync_command)
     end
   end
 end

data/lib/packaging/sign/dmg.rb

@@ -15,10 +15,10 @@ module Pkg::Sign::Dmg
     work_dir  = "/tmp/#{Pkg::Util.rand_string}"
     mount     = File.join(work_dir, "mount")
     signed    = File.join(work_dir, "signed")
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, "mkdir -p #{mount} #{signed}")
+    Pkg::Util::Net.remote_execute(ssh_host_string, "mkdir -p #{mount} #{signed}")
     dmgs = Dir.glob("#{target_dir}/apple/**/*.dmg")
     Pkg::Util::Net.rsync_to(dmgs.join(" "), rsync_host_string, work_dir)
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, %Q[for dmg in #{dmgs.map { |d| File.basename(d, ".dmg") }.join(" ")}; do
+    Pkg::Util::Net.remote_execute(ssh_host_string, %Q[for dmg in #{dmgs.map { |d| File.basename(d, ".dmg") }.join(" ")}; do
       /usr/bin/hdiutil attach #{work_dir}/$dmg.dmg -mountpoint #{mount} -nobrowse -quiet ;
       /usr/bin/security -q unlock-keychain -p "#{Pkg::Config.osx_signing_keychain_pw}" "#{Pkg::Config.osx_signing_keychain}" ;
         for pkg in $(ls #{mount}/*.pkg | xargs -n 1 basename); do
@@ -36,6 +36,6 @@ module Pkg::Sign::Dmg
     dmgs.each do | dmg |
       Pkg::Util::Net.rsync_from("#{work_dir}/#{File.basename(dmg)}", rsync_host_string, File.dirname(dmg))
     end
-    Pkg::Util::Net.remote_ssh_cmd(ssh_host_string, "if [ -d '#{work_dir}' ]; then rm -rf '#{work_dir}'; fi")
+    Pkg::Util::Net.remote_execute(ssh_host_string, "if [ -d '#{work_dir}' ]; then rm -rf '#{work_dir}'; fi")
   end
 end