simp-rake-helpers 5.12.3 → 5.12.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33beb8e656840401f0fb3a07f7aee982fe3931d53b2e233f2e0024398fd03d6c
-  data.tar.gz: 81111ce62e4e8b3ca03a478d765e3944ee8b297af7b2ee1ef5e182e8bacf36f3
+  metadata.gz: 97111c5cebe9473b33a3ea7950af3cb1b26a48a67d46485435473bad9660cc46
+  data.tar.gz: b5e86d3cfad3b1a21300fef1ae99c81f38c87a7c41ce6304e9ceb2e48c1e6f5e
 SHA512:
-  metadata.gz: 9589c0d228dc77b2e75a641f158ca878c56498566696039c71eca0c414078299c2a7c4951cf48d45e0388b9cb72a1615321381084a11837fbbc42259a65ea880
-  data.tar.gz: 4267702fa365999433e673f26ff0896ad9a088ea7ce7ae26fff38bbc4d9c37b1c6239956bc016644ea8b2aa3b10925989f44fbb178a3d611380eb79db4b5cc6e
+  metadata.gz: eb8f742f59d39f13a22b79781a69c0ac2855bd8a57e1928e8cc8426d958b6f2724a53e9182cd7bc4139a808308cdeb668334fc707661f6a4576601a5ccbae191
+  data.tar.gz: 8b1e9f3584bec5a5c29397802d3c8048ea9552148934f230c15c0bdd226ca763c3a97801c8e98abdc45110bdc6527be68ba504ccd7e0c34c984322bb6b11c6a9

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+### 5.12.7 / 2021-10-26
+- Added:
+  - Env var `SIMP_PKG_progress_bar=no` to turn off pkg RPM build progress bars
+  - Env var `SIMP_PKG_fetch_published_rpm=no` to prevent downloading a
+    published package
+- Fixed:
+  - RPM builds no longer fail with leftover generated
+    `build/rpm_metadata/releases` files
+  - Removed unused code, tidied up some confusing bits to make structure more
+    obvious
+
+### 5.12.6 / 2021-10-19
+- Ensure that the `Updates` directory does not link to files in itself
+
+### 5.12.5 / 2021-10-07
+- Fixed a bug where `build:auto` failed when building the SIMP ISO for EL7,
+  because the code attempted to move a directory onto itself.
+- Ensured GPG keys in simp-gpgkeys are available in the DVD overlay tar file
+
+### 5.12.4 / 2021-10-05
+- Ensure that the DIST qualifier is added to all built RPMs
+- Use the new SimpRepos directory layout when building an ISO using externally
+  copied repos.
+
 ### 5.12.3 / 2021-09-15
 - Handle multiple options for required applications in build:auto
 - Allow users to populate a `reposync` directory in the YUM build space that

data/lib/simp/rake/build/build.rb

@@ -282,7 +282,7 @@ module Simp::Rake::Build
         Error: Could not find anything to do!
 
         In #{target_dir}:
-            No packages in either packages.yaml or the packages/ directory
+            No packages in either packages.yaml or the packages/ directory and no repos found in the reposync/ directory
                 EOM
               end
 

data/lib/simp/rake/build/iso.rb

@@ -190,12 +190,20 @@ module Simp::Rake::Build
                 end
               end
 
+
+              repo_target_dir = dir
+
               # If we've pulled in reposync directories, we expect them to
               # completely overwrite the directory of the same name in the
               # target ISO so no pruning is required
               #
               # Note: CASE MATTERS on the directory names
               if reposync_active
+                # We're working with the new EL8+ layout, so we need to target
+                # the SimpRepos subdirectory
+                repo_target_dir = File.join(dir,'SimpRepos')
+                mkdir_p(repo_target_dir, :verbose => verbose)
+
                 repos_to_overwrite = Dir.glob(File.join(reposync_location, '*'))
                   .delete_if{|x| !File.directory?(x)}
                   .map{|x| File.basename(x)}
@@ -204,13 +212,18 @@ module Simp::Rake::Build
                   src = File.join(reposync_location, repo)
                   target = File.join(dir, repo)
 
-                  rm_rf(target, :verbose => verbose) if File.directory?(target)
-                  cp_r(src, dir, :verbose => verbose)
+                  if File.directory?(target)
+                    rm_rf(target, :verbose => verbose) if File.directory?(target)
+                  else
+                    target = File.join(repo_target_dir, repo)
+                  end
+
+                  cp_r(src, target, :verbose => verbose)
                 end
               else
                 # Prune unwanted packages
                 begin
-                  system("tar --no-same-permissions -C #{dir} -xzf #{tball} *simp_pkglist.txt")
+                  system("tar --no-same-permissions -C #{repo_target_dir} -xzf #{tball} *simp_pkglist.txt")
                 rescue
                   # Does not matter if the command fails
                 end
@@ -235,14 +248,25 @@ module Simp::Rake::Build
                     next if line =~ /^(\s+|#.*)$/
                     exclude_pkgs.push(line.chomp)
                   end
-                  prune_packages(dir,['SIMP'],exclude_pkgs,mkrepo)
+                  prune_packages(dir,['SIMP','SimpRepos'],exclude_pkgs,mkrepo)
                 end
               end
 
               # Add the SIMP code
               system("tar --no-same-permissions -C #{dir} -xzf #{tball}")
 
-              Dir.chdir("#{dir}/SIMP") do
+              # Pop the SIMP directory from the tarball into the correct spot
+              # FIXME: This is a hack
+              unless dir == repo_target_dir
+                simpdir = File.join(dir,'SIMP')
+
+                if File.directory?(simpdir)
+                   cp_r(simpdir, repo_target_dir, :verbose => verbose)
+                   rm_rf(simpdir, :verbose => verbose)
+                end
+              end
+
+              Dir.chdir("#{repo_target_dir}/SIMP") do
                 # Add the SIMP Dependencies
                 simp_base_ver = simpver.split('-').first
 
@@ -285,26 +309,81 @@ module Simp::Rake::Build
                   cp(rpm,rpm_arch, :verbose => verbose)
                 end
 
-                ln_s('noarch', arch, :verbose => verbose) if (!File.directory?(arch) && File.directory?('noarch'))
-                fail("Could not find architecture '#{arch}' in the SIMP distribution") unless (File.directory?(arch) || File.symlink?(arch))
+                if reposync_active
+                    fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+                else
+                  ln_sf('noarch', arch, :verbose => verbose) if (!File.directory?(arch) && File.directory?('noarch'))
+                  fail("Could not find architecture '#{arch}' in the SIMP distribution") unless (File.directory?(arch) || File.symlink?(arch))
+
+                  # Get everything set up properly...
+                  Dir.chdir(arch) do
+                    Dir.glob('../*') do |rpm_dir|
+                      # Don't dive into ourselves
+                      next if File.basename(rpm_dir) == arch
+
+                      Dir.glob(%(#{rpm_dir}/*.rpm)) do |source_rpm|
+                        link_target = File.basename(source_rpm)
+                        if File.exist?(source_rpm) && File.exist?(link_target)
+                          next if Pathname.new(source_rpm).realpath == Pathname.new(link_target).realpath
+                        end
+
+                        ln_sf(source_rpm,link_target, :verbose => verbose)
+                      end
+                    end
+
+                    fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+                  end
+                end
+              end
+
+              ### Munge the Repos
 
-                # Get everything set up properly...
-                Dir.chdir(arch) do
-                  Dir.glob('../*') do |rpm_dir|
-                    # Don't dive into ourselves
-                    next if File.basename(rpm_dir) == arch
+              # Create an Updates directory that is properly populated
+              updates_readme = <<~README
+                This directory houses updates to NON-MODULAR RPMs.
 
-                    Dir.glob(%(#{rpm_dir}/*.rpm)) do |source_rpm|
-                      link_target = File.basename(source_rpm)
-                      if File.exist?(source_rpm) && File.exist?(link_target)
-                        next if Pathname.new(source_rpm).realpath == Pathname.new(link_target).realpath
-                      end
+                DO NOT put modular RPMs in this directory or you will break your
+                system updates!
+                README
+
+              updates_dir = File.join(dir, 'Updates')
+              mkdir_p(updates_dir)
+
+              Dir.chdir(updates_dir) do
+                File.open('README','w'){|fh| fh.puts(updates_readme) }
+
+                repos = Dir.glob(File.join('..','**','repodata'))
+                modular_repos = Dir.glob(File.join('..','**','repodata','*-modules.*'))
+                non_modular_repos = repos.select{|x| modular_repos.grep(%r{^#{Regexp.escape(x)}}).empty? }
+                non_modular_repos.map!{|x| File.split(x).first}
+                non_modular_repos.delete_if{|x| x.match(%r{/SimpRepos|/SIMP}) }
+                non_modular_repos.each do |non_modular_repo|
+                  Dir.glob(File.join(non_modular_repo, '**', '*.rpm')).each do |rpm|
+                    # when non_modular_repo is '..', can still find RPMs we need
+                    # to exclude
+                    next if rpm.match(%r{/SimpRepos|/SIMP})
 
-                      ln_sf(source_rpm,link_target, :verbose => verbose)
+                    rpm_dest = File.basename(rpm)
+                    if File.exist?(rpm) && File.exist?(rpm_dest)
+                      next if (File.realpath(rpm) == File.realpath(rpm_dest))
                     end
+
+                    ln_sf(rpm, rpm_dest, :verbose => verbose)
                   end
+                end
+
+                fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+              end
 
-                  fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+              # New ISO Layout
+              if reposync_active
+                Dir.chdir(repo_target_dir) do
+                  gpgkeysdir = File.join('SIMP','GPGKEYS')
+
+                  if File.directory?(gpgkeysdir)
+                    cp_r(gpgkeysdir, '.', :verbose => verbose)
+                    rm_rf(gpgkeysdir, :verbose => verbose)
+                  end
                 end
               end
 

data/lib/simp/rake/build/pkg.rb

@@ -21,13 +21,17 @@ module Simp::Rake::Build
       @rpm_build_metadata = 'last_rpm_build_metadata.yaml'
       @rpm_dependency_file = File.join(@base_dir, 'build', 'rpm', 'dependencies.yaml')
       @build_keys_dir = ENV.fetch('SIMP_PKG_build_keys_dir', File.join(@base_dir, '.dev_gpgkeys'))
-      @long_gpg_socket_err_msg = <<~EOM
+      @long_gpg_socket_err_msg = <<~MSG
         If the problem is 'socket name <xxx> is too long', use SIMP_PKG_build_keys_dir
         to override
         #{@build_keys_dir}
         with a shorter path. The socket name must be < 108 characters.
 
-      EOM
+      MSG
+      # nil = default not set; use local preference
+      @fetch_published_rpm_default = ENV['SIMP_PKG_fetch_published_rpm'] ? (
+        ENV['SIMP_PKG_fetch_published_rpm'] =~ /\A(yes|true)\Z/i ? true : false
+      ) : nil
 
       define_tasks
     end
@@ -114,7 +118,7 @@ module Simp::Rake::Build
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Prepare a GPG signing key to sign build packages
 
             * :key - the name of the build keys subdirectory to prepare
@@ -142,7 +146,7 @@ module Simp::Rake::Build
           ENV vars:
             - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
             - Set `SIMP_PKG_build_keys_dir` to override the default build keys path.
-        EOM
+        DESC
         task :key_prep,[:key] => [:prep] do |t,args|
           args.with_defaults(:key => 'dev')
           key = args.key
@@ -251,7 +255,7 @@ module Simp::Rake::Build
 
 
 =begin
-        desc <<-EOM
+        desc <<~DESC
           Build the entire SIMP release.
 
             * :docs - Build the docs. Set this to false if you wish to skip building the docs.
@@ -261,7 +265,7 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
                 build-specific YUM cache
-        EOM
+        DESC
 =end
         task :build,[:docs,:key] => [:prep,:key_prep] do |t,args|
           args.with_defaults(:key => 'dev')
@@ -269,12 +273,7 @@ module Simp::Rake::Build
 
           check_dvd_env
 
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
-              ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
-          rescue Simp::YUM::Error
-          end
+          get_yum_helper
 
           Rake::Task['pkg:aux'].invoke
           if "#{args.docs}" == 'true'
@@ -287,7 +286,7 @@ module Simp::Rake::Build
           Rake::Task['pkg:signrpms'].invoke(args[:key])
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the Puppet module RPMs.
 
             * :method - The Puppetfile from which the repository information
@@ -295,12 +294,12 @@ module Simp::Rake::Build
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :modules,[:method] => [:prep] do |t,args|
           build(@build_dirs[:modules],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build a single Puppet Module RPM.
 
             * :name   - The path, or name, of the module to build. If a name is
@@ -314,11 +313,12 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
                 build-specific YUM cache
-        EOM
+        DESC
         task :single,[:name,:method] => [:prep] do |t,args|
           fail("You must pass :name to '#{t.name}'") unless args[:name]
 
           mod_path = File.absolute_path(args[:name])
+          get_yum_helper
 
           if args[:name].include?('/')
             fail("'#{args[:name]}' does not exist!") unless File.directory?(mod_path)
@@ -333,35 +333,27 @@ module Simp::Rake::Build
             mod_path = local_module[:path]
           end
 
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
-              ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
-          rescue Simp::YUM::Error
-          end
-
-          ENV['SIMP_PKG_rand_name'] = 'yes'
           build(Array(mod_path), t)
 
           puts("Your packages can be found in '#{mod_path}/dist'")
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the SIMP non-module RPMs.
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :aux => [:prep]  do |t,args|
           build(@build_dirs[:aux],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the SIMP documentation.
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :doc => [:prep] do |t,args|
           # Need to make sure that the docs have the version updated
           # appropriately prior to building
@@ -373,7 +365,7 @@ module Simp::Rake::Build
           build(@build_dirs[:doc],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Sign a set of RPMs.
 
             Signs any unsigned RPMs in the specified directory
@@ -394,7 +386,7 @@ module Simp::Rake::Build
             * Set `SIMP_PKG_rpmsign_timeout` to override the maximum time in seconds
               to wait for an individual RPM signing operation to complete.
               - Defaults to 60 seconds.
-        EOM
+        DESC
         task :signrpms,[:key,:rpm_dir,:force,:digest_algo] => [:prep,:key_prep] do |t,args|
           require 'simp/rpm_signer'
 
@@ -455,13 +447,13 @@ module Simp::Rake::Build
         end
 
 =begin
-        desc <<-EOM
+        desc <<~DESC
           Check that RPMs are signed.
 
             Checks all RPM files in a directory to see if they are trusted.
               * :rpm_dir - A directory containing RPM files to check. Default #{@build_dir}/SIMP
               * :key_dir - The path to the GPG keys you want to check the packages against. Default #{@src_dir}/assets/gpgkeys/
-        EOM
+        DESC
 =end
         task :checksig,[:rpm_dir,:key_dir] => [:prep] do |t,args|
           begin
@@ -547,7 +539,7 @@ module Simp::Rake::Build
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Run repoclosure on RPM files.
 
             Finds all rpm files in the target dir and all of its subdirectories, then
@@ -561,7 +553,7 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_PKG_repoclose_pe=yes` to enable repoclosure on PE-related RPMs.
 
-        EOM
+        DESC
         task :repoclosure,[:target_dir,:aux_dir] => [:prep] do |t,args|
           default_target = @pkg_dirs[:simp]
           args.with_defaults(:target_dir => File.expand_path(default_target))
@@ -572,31 +564,31 @@ module Simp::Rake::Build
 
           _repoclose_pe = ENV.fetch('SIMP_PKG_repoclose_pe','no') == 'yes'
 
-          yum_conf_template = <<-EOF
-[main]
-keepcache=0
-exactarch=1
-obsoletes=1
-gpgcheck=0
-plugins=1
-installonly_limit=5
-<% unless #{_repoclose_pe} -%>
-exclude=*-pe-*
-<% end -%>
-
-<% repo_files.each do |repo| -%>
-include=file://<%= repo %>
-<% end -%>
-  EOF
-
-          yum_repo_template = <<-EOF
-[<%= repo_name %>]
-name=<%= repo_name %>
-baseurl=file://<%= repo_path %>
-enabled=1
-gpgcheck=0
-protect=1
-  EOF
+          yum_conf_template = <<~YUM_CONF
+            [main]
+            keepcache=0
+            exactarch=1
+            obsoletes=1
+            gpgcheck=0
+            plugins=1
+            installonly_limit=5
+            <% unless #{_repoclose_pe} -%>
+            exclude=*-pe-*
+            <% end -%>
+
+            <% repo_files.each do |repo| -%>
+            include=file://<%= repo %>
+            <% end -%>
+          YUM_CONF
+
+          yum_repo_template = <<~YUM_REPO
+            [<%= repo_name %>]
+            name=<%= repo_name %>
+            baseurl=file://<%= repo_path %>
+            enabled=1
+            gpgcheck=0
+            protect=1
+          YUM_REPO
 
           fail("#{args[:target_dir]} does not exist!") unless File.directory?(args[:target_dir])
 
@@ -610,7 +602,7 @@ protect=1
                 .delete_if{|x| x =~ /\.src\.rpm$/}
                 .each do |path|
                   sym_path = "repos/base/#{File.basename(path)}"
-                  ln_s(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
+                  ln_sf(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
               end
 
               if args[:aux_dir]
@@ -618,7 +610,7 @@ protect=1
                   .delete_if{|x| x =~ /\.src\.rpm$/}
                   .each do |path|
                     sym_path = "repos/lookaside/#{File.basename(path)}"
-                    ln_s(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
+                    ln_sf(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
                 end
               end
 
@@ -643,7 +635,8 @@ protect=1
                 file.write(ERB.new(yum_conf_template,nil,'-').result(binding))
               end
 
-              if which('dnf')
+              dnf_system = which('dnf')
+              if dnf_system
                 cmd = 'repoclosure -c base.conf --disablerepo=* --enablerepo=base'
               else
                 cmd = 'repoclosure -c repodata -n -t -r base -l lookaside -c yum.conf'
@@ -661,15 +654,24 @@ protect=1
               if (!$?.success? || (repoclosure_output =~ /nresolved/))
                 errmsg = ['Error: REPOCLOSURE FAILED:']
                 errmsg << [repoclosure_output]
-                fail(errmsg.join("\n"))
+                puts(errmsg.join("\n"))
+
+                if dnf_system
+                  if ENV.fetch('SIMP_BUILD_prompt', 'yes') != 'no'
+                    puts('- Press any key to continue or ^C to abort -')
+                    $stdin.gets
+                  end
+                else
+                  fail('Repoclosure Failed')
+                end
               end
             end
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Print published status of all project RPMs
-        EOM
+        DESC
         task :check_published => [:prep] do |t,args|
           begin
             yum_helper = Simp::YUM.new(
@@ -687,7 +689,12 @@ protect=1
 
             if Dir.exist?(dir)
               begin
-                require_rebuild?(dir, yum_helper, { :verbose => true, :check_git => true, :prefix => '' })
+                require_rebuild?(dir, yum_helper, {
+                  fetch:      false,
+                  verbose:    true,
+                  check_git:  true,
+                  prefix:     ''
+                })
               rescue => e
                 _errmsg = "Error: require_rebuild?(): Status check failed on '#{dir}' => #{e}"
               end
@@ -714,9 +721,24 @@ protect=1
           return (0...24).map{ (65 + rand(26)).chr }.join.downcase
         end
 
+        # Return a SIMP::YUM object configured for the local build dir or nil
+        #
+        # ENV vars:
+        #   - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
+        #   - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
+        #     build-specific YUM cache
+        def get_yum_helper( makecache = (ENV.fetch('SIMP_YUM_makecache','yes') == 'yes') )
+          begin
+            build_yum_data = File.join(@distro_build_dir, 'yum_data')
+            return Simp::YUM.new(Simp::YUM.generate_yum_conf(build_yum_data), makecache)
+          rescue Simp::YUM::Error => e
+            $stderr.puts "Simp::YUM::Error: #{e}" if @verbose
+          end
+        end
+
         # Check and see if 'dir' requires a rebuild based on published packages
         #
-        # If 'fetch' is true => Download the RPM
+        # If 'fetch' is true => Download the published RPM
         # If 'verbose' is true => Print helpful information to stderr
         # If 'check_git' is true => Print the git tag status if 'verbose' is true
         # 'prefix' is used to prepend verbose messages
@@ -725,7 +747,13 @@ protect=1
         # - Method is too long
         # - Method needs to be passed in class variables (@xxx) so that it
         #   can be pulled out into a library that is easily unit-testable
-        def require_rebuild?(dir, yum_helper, opts={ :unique_namespace => generate_namespace, :fetch => false, :verbose => @verbose, :check_git => false, :prefix => '' })
+        def require_rebuild?(dir, yum_helper, opts={
+          unique_namespace:  generate_namespace,
+          fetch:             false,
+          verbose:           @verbose,
+          check_git:         false,
+          prefix:            ''
+        })
           result = false
           rpm_metadata = File.exist?(@rpm_dependency_file) ? YAML.load(File.read(@rpm_dependency_file)) : {}
           dir_relpath = Pathname.new(dir).relative_path_from(Pathname.new(Dir.pwd)).to_path
@@ -860,7 +888,8 @@ protect=1
                     else
                       $stderr.puts "#{opts[:prefix]}Found Existing Remote RPM: #{published_rpm}" if opts[:verbose]
                       if opts[:fetch]
-                        # Download remote RPM, unless already downloaded it.
+                        # Download published RPM, unless it's already been downloaded.
+                        # TODO Unhandled edge case: Validate that downloaded RPM is valid
                         if File.exist?(File.join('dist', published_rpm))
                           $stderr.puts "#{opts[:prefix]}#{published_rpm} previously downloaded" if opts[:verbose]
                         else
@@ -904,101 +933,107 @@ protect=1
           return result
         end
 
+        def build_pupmod_rpm(dir, yum_helper, dbg_prefix = '  ')
+          unique_namespace = generate_namespace
+          if require_rebuild?(dir, yum_helper, {
+            unique_namespace:  unique_namespace,
+            fetch:             @fetch_published_rpm.nil? ? true : @fetch_published_rpm,
+            verbose:           @verbose,
+            prefix:            dbg_prefix,
+          })
+            $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' on #{File.basename(dir)}") if @verbose
+            Rake::Task["#{unique_namespace}:pkg:rpm"].invoke
+          else
+            # Record metadata for the downloaded RPM
+            Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
+          end
+          true
+        end
+
+
+        def build_rakefile_rpm(dir, yum_helper, dbg_prefix = '  ')
+          if require_rebuild?(dir, yum_helper, {
+            fetch:    @fetch_published_rpm.nil? ? true : @fetch_published_rpm,
+            verbose:  @verbose,
+            prefix:   dbg_prefix
+          })
+            $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' in #{File.basename(dir)}") if @verbose
+            rake_flags = Rake.application.options.trace ? '--trace' : ''
+            cmd = %{SIMP_BUILD_version=#{@simp_version} rake pkg:rpm #{rake_flags} 2>&1}
+
+            build_success = true
+            begin
+              %x{#{cmd}}
+              build_success = $?.success?
+
+              built_rpm = true
+            rescue
+              build_success = false
+            end
+
+            unless build_success
+              if @verbose
+                $stderr.puts("First 'rake pkg:rpm' attempt for #{dir} failed, running bundle and trying again.")
+              end
+
+              if Bundler.respond_to?(:with_unbundled_env)
+                # Bundler 2.1+
+                clean_env_method = :with_unbundled_env
+                bundle_install_cmd = %{bundle config set with 'development' && bundle install}
+              else
+                # Old Bundler
+                clean_env_method = :with_clean_env
+                bundle_install_cmd = %{bundle install --with development}
+              end
+
+              ::Bundler.send(clean_env_method) do
+                %x{#{bundle_install_cmd}}
+
+                output = %x{#{cmd} 2>&1}
+
+                unless $?.success?
+                  raise("Error in #{dir} running #{cmd}\n#{output}")
+                end
+              end
+            end
+          else
+            # Record metadata for the downloaded RPM
+            Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
+            built_rpm = true
+          end
+        end
+
+
         # Takes a list of directories to hop into and perform builds within
         #
         # The task must be passed so that we can output the calling name in the
         # status bar.
         def build(dirs, task, rebuild_for_arch=false, remake_yum_cache = false)
-          _verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
-          dbg_prefix = '  ' # prefix for debug messages
-
           fail("Could not find RPM dependency file '#{@rpm_dependency_file}'") unless File.exist?(@rpm_dependency_file)
-
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data'))
-            )
-          rescue Simp::YUM::Error
-          end
+          yum_helper = get_yum_helper( remake_yum_cache )
 
           Parallel.map(
             # Allow for shell globs
-            Array(dirs),
-            :in_processes => @cpu_limit,
-            :progress => task.name
+            Array(dirs), {
+              in_processes:  @cpu_limit,
+              progress:      (ENV.fetch('SIMP_PKG_progress_bar','yes') == 'yes') ? task.name : nil,
+            }
           ) do |dir|
             fail("Could not find directory #{dir}") unless Dir.exist?(dir)
 
             Dir.chdir(dir) do
               built_rpm = false
+              $stderr.puts("\nPackaging #{File.basename(dir)}") if @verbose
 
-              if @verbose
-                $stderr.puts("\nPackaging #{File.basename(dir)}")
-              end
-
-              # We're building a module, override anything down there
               if File.exist?('metadata.json')
-                unique_namespace = generate_namespace
-                if require_rebuild?(dir, yum_helper, { :unique_namespace => unique_namespace, :fetch => true, :verbose => @verbose, :prefix => dbg_prefix})
-                  $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' on #{File.basename(dir)}") if @verbose
-                  Rake::Task["#{unique_namespace}:pkg:rpm"].invoke
-                else
-                  # Record metadata for the downloaded RPM
-                  Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
-                end
-
-                built_rpm = true
-
-              # We're building one of the extra assets and should honor its Rakefile
-              # and RPM spec file.
+                # We're building a module, override anything down there
+                built_rpm = build_pupmod_rpm(dir, yum_helper)
               elsif File.exist?('Rakefile')
-                if require_rebuild?(dir, yum_helper, { :fetch => true, :verbose => @verbose, :prefix => dbg_prefix })
-                  $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' in #{File.basename(dir)}") if @verbose
-                  rake_flags = Rake.application.options.trace ? '--trace' : ''
-                  cmd = %{SIMP_BUILD_version=#{@simp_version} rake pkg:rpm #{rake_flags} 2>&1}
-
-                  build_success = true
-                  begin
-                    %x{#{cmd}}
-                    build_success = $?.success?
-
-                    built_rpm = true
-                  rescue
-                    build_success = false
-                  end
-
-                  unless build_success
-                    if @verbose
-                      $stderr.puts("First 'rake pkg:rpm' attempt for #{dir} failed, running bundle and trying again.")
-                    end
-
-                    if Bundler.respond_to?(:with_unbundled_env)
-                      # Bundler 2.1+
-                      clean_env_method = :with_unbundled_env
-                      bundle_install_cmd = %{bundle config set with 'development' && bundle install}
-                    else
-                      # Old Bundler
-                      clean_env_method = :with_clean_env
-                      bundle_install_cmd = %{bundle install --with development}
-                    end
-
-                    ::Bundler.send(clean_env_method) do
-                      %x{#{bundle_install_cmd}}
-
-                      output = %x{#{cmd} 2>&1}
-
-                      unless $?.success?
-                        raise("Error in #{dir} running #{cmd}\n#{output}")
-                      end
-                    end
-                  end
-                else
-                  # Record metadata for the downloaded RPM
-                  Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
-                  built_rpm = true
-                end
+                # We're building one of the extra assets and should honor its Rakefile
+                # and RPM spec file.
+                built_rpm = build_rakefile_rpm(dir, yum_helper)
               else
-                puts "Warning: '#{dir}' could not be built via Rake"
+                puts "Warning: '#{dir}' could not be built (not a pupmod & no Rakefile!)"
               end
 
               if built_rpm
@@ -1014,12 +1049,7 @@ protect=1
 
                 raise("No RPMs generated for #{dir}") if rpms.empty?
               end
-
-              if @verbose
-                rpms = Dir.glob('dist/*.rpm')
-#                $stderr.puts("#{dbg_prefix}RPMS: #{rpms.join("\n#{dbg_prefix}      ")}")
-                $stderr.puts("Finished  #{File.basename(dir)}")
-              end
+              $stderr.puts("Finished  #{File.basename(dir)}") if @verbose
             end
           end
         end

data/lib/simp/rake/build/rpmdeps.rb

@@ -267,7 +267,13 @@ module Simp::Rake::Build::RpmDeps
   def self.release_file_up_to_date?(new_release_info, rpm_release_file)
     return false unless File.exist?(rpm_release_file)
 
-    return File.read(rpm_release_file).strip == new_release_info.strip
+    # remove comments like "# release set by simp-core dependencies.yaml"
+    release_file_content = File.readlines(rpm_release_file).reject{|x| x =~ /^ *#/}.join("\n").strip
+
+    # sanitize numerics, etc.
+    new_release_content = "#{new_release_info}".strip
+
+    return release_file_content == new_release_content
   end
 
   # Generate 'build/rpm_metadata/release' file containing release qualifier

data/lib/simp/rake/build/tar.rb

@@ -51,7 +51,8 @@ module Simp::Rake::Build
               'rubygem-simp-cli',
               'simp',
               'simp-gpgkeys',
-              'simp-utils'
+              'simp-utils',
+              'simp-vendored-r10k'
             ]
           }
 
@@ -83,8 +84,11 @@ module Simp::Rake::Build
             end
 
             unless failures.empty?
-              msg = ['Error: Could not find the following packages:']
-              fail((msg + failures).join("\n"))
+              msg = ['Error: Could not find the following packages:'] +
+                failures +
+                ['Did "dist/logs/last_rpm_build_metadata.yaml" get generated in the build directory?']
+
+              fail(msg.join("\n"))
             end
           end
         end
@@ -126,10 +130,16 @@ module Simp::Rake::Build
 
             Simp::RPM.copy_wo_vcs(@dvd_src,".",base_dir)
 
-            # Copy in the GPG Public Keys
+            # Copy in the GPG Public Keys from @build_dir
             mkdir_p("#{destdir}/GPGKEYS")
             ln(Dir.glob("#{@build_dir}/GPGKEYS/RPM-GPG-KEY*"), "#{destdir}/GPGKEYS", { :force => true })
 
+            # Copy in the GPG Public Keys packaged by simp-gpgkeys
+            simp_gpgkeys = File.join(@src_dir, 'assets', 'gpgkeys', 'GPGKEYS')
+            if Dir.exist?(simp_gpgkeys)
+              ln(Dir.glob("#{simp_gpgkeys}/RPM-GPG-KEY*"), "#{destdir}/GPGKEYS", { :force => true })
+            end
+
             # Copy in the auto-build RPMs
             Dir.chdir("#{@build_dir}/SIMP/RPMS") do
               Dir.glob('*').each do |type|

data/lib/simp/rake/helpers/assets/rpm_spec/simp6.spec

@@ -301,7 +301,7 @@ end
 
 if req_file then
   for line in req_file:lines() do
-    valid_line = (line:match("^Requires: ") or line:match("^Obsoletes: ") or line:match("^Provides: "))
+    valid_line = (line:match("^Requires: ") or line:match("^Obsoletes: ") or line:match("^Provides: ") or line:match("^Recommends: "))
 
     if valid_line then
       module_requires = (module_requires .. "\n" .. line)
@@ -319,7 +319,7 @@ Summary:   %{module_name} Puppet Module
 Name:      %{package_name}
 
 Version:   %{lua: print(package_version)}
-Release:   %{lua: print(package_release)}
+Release:   %{lua: print(package_release)}%{?dist}
 License:   %{lua: print(module_license)}
 Group:     Applications/System
 Source0:   %{package_name}-%{version}-%{release}.tar.gz

data/lib/simp/rake/helpers/assets/rpm_spec/simpdefault.spec

@@ -301,7 +301,7 @@ end
 
 if req_file then
   for line in req_file:lines() do
-    valid_line = (line:match("^Requires: ") or line:match("^Obsoletes: ") or line:match("^Provides: "))
+    valid_line = (line:match("^Requires: ") or line:match("^Obsoletes: ") or line:match("^Provides: ") or line:match("^Recommends: "))
 
     if valid_line then
       module_requires = (module_requires .. "\n" .. line)
@@ -319,7 +319,7 @@ Summary:   %{module_name} Puppet Module
 Name:      %{package_name}
 
 Version:   %{lua: print(package_version)}
-Release:   %{lua: print(package_release)}
+Release:   %{lua: print(package_release)}%{?dist}
 License:   %{lua: print(module_license)}
 Group:     Applications/System
 Source0:   %{package_name}-%{version}-%{release}.tar.gz

data/lib/simp/rake/helpers/version.rb

@@ -2,5 +2,5 @@ module Simp; end
 module Simp::Rake; end
 
 class Simp::Rake::Helpers
-  VERSION = '5.12.3'
+  VERSION = '5.12.7'
 end

data/lib/simp/rpm_signer.rb

@@ -195,8 +195,13 @@ class Simp::RpmSigner
               # the first RPM to be signed with the key after the gpg-agent is
               # started and the key's passphrase has not been cleared from the
               # agent's cache.
-              read.expect(/pass\s?phrase:.*/) do |text|
-                write.puts(gpgkey[:password])
+              read.expect(/(pass\s?phrase:|verwrite).*/) do |text|
+                if text.last.include?('verwrite')
+                  write.puts('y')
+                else
+                  write.puts(gpgkey[:password])
+                end
+
                 write.flush
               end
             end
@@ -287,7 +292,7 @@ class Simp::RpmSigner
     begin
       results = Parallel.map(
         to_sign,
-        :in_processes => opts[:max_concurrent],
+        :in_processes => 1,
         :progress => opts[:progress_bar_title]
       ) do |rpm|
         _result = nil

data/spec/acceptance/suites/default/00_pkg_rpm_custom_scriptlets_spec.rb

@@ -11,7 +11,7 @@ shared_examples_for 'an RPM generator with customized scriptlets' do
     scriptlets = rpm_scriptlets_for(
       host,
       "#{pkg_root_dir}/testpackage_custom_scriptlet/dist/" +
-      'pupmod-simp-testpackage-0.0.1-1.noarch.rpm'
+      "pupmod-simp-testpackage-0.0.1-1#{rpm_dist}.noarch.rpm"
     )
 
     comment '...the expected scriptlet types are present'
@@ -69,7 +69,7 @@ shared_examples_for 'an RPM generator with customized triggers' do
     triggers = rpm_triggers_for(
       host,
       "#{pkg_root_dir}/testpackage_custom_scriptlet/dist/" +
-      'pupmod-simp-testpackage-0.0.1-1.noarch.rpm'
+      "pupmod-simp-testpackage-0.0.1-1#{rpm_dist}.noarch.rpm"
     )
 
 
@@ -107,6 +107,7 @@ describe 'rake pkg:rpm with customized content' do
   hosts.each do |_host|
     context "on #{_host}" do
       let!(:host){ _host }
+      let(:rpm_dist){ on(host, %{rpm --eval '%{dist}'}).output.strip.gsub(/\.centos$/, '') }
 
       it 'can prep the package directories' do
         testpackages = [

data/spec/acceptance/suites/default/10_pkg_rpm_spec.rb

@@ -10,7 +10,7 @@ shared_examples_for 'an RPM generator with edge cases' do
   it 'should use specified release number for the RPM' do
     on host, %(#{run_cmd} "cd #{pkg_root_dir}/testpackage_with_release; #{rake_cmd} pkg:rpm")
     release_test_rpm = File.join(pkg_root_dir, 'testpackage_with_release',
-      'dist', 'pupmod-simp-testpackage-0.0.1-42.noarch.rpm')
+      'dist', "pupmod-simp-testpackage-0.0.1-42#{rpm_dist}.noarch.rpm")
     on host, %(test -f #{release_test_rpm})
   end
 
@@ -73,10 +73,10 @@ describe 'rake pkg:rpm' do
     copy_host_files_into_build_user_homedir(hosts)
   end
 
-
   hosts.each do |_host|
     context "on #{_host}" do
       let!(:host){ _host }
+      let(:rpm_dist){ on(host, %{rpm --eval '%{dist}'}).output.strip.gsub(/\.centos$/, '') }
 
       context 'rpm building' do
 
@@ -107,7 +107,7 @@ describe 'rake pkg:rpm' do
         context 'using simpdefault.spec' do
 
           let(:build_type) {:default}
-          let(:testpackage_rpm) { File.join(testpackage_dir, 'dist/pupmod-simp-testpackage-0.0.1-1.noarch.rpm') }
+          let(:testpackage_rpm) { File.join(testpackage_dir, "dist/pupmod-simp-testpackage-0.0.1-1#{rpm_dist}.noarch.rpm") }
 
           it 'should create an RPM' do
             comment "produces RPM on #{host}"

data/spec/acceptance/suites/default/55_build_pkg_signing_spec.rb

@@ -320,7 +320,7 @@ describe 'rake pkg:signrpms and pkg:checksig' do
 
     it 'should corrupt the password of new key' do
       key_gen_file = File.join(dev_keydir, 'gengpgkey')
-      on(hosts, "sed -i -e \"s/^Passphrase: /Passphrase: OOPS/\" #{key_gen_file}")
+      on(hosts, "sed -ci -e \"s/^Passphrase: /Passphrase: OOPS/\" #{key_gen_file}")
     end
 
     include_examples('it begins with unsigned RPMs')
@@ -342,7 +342,7 @@ describe 'rake pkg:signrpms and pkg:checksig' do
   end
 
   hosts.each do |host|
-    os_major =  fact_on(host,'operatingsystemmajrelease')
+    os_major =  fact_on(host,'os.release.major')
     if os_major > '7'
       # this problem only happens on EL > 7 in a docker container
       describe "when gpg-agent's socket path is too long on #{host}" do
@@ -378,7 +378,7 @@ describe 'rake pkg:signrpms and pkg:checksig' do
               :acceptable_exit_codes => [1]
             )
 
-            err_msg = %r(Failed to sign #{test_rpm} in 5 seconds)
+            err_msg = %r(Failed to sign)
             expect(result.stderr).to match(err_msg)
 
             signature_check = on(host, "rpm -qip '#{test_rpm}' | grep ^Signature", run_opts)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simp-rake-helpers
 version: !ruby/object:Gem::Version
-  version: 5.12.3
+  version: 5.12.7
 platform: ruby
 authors:
 - Chris Tessmer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-28 00:00:00.000000000 Z
+date: 2021-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simp-beaker-helpers