simp-rake-helpers 5.12.4 → 5.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d79c389d3f40c038b0ba81fe8351e4f71b256bd1eaeeef52eb0af07239ec793b
-  data.tar.gz: 1025900b3bbfa941429f228c7952bf79ccfec9a0b388a5a8fcfaf670b3c6f590
+  metadata.gz: 5686fa0464ed4e47817e43609d4aa2ec52ccab1a9facd6429d73ec39b321f487
+  data.tar.gz: f787b34312ad9033c21811615adb5eb8d069655038e309109fab82560f1e0b81
 SHA512:
-  metadata.gz: 9c2e9f763a211891553cde62a1963f9fd603f1ce9bfed11b828b36206fe5ec73e1b55f7d1f570c5d1df79cdc00c534d57cc8df24cd205e4cabe946895e820d47
-  data.tar.gz: 818c69dcf5bab3bba5516539852abc856973f97275bc186d46ef268c2878d6a4e0542d179a47fd3ed6e48266ff819fc7c70127953694f92b6b7c1b5c03377e98
+  metadata.gz: 64c644cf1b04dbceaface541e8b6d219a7d6d9f93ed863a18062b25dece5307d62715a1fee0351e444c3f00e9c8ce476b7455a08f2b76291184949796e1770cf
+  data.tar.gz: 18663a6da6f497810c0d505d2fd73109cd97355b664deaa3609f60787dd280fb6a6bee6f3e0f2fa473dacffc696bb369994396b271f48bebeae6fb1a31fabdc4

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+### 5.13.0 /2021-11-14
+- Added:
+  - Output the full `mkisofs` command when building an ISO
+
+### 5.12.7 / 2021-10-26
+- Added:
+  - Env var `SIMP_PKG_progress_bar=no` to turn off pkg RPM build progress bars
+  - Env var `SIMP_PKG_fetch_published_rpm=no` to prevent downloading a
+    published package
+- Fixed:
+  - RPM builds no longer fail with leftover generated
+    `build/rpm_metadata/releases` files
+  - Removed unused code, tidied up some confusing bits to make structure more
+    obvious
+
+### 5.12.6 / 2021-10-19
+- Ensure that the `Updates` directory does not link to files in itself
+
+### 5.12.5 / 2021-10-07
+- Fixed a bug where `build:auto` failed when building the SIMP ISO for EL7,
+  because the code attempted to move a directory onto itself.
+- Ensured GPG keys in simp-gpgkeys are available in the DVD overlay tar file
+
 ### 5.12.4 / 2021-10-05
 - Ensure that the DIST qualifier is added to all built RPMs
 - Use the new SimpRepos directory layout when building an ISO using externally

data/lib/simp/rake/build/build.rb

@@ -282,7 +282,7 @@ module Simp::Rake::Build
         Error: Could not find anything to do!
 
         In #{target_dir}:
-            No packages in either packages.yaml or the packages/ directory
+            No packages in either packages.yaml or the packages/ directory and no repos found in the reposync/ directory
                 EOM
               end
 

data/lib/simp/rake/build/iso.rb

@@ -190,6 +190,7 @@ module Simp::Rake::Build
                 end
               end
 
+
               repo_target_dir = dir
 
               # If we've pulled in reposync directories, we expect them to
@@ -256,7 +257,14 @@ module Simp::Rake::Build
 
               # Pop the SIMP directory from the tarball into the correct spot
               # FIXME: This is a hack
-              FileUtils.mv("#{dir}/SIMP", repo_target_dir) if File.directory?("#{dir}/SIMP")
+              unless dir == repo_target_dir
+                simpdir = File.join(dir,'SIMP')
+
+                if File.directory?(simpdir)
+                   cp_r(simpdir, repo_target_dir, :verbose => verbose)
+                   rm_rf(simpdir, :verbose => verbose)
+                end
+              end
 
               Dir.chdir("#{repo_target_dir}/SIMP") do
                 # Add the SIMP Dependencies
@@ -301,8 +309,10 @@ module Simp::Rake::Build
                   cp(rpm,rpm_arch, :verbose => verbose)
                 end
 
-                unless reposync_active
-                  ln_s('noarch', arch, :verbose => verbose) if (!File.directory?(arch) && File.directory?('noarch'))
+                if reposync_active
+                    fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+                else
+                  ln_sf('noarch', arch, :verbose => verbose) if (!File.directory?(arch) && File.directory?('noarch'))
                   fail("Could not find architecture '#{arch}' in the SIMP distribution") unless (File.directory?(arch) || File.symlink?(arch))
 
                   # Get everything set up properly...
@@ -320,6 +330,45 @@ module Simp::Rake::Build
                         ln_sf(source_rpm,link_target, :verbose => verbose)
                       end
                     end
+
+                    fail("Error: Could not run createrepo in #{Dir.pwd}") unless system(%(#{mkrepo} .))
+                  end
+                end
+              end
+
+              ### Munge the Repos
+
+              # Create an Updates directory that is properly populated
+              updates_readme = <<~README
+                This directory houses updates to NON-MODULAR RPMs.
+
+                DO NOT put modular RPMs in this directory or you will break your
+                system updates!
+                README
+
+              updates_dir = File.join(dir, 'Updates')
+              mkdir_p(updates_dir)
+
+              Dir.chdir(updates_dir) do
+                File.open('README','w'){|fh| fh.puts(updates_readme) }
+
+                repos = Dir.glob(File.join('..','**','repodata'))
+                modular_repos = Dir.glob(File.join('..','**','repodata','*-modules.*'))
+                non_modular_repos = repos.select{|x| modular_repos.grep(%r{^#{Regexp.escape(x)}}).empty? }
+                non_modular_repos.map!{|x| File.split(x).first}
+                non_modular_repos.delete_if{|x| x.match(%r{/SimpRepos|/SIMP}) }
+                non_modular_repos.each do |non_modular_repo|
+                  Dir.glob(File.join(non_modular_repo, '**', '*.rpm')).each do |rpm|
+                    # when non_modular_repo is '..', can still find RPMs we need
+                    # to exclude
+                    next if rpm.match(%r{/SimpRepos|/SIMP})
+
+                    rpm_dest = File.basename(rpm)
+                    if File.exist?(rpm) && File.exist?(rpm_dest)
+                      next if (File.realpath(rpm) == File.realpath(rpm_dest))
+                    end
+
+                    ln_sf(rpm, rpm_dest, :verbose => verbose)
                   end
                 end
 
@@ -372,9 +421,11 @@ module Simp::Rake::Build
                 '-x ./lost+found',
                 "-o #{@simp_output_iso}",
                 dir
-              ]
+              ].join(' ')
+
+              $stdout.puts "Running: #{mkisofs_cmd}"
 
-              system(mkisofs_cmd.join(' '))
+              system(mkisofs_cmd)
             end
           end # End of tarfiles loop
 

data/lib/simp/rake/build/pkg.rb

@@ -21,13 +21,17 @@ module Simp::Rake::Build
       @rpm_build_metadata = 'last_rpm_build_metadata.yaml'
       @rpm_dependency_file = File.join(@base_dir, 'build', 'rpm', 'dependencies.yaml')
       @build_keys_dir = ENV.fetch('SIMP_PKG_build_keys_dir', File.join(@base_dir, '.dev_gpgkeys'))
-      @long_gpg_socket_err_msg = <<~EOM
+      @long_gpg_socket_err_msg = <<~MSG
         If the problem is 'socket name <xxx> is too long', use SIMP_PKG_build_keys_dir
         to override
         #{@build_keys_dir}
         with a shorter path. The socket name must be < 108 characters.
 
-      EOM
+      MSG
+      # nil = default not set; use local preference
+      @fetch_published_rpm_default = ENV['SIMP_PKG_fetch_published_rpm'] ? (
+        ENV['SIMP_PKG_fetch_published_rpm'] =~ /\A(yes|true)\Z/i ? true : false
+      ) : nil
 
       define_tasks
     end
@@ -114,7 +118,7 @@ module Simp::Rake::Build
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Prepare a GPG signing key to sign build packages
 
             * :key - the name of the build keys subdirectory to prepare
@@ -142,7 +146,7 @@ module Simp::Rake::Build
           ENV vars:
             - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
             - Set `SIMP_PKG_build_keys_dir` to override the default build keys path.
-        EOM
+        DESC
         task :key_prep,[:key] => [:prep] do |t,args|
           args.with_defaults(:key => 'dev')
           key = args.key
@@ -251,7 +255,7 @@ module Simp::Rake::Build
 
 
 =begin
-        desc <<-EOM
+        desc <<~DESC
           Build the entire SIMP release.
 
             * :docs - Build the docs. Set this to false if you wish to skip building the docs.
@@ -261,7 +265,7 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
                 build-specific YUM cache
-        EOM
+        DESC
 =end
         task :build,[:docs,:key] => [:prep,:key_prep] do |t,args|
           args.with_defaults(:key => 'dev')
@@ -269,12 +273,7 @@ module Simp::Rake::Build
 
           check_dvd_env
 
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
-              ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
-          rescue Simp::YUM::Error
-          end
+          get_yum_helper
 
           Rake::Task['pkg:aux'].invoke
           if "#{args.docs}" == 'true'
@@ -287,7 +286,7 @@ module Simp::Rake::Build
           Rake::Task['pkg:signrpms'].invoke(args[:key])
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the Puppet module RPMs.
 
             * :method - The Puppetfile from which the repository information
@@ -295,12 +294,12 @@ module Simp::Rake::Build
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :modules,[:method] => [:prep] do |t,args|
           build(@build_dirs[:modules],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build a single Puppet Module RPM.
 
             * :name   - The path, or name, of the module to build. If a name is
@@ -314,11 +313,12 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
                 build-specific YUM cache
-        EOM
+        DESC
         task :single,[:name,:method] => [:prep] do |t,args|
           fail("You must pass :name to '#{t.name}'") unless args[:name]
 
           mod_path = File.absolute_path(args[:name])
+          get_yum_helper
 
           if args[:name].include?('/')
             fail("'#{args[:name]}' does not exist!") unless File.directory?(mod_path)
@@ -333,35 +333,27 @@ module Simp::Rake::Build
             mod_path = local_module[:path]
           end
 
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data')),
-              ENV.fetch('SIMP_YUM_makecache','yes') == 'yes')
-          rescue Simp::YUM::Error
-          end
-
-          ENV['SIMP_PKG_rand_name'] = 'yes'
           build(Array(mod_path), t)
 
           puts("Your packages can be found in '#{mod_path}/dist'")
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the SIMP non-module RPMs.
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :aux => [:prep]  do |t,args|
           build(@build_dirs[:aux],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Build the SIMP documentation.
 
             ENV vars:
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
-        EOM
+        DESC
         task :doc => [:prep] do |t,args|
           # Need to make sure that the docs have the version updated
           # appropriately prior to building
@@ -373,7 +365,7 @@ module Simp::Rake::Build
           build(@build_dirs[:doc],t)
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Sign a set of RPMs.
 
             Signs any unsigned RPMs in the specified directory
@@ -394,7 +386,7 @@ module Simp::Rake::Build
             * Set `SIMP_PKG_rpmsign_timeout` to override the maximum time in seconds
               to wait for an individual RPM signing operation to complete.
               - Defaults to 60 seconds.
-        EOM
+        DESC
         task :signrpms,[:key,:rpm_dir,:force,:digest_algo] => [:prep,:key_prep] do |t,args|
           require 'simp/rpm_signer'
 
@@ -455,13 +447,13 @@ module Simp::Rake::Build
         end
 
 =begin
-        desc <<-EOM
+        desc <<~DESC
           Check that RPMs are signed.
 
             Checks all RPM files in a directory to see if they are trusted.
               * :rpm_dir - A directory containing RPM files to check. Default #{@build_dir}/SIMP
               * :key_dir - The path to the GPG keys you want to check the packages against. Default #{@src_dir}/assets/gpgkeys/
-        EOM
+        DESC
 =end
         task :checksig,[:rpm_dir,:key_dir] => [:prep] do |t,args|
           begin
@@ -547,7 +539,7 @@ module Simp::Rake::Build
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Run repoclosure on RPM files.
 
             Finds all rpm files in the target dir and all of its subdirectories, then
@@ -561,7 +553,7 @@ module Simp::Rake::Build
               - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
               - Set `SIMP_PKG_repoclose_pe=yes` to enable repoclosure on PE-related RPMs.
 
-        EOM
+        DESC
         task :repoclosure,[:target_dir,:aux_dir] => [:prep] do |t,args|
           default_target = @pkg_dirs[:simp]
           args.with_defaults(:target_dir => File.expand_path(default_target))
@@ -572,31 +564,31 @@ module Simp::Rake::Build
 
           _repoclose_pe = ENV.fetch('SIMP_PKG_repoclose_pe','no') == 'yes'
 
-          yum_conf_template = <<-EOF
-[main]
-keepcache=0
-exactarch=1
-obsoletes=1
-gpgcheck=0
-plugins=1
-installonly_limit=5
-<% unless #{_repoclose_pe} -%>
-exclude=*-pe-*
-<% end -%>
-
-<% repo_files.each do |repo| -%>
-include=file://<%= repo %>
-<% end -%>
-  EOF
-
-          yum_repo_template = <<-EOF
-[<%= repo_name %>]
-name=<%= repo_name %>
-baseurl=file://<%= repo_path %>
-enabled=1
-gpgcheck=0
-protect=1
-  EOF
+          yum_conf_template = <<~YUM_CONF
+            [main]
+            keepcache=0
+            exactarch=1
+            obsoletes=1
+            gpgcheck=0
+            plugins=1
+            installonly_limit=5
+            <% unless #{_repoclose_pe} -%>
+            exclude=*-pe-*
+            <% end -%>
+
+            <% repo_files.each do |repo| -%>
+            include=file://<%= repo %>
+            <% end -%>
+          YUM_CONF
+
+          yum_repo_template = <<~YUM_REPO
+            [<%= repo_name %>]
+            name=<%= repo_name %>
+            baseurl=file://<%= repo_path %>
+            enabled=1
+            gpgcheck=0
+            protect=1
+          YUM_REPO
 
           fail("#{args[:target_dir]} does not exist!") unless File.directory?(args[:target_dir])
 
@@ -610,7 +602,7 @@ protect=1
                 .delete_if{|x| x =~ /\.src\.rpm$/}
                 .each do |path|
                   sym_path = "repos/base/#{File.basename(path)}"
-                  ln_s(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
+                  ln_sf(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
               end
 
               if args[:aux_dir]
@@ -618,7 +610,7 @@ protect=1
                   .delete_if{|x| x =~ /\.src\.rpm$/}
                   .each do |path|
                     sym_path = "repos/lookaside/#{File.basename(path)}"
-                    ln_s(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
+                    ln_sf(path,sym_path, :verbose => @verbose) unless File.exists?(sym_path)
                 end
               end
 
@@ -643,7 +635,8 @@ protect=1
                 file.write(ERB.new(yum_conf_template,nil,'-').result(binding))
               end
 
-              if which('dnf')
+              dnf_system = which('dnf')
+              if dnf_system
                 cmd = 'repoclosure -c base.conf --disablerepo=* --enablerepo=base'
               else
                 cmd = 'repoclosure -c repodata -n -t -r base -l lookaside -c yum.conf'
@@ -661,15 +654,24 @@ protect=1
               if (!$?.success? || (repoclosure_output =~ /nresolved/))
                 errmsg = ['Error: REPOCLOSURE FAILED:']
                 errmsg << [repoclosure_output]
-                fail(errmsg.join("\n"))
+                puts(errmsg.join("\n"))
+
+                if dnf_system
+                  if ENV.fetch('SIMP_BUILD_prompt', 'yes') != 'no'
+                    puts('- Press any key to continue or ^C to abort -')
+                    $stdin.gets
+                  end
+                else
+                  fail('Repoclosure Failed')
+                end
               end
             end
           end
         end
 
-        desc <<-EOM
+        desc <<~DESC
           Print published status of all project RPMs
-        EOM
+        DESC
         task :check_published => [:prep] do |t,args|
           begin
             yum_helper = Simp::YUM.new(
@@ -687,7 +689,12 @@ protect=1
 
             if Dir.exist?(dir)
               begin
-                require_rebuild?(dir, yum_helper, { :verbose => true, :check_git => true, :prefix => '' })
+                require_rebuild?(dir, yum_helper, {
+                  fetch:      false,
+                  verbose:    true,
+                  check_git:  true,
+                  prefix:     ''
+                })
               rescue => e
                 _errmsg = "Error: require_rebuild?(): Status check failed on '#{dir}' => #{e}"
               end
@@ -714,9 +721,24 @@ protect=1
           return (0...24).map{ (65 + rand(26)).chr }.join.downcase
         end
 
+        # Return a SIMP::YUM object configured for the local build dir or nil
+        #
+        # ENV vars:
+        #   - Set `SIMP_PKG_verbose=yes` to report file operations as they happen.
+        #   - Set `SIMP_YUM_makecache=no` if you do NOT want to rebuild the
+        #     build-specific YUM cache
+        def get_yum_helper( makecache = (ENV.fetch('SIMP_YUM_makecache','yes') == 'yes') )
+          begin
+            build_yum_data = File.join(@distro_build_dir, 'yum_data')
+            return Simp::YUM.new(Simp::YUM.generate_yum_conf(build_yum_data), makecache)
+          rescue Simp::YUM::Error => e
+            $stderr.puts "Simp::YUM::Error: #{e}" if @verbose
+          end
+        end
+
         # Check and see if 'dir' requires a rebuild based on published packages
         #
-        # If 'fetch' is true => Download the RPM
+        # If 'fetch' is true => Download the published RPM
         # If 'verbose' is true => Print helpful information to stderr
         # If 'check_git' is true => Print the git tag status if 'verbose' is true
         # 'prefix' is used to prepend verbose messages
@@ -725,7 +747,13 @@ protect=1
         # - Method is too long
         # - Method needs to be passed in class variables (@xxx) so that it
         #   can be pulled out into a library that is easily unit-testable
-        def require_rebuild?(dir, yum_helper, opts={ :unique_namespace => generate_namespace, :fetch => false, :verbose => @verbose, :check_git => false, :prefix => '' })
+        def require_rebuild?(dir, yum_helper, opts={
+          unique_namespace:  generate_namespace,
+          fetch:             false,
+          verbose:           @verbose,
+          check_git:         false,
+          prefix:            ''
+        })
           result = false
           rpm_metadata = File.exist?(@rpm_dependency_file) ? YAML.load(File.read(@rpm_dependency_file)) : {}
           dir_relpath = Pathname.new(dir).relative_path_from(Pathname.new(Dir.pwd)).to_path
@@ -860,7 +888,8 @@ protect=1
                     else
                       $stderr.puts "#{opts[:prefix]}Found Existing Remote RPM: #{published_rpm}" if opts[:verbose]
                       if opts[:fetch]
-                        # Download remote RPM, unless already downloaded it.
+                        # Download published RPM, unless it's already been downloaded.
+                        # TODO Unhandled edge case: Validate that downloaded RPM is valid
                         if File.exist?(File.join('dist', published_rpm))
                           $stderr.puts "#{opts[:prefix]}#{published_rpm} previously downloaded" if opts[:verbose]
                         else
@@ -904,101 +933,107 @@ protect=1
           return result
         end
 
+        def build_pupmod_rpm(dir, yum_helper, dbg_prefix = '  ')
+          unique_namespace = generate_namespace
+          if require_rebuild?(dir, yum_helper, {
+            unique_namespace:  unique_namespace,
+            fetch:             @fetch_published_rpm.nil? ? true : @fetch_published_rpm,
+            verbose:           @verbose,
+            prefix:            dbg_prefix,
+          })
+            $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' on #{File.basename(dir)}") if @verbose
+            Rake::Task["#{unique_namespace}:pkg:rpm"].invoke
+          else
+            # Record metadata for the downloaded RPM
+            Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
+          end
+          true
+        end
+
+
+        def build_rakefile_rpm(dir, yum_helper, dbg_prefix = '  ')
+          if require_rebuild?(dir, yum_helper, {
+            fetch:    @fetch_published_rpm.nil? ? true : @fetch_published_rpm,
+            verbose:  @verbose,
+            prefix:   dbg_prefix
+          })
+            $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' in #{File.basename(dir)}") if @verbose
+            rake_flags = Rake.application.options.trace ? '--trace' : ''
+            cmd = %{SIMP_BUILD_version=#{@simp_version} rake pkg:rpm #{rake_flags} 2>&1}
+
+            build_success = true
+            begin
+              %x{#{cmd}}
+              build_success = $?.success?
+
+              built_rpm = true
+            rescue
+              build_success = false
+            end
+
+            unless build_success
+              if @verbose
+                $stderr.puts("First 'rake pkg:rpm' attempt for #{dir} failed, running bundle and trying again.")
+              end
+
+              if Bundler.respond_to?(:with_unbundled_env)
+                # Bundler 2.1+
+                clean_env_method = :with_unbundled_env
+                bundle_install_cmd = %{bundle config set with 'development' && bundle install}
+              else
+                # Old Bundler
+                clean_env_method = :with_clean_env
+                bundle_install_cmd = %{bundle install --with development}
+              end
+
+              ::Bundler.send(clean_env_method) do
+                %x{#{bundle_install_cmd}}
+
+                output = %x{#{cmd} 2>&1}
+
+                unless $?.success?
+                  raise("Error in #{dir} running #{cmd}\n#{output}")
+                end
+              end
+            end
+          else
+            # Record metadata for the downloaded RPM
+            Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
+            built_rpm = true
+          end
+        end
+
+
         # Takes a list of directories to hop into and perform builds within
         #
         # The task must be passed so that we can output the calling name in the
         # status bar.
         def build(dirs, task, rebuild_for_arch=false, remake_yum_cache = false)
-          _verbose = ENV.fetch('SIMP_PKG_verbose','no') == 'yes'
-          dbg_prefix = '  ' # prefix for debug messages
-
           fail("Could not find RPM dependency file '#{@rpm_dependency_file}'") unless File.exist?(@rpm_dependency_file)
-
-          begin
-            yum_helper = Simp::YUM.new(
-              Simp::YUM.generate_yum_conf(File.join(@distro_build_dir, 'yum_data'))
-            )
-          rescue Simp::YUM::Error
-          end
+          yum_helper = get_yum_helper( remake_yum_cache )
 
           Parallel.map(
             # Allow for shell globs
-            Array(dirs),
-            :in_processes => @cpu_limit,
-            :progress => task.name
+            Array(dirs), {
+              in_processes:  @cpu_limit,
+              progress:      (ENV.fetch('SIMP_PKG_progress_bar','yes') == 'yes') ? task.name : nil,
+            }
           ) do |dir|
             fail("Could not find directory #{dir}") unless Dir.exist?(dir)
 
             Dir.chdir(dir) do
               built_rpm = false
+              $stderr.puts("\nPackaging #{File.basename(dir)}") if @verbose
 
-              if @verbose
-                $stderr.puts("\nPackaging #{File.basename(dir)}")
-              end
-
-              # We're building a module, override anything down there
               if File.exist?('metadata.json')
-                unique_namespace = generate_namespace
-                if require_rebuild?(dir, yum_helper, { :unique_namespace => unique_namespace, :fetch => true, :verbose => @verbose, :prefix => dbg_prefix})
-                  $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' on #{File.basename(dir)}") if @verbose
-                  Rake::Task["#{unique_namespace}:pkg:rpm"].invoke
-                else
-                  # Record metadata for the downloaded RPM
-                  Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
-                end
-
-                built_rpm = true
-
-              # We're building one of the extra assets and should honor its Rakefile
-              # and RPM spec file.
+                # We're building a module, override anything down there
+                built_rpm = build_pupmod_rpm(dir, yum_helper)
               elsif File.exist?('Rakefile')
-                if require_rebuild?(dir, yum_helper, { :fetch => true, :verbose => @verbose, :prefix => dbg_prefix })
-                  $stderr.puts("#{dbg_prefix}Running 'rake pkg:rpm' in #{File.basename(dir)}") if @verbose
-                  rake_flags = Rake.application.options.trace ? '--trace' : ''
-                  cmd = %{SIMP_BUILD_version=#{@simp_version} rake pkg:rpm #{rake_flags} 2>&1}
-
-                  build_success = true
-                  begin
-                    %x{#{cmd}}
-                    build_success = $?.success?
-
-                    built_rpm = true
-                  rescue
-                    build_success = false
-                  end
-
-                  unless build_success
-                    if @verbose
-                      $stderr.puts("First 'rake pkg:rpm' attempt for #{dir} failed, running bundle and trying again.")
-                    end
-
-                    if Bundler.respond_to?(:with_unbundled_env)
-                      # Bundler 2.1+
-                      clean_env_method = :with_unbundled_env
-                      bundle_install_cmd = %{bundle config set with 'development' && bundle install}
-                    else
-                      # Old Bundler
-                      clean_env_method = :with_clean_env
-                      bundle_install_cmd = %{bundle install --with development}
-                    end
-
-                    ::Bundler.send(clean_env_method) do
-                      %x{#{bundle_install_cmd}}
-
-                      output = %x{#{cmd} 2>&1}
-
-                      unless $?.success?
-                        raise("Error in #{dir} running #{cmd}\n#{output}")
-                      end
-                    end
-                  end
-                else
-                  # Record metadata for the downloaded RPM
-                  Simp::RPM::create_rpm_build_metadata(File.expand_path(dir))
-                  built_rpm = true
-                end
+                # We're building one of the extra assets and should honor its Rakefile
+                # and RPM spec file.
+                built_rpm = build_rakefile_rpm(dir, yum_helper)
               else
-                puts "Warning: '#{dir}' could not be built via Rake"
+                puts "Warning: '#{dir}' could not be built (not a pupmod & no Rakefile!)"
               end
 
               if built_rpm
@@ -1014,12 +1049,7 @@ protect=1
 
                 raise("No RPMs generated for #{dir}") if rpms.empty?
               end
-
-              if @verbose
-                rpms = Dir.glob('dist/*.rpm')
-#                $stderr.puts("#{dbg_prefix}RPMS: #{rpms.join("\n#{dbg_prefix}      ")}")
-                $stderr.puts("Finished  #{File.basename(dir)}")
-              end
+              $stderr.puts("Finished  #{File.basename(dir)}") if @verbose
             end
           end
         end

data/lib/simp/rake/build/rpmdeps.rb

@@ -267,7 +267,13 @@ module Simp::Rake::Build::RpmDeps
   def self.release_file_up_to_date?(new_release_info, rpm_release_file)
     return false unless File.exist?(rpm_release_file)
 
-    return File.read(rpm_release_file).strip == new_release_info.strip
+    # remove comments like "# release set by simp-core dependencies.yaml"
+    release_file_content = File.readlines(rpm_release_file).reject{|x| x =~ /^ *#/}.join("\n").strip
+
+    # sanitize numerics, etc.
+    new_release_content = "#{new_release_info}".strip
+
+    return release_file_content == new_release_content
   end
 
   # Generate 'build/rpm_metadata/release' file containing release qualifier

data/lib/simp/rake/build/tar.rb

@@ -51,7 +51,8 @@ module Simp::Rake::Build
               'rubygem-simp-cli',
               'simp',
               'simp-gpgkeys',
-              'simp-utils'
+              'simp-utils',
+              'simp-vendored-r10k'
             ]
           }
 
@@ -83,8 +84,11 @@ module Simp::Rake::Build
             end
 
             unless failures.empty?
-              msg = ['Error: Could not find the following packages:']
-              fail((msg + failures).join("\n"))
+              msg = ['Error: Could not find the following packages:'] +
+                failures +
+                ['Did "dist/logs/last_rpm_build_metadata.yaml" get generated in the build directory?']
+
+              fail(msg.join("\n"))
             end
           end
         end
@@ -126,10 +130,16 @@ module Simp::Rake::Build
 
             Simp::RPM.copy_wo_vcs(@dvd_src,".",base_dir)
 
-            # Copy in the GPG Public Keys
+            # Copy in the GPG Public Keys from @build_dir
             mkdir_p("#{destdir}/GPGKEYS")
             ln(Dir.glob("#{@build_dir}/GPGKEYS/RPM-GPG-KEY*"), "#{destdir}/GPGKEYS", { :force => true })
 
+            # Copy in the GPG Public Keys packaged by simp-gpgkeys
+            simp_gpgkeys = File.join(@src_dir, 'assets', 'gpgkeys', 'GPGKEYS')
+            if Dir.exist?(simp_gpgkeys)
+              ln(Dir.glob("#{simp_gpgkeys}/RPM-GPG-KEY*"), "#{destdir}/GPGKEYS", { :force => true })
+            end
+
             # Copy in the auto-build RPMs
             Dir.chdir("#{@build_dir}/SIMP/RPMS") do
               Dir.glob('*').each do |type|

data/lib/simp/rake/helpers/version.rb

@@ -2,5 +2,5 @@ module Simp; end
 module Simp::Rake; end
 
 class Simp::Rake::Helpers
-  VERSION = '5.12.4'
+  VERSION = '5.13.0'
 end

data/lib/simp/rpm_signer.rb

@@ -195,8 +195,13 @@ class Simp::RpmSigner
               # the first RPM to be signed with the key after the gpg-agent is
               # started and the key's passphrase has not been cleared from the
               # agent's cache.
-              read.expect(/pass\s?phrase:.*/) do |text|
-                write.puts(gpgkey[:password])
+              read.expect(/(pass\s?phrase:|verwrite).*/) do |text|
+                if text.last.include?('verwrite')
+                  write.puts('y')
+                else
+                  write.puts(gpgkey[:password])
+                end
+
                 write.flush
               end
             end
@@ -287,7 +292,7 @@ class Simp::RpmSigner
     begin
       results = Parallel.map(
         to_sign,
-        :in_processes => opts[:max_concurrent],
+        :in_processes => 1,
         :progress => opts[:progress_bar_title]
       ) do |rpm|
         _result = nil

data/spec/acceptance/suites/default/55_build_pkg_signing_spec.rb

@@ -342,7 +342,7 @@ describe 'rake pkg:signrpms and pkg:checksig' do
   end
 
   hosts.each do |host|
-    os_major =  fact_on(host,'operatingsystemmajrelease')
+    os_major =  fact_on(host,'os.release.major')
     if os_major > '7'
       # this problem only happens on EL > 7 in a docker container
       describe "when gpg-agent's socket path is too long on #{host}" do
@@ -378,7 +378,7 @@ describe 'rake pkg:signrpms and pkg:checksig' do
               :acceptable_exit_codes => [1]
             )
 
-            err_msg = %r(Failed to sign #{test_rpm} in 5 seconds)
+            err_msg = %r(Failed to sign)
             expect(result.stderr).to match(err_msg)
 
             signature_check = on(host, "rpm -qip '#{test_rpm}' | grep ^Signature", run_opts)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simp-rake-helpers
 version: !ruby/object:Gem::Version
-  version: 5.12.4
+  version: 5.13.0
 platform: ruby
 authors:
 - Chris Tessmer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-06 00:00:00.000000000 Z
+date: 2022-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simp-beaker-helpers