simp-rake-helpers 3.0.2 → 3.1.0

data/lib/simp/rake/build/build.rb

@@ -5,7 +5,7 @@ require 'simp/rake/build/constants'
 module Simp; end
 module Simp::Rake; end
 module Simp::Rake::Build
-  class SIMPBuildException < Exception
+  class SIMPBuildException < StandardError
   end
 
   include Simp::Rake
@@ -15,12 +15,19 @@ module Simp::Rake::Build
 
     def initialize( base_dir )
       init_member_vars( base_dir )
+
       define_tasks
     end
 
     # define rake tasks
     def define_tasks
       namespace :build do
+        task :prep do
+          if $simp6
+            @build_dir = $simp6_build_dir
+          end
+        end
+
         desc <<-EOM
         Run bundle at every level of the project.
 
@@ -33,7 +40,7 @@ module Simp::Rake::Build
           * :verbose => Enable verbose reporting. Default => 'false'
         EOM
 
-        task :bundle, [:action, :verbose, :method] do |t, args|
+        task :bundle, [:action, :verbose, :method] => [:prep] do |t, args|
           args.with_defaults(:action => 'install')
           args.with_defaults(:verbose => 'false')
           args.with_defaults(:method => 'tracking')
@@ -84,8 +91,22 @@ module Simp::Rake::Build
         end
 
         namespace :yum do
-          @base_dir = File.join(@build_dir,'yum_data')
-          @build_arch = 'x86_64'
+          task :prep do
+            if $simp6
+              @build_dir = $simp6_build_dir
+
+              unless @build_dir
+                if ENV['SIMP_BUILD_yum_dir'] && File.exist?(File.join(ENV['SIMP_BUILD_yum_dir'], 'yum_data'))
+                  @build_dir = ENV['SIMP_BUILD_yum_dir']
+                end
+              end
+
+              raise('Error: For SIMP 6+ builds, you need to set SIMP_BUILD_yum_dir to the directory holding the "yum_data" directory that you wish to sync') unless @build_dir
+            end
+
+            @build_base_dir = File.join(@build_dir,'yum_data')
+            @build_arch = 'x86_64'
+          end
 
           ##############################################################################
           # Helpers
@@ -114,20 +135,24 @@ module Simp::Rake::Build
           # Return the target directory
           # Expects one argument wich is the 'arguments' hash to one of the tasks.
           def get_target_dir(args)
-            fail("Error: You must specify 'os'") unless args[:os]
-            fail("Error: You must specify 'os_version'") unless args[:os_version]
-            fail("Error: You must specify both major and minor version for the OS") unless args[:os_version] =~ /^.+\..+$/
-            fail("Error: You must specify 'simp_version'") unless args[:simp_version]
-            fail("Error: You must specify 'arch'") unless args[:arch]
-
-            # Yes, this is a kluge but the amount of variable passing that would need
-            # to be done to support this is silly.
-            @build_arch = args[:arch]
-
-            return File.join(
-              @base_dir,
-              "SIMP#{args[:simp_version]}_#{args[:os]}#{args[:os_version]}_#{args[:arch]}"
-            )
+            if $simp6
+              return @build_base_dir
+            else
+              fail("Error: You must specify 'os'") unless args[:os]
+              fail("Error: You must specify 'os_version'") unless args[:os_version]
+              fail("Error: You must specify both major and minor version for the OS") unless args[:os_version] =~ /^.+\..+$/
+              fail("Error: You must specify 'simp_version'") unless args[:simp_version]
+              fail("Error: You must specify 'arch'") unless args[:arch]
+
+              # Yes, this is a kluge but the amount of variable passing that would need
+              # to be done to support this is silly.
+              @build_arch = args[:arch]
+
+              return File.join(
+                @build_base_dir,
+                "SIMP#{args[:simp_version]}_#{args[:os]}#{args[:os_version]}_#{args[:arch]}"
+              )
+            end
           end
 
           # Return where YUM finds the passed RPM
@@ -192,7 +217,7 @@ module Simp::Rake::Build
                 puts("Downloading: #{full_pkg}")
                 unless @use_yumdownloader
                   %x(curl -L --max-redirs 10 -s -o "#{full_pkg}" -k "#{source}")
-                  unless %x(file #{full_pkg}).include?('RPM')
+                  if File.exist?(full_pkg) && !%x(file #{full_pkg}).include?('RPM')
                     @use_yumdownloader = true
                     FileUtils.rm(full_pkg)
                   end
@@ -200,6 +225,11 @@ module Simp::Rake::Build
 
                 if @use_yumdownloader
                   %x(yumdownloader -c #{yum_conf} #{File.basename(full_pkg,'.rpm')} 2>/dev/null )
+
+                  if File.exist?(full_pkg) && !%x(file #{full_pkg}).include?('RPM')
+                    @use_yumdownloader = true
+                    FileUtils.rm(full_pkg)
+                  end
                 end
 
                 unless $?.success?
@@ -231,7 +261,7 @@ module Simp::Rake::Build
 
               if clean
                 errmsg += ', removing'
-                FileUtils.rm(rpm)
+                FileUtils.rm(rpm) if File.exist?(rpm)
               end
 
               raise(SIMPBuildException,errmsg)
@@ -334,7 +364,7 @@ module Simp::Rake::Build
           # Update the packages.yaml and packages/ directories
           #   * target_dir => The actual distribution directory where packages.yaml and
           #                   packages/ reside.
-          def update_packages(target_dir,bootstrap=false)
+          def update_packages(target_dir, bootstrap=false)
             # This really should never happen....
             unless File.directory?(target_dir)
               fail <<-EOM
@@ -483,6 +513,8 @@ module Simp::Rake::Build
                 failed_updates.keys.sort.each do |k|
                   $stderr.puts("  * #{k} => #{failed_updates[k]}")
                 end
+
+                raise('Could not update all packages')
               end
             end
           end
@@ -559,7 +591,7 @@ module Simp::Rake::Build
 
           * :arch         - The architecture that you support. Default: x86_64
           EOM
-          task :scaffold,[:os,:os_version,:simp_version,:arch] do |t,args|
+          task :scaffold,[:os,:os_version,:simp_version,:arch] => [:prep] do |t,args|
             # @simp_version is set in the main Rakefile
             args.with_defaults(:simp_version => @simp_version.split('-').first)
             args.with_defaults(:arch => @build_arch)
@@ -572,8 +604,12 @@ module Simp::Rake::Build
             end
 
             # Put together the rest of the scaffold directories
-            Dir.chdir(@base_dir) do
-              mkdir('my_repos') unless File.exist?('my_repos')
+            Dir.chdir(@build_base_dir) do
+              if $simp6
+                mkdir('../my_repos') unless File.exist?('../my_repos')
+              else
+                mkdir('my_repos') unless File.exist?('my_repos')
+              end
             end
 
             Dir.chdir(target_dir) do
@@ -597,7 +633,7 @@ module Simp::Rake::Build
 
           * :arch         - The architecture that you support. Default: x86_64
           EOM
-          task :sync,[:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
+          task :sync,[:os,:os_version,:simp_version,:arch] => [:prep, :scaffold] do |t,args|
             # @simp_version is set in the main Rakefile
             args.with_defaults(:simp_version => @simp_version.split('-').first)
             args.with_defaults(:arch => @build_arch)
@@ -622,7 +658,7 @@ module Simp::Rake::Build
 
           * :arch         - The architecture that you support. Default: x86_64
           EOM
-          task :diff,[:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
+          task :diff,[:os,:os_version,:simp_version,:arch] => [:prep, :scaffold] do |t,args|
             args.with_defaults(:simp_version => @simp_version.split('-').first)
             args.with_defaults(:arch => @build_arch)
 
@@ -688,7 +724,7 @@ module Simp::Rake::Build
 
           * :arch         - The architecture that you support. Default: x86_64
           EOM
-          task :fetch,[:pkg,:os,:os_version,:simp_version,:arch] => [:scaffold] do |t,args|
+          task :fetch,[:pkg,:os,:os_version,:simp_version,:arch] => [:prep, :clean_cache, :scaffold] do |t,args|
             args.with_defaults(:simp_version => @simp_version.split('-').first)
             args.with_defaults(:arch => @build_arch)
 

data/lib/simp/rake/build/clean.rb

@@ -21,6 +21,10 @@ module Simp::Rake::Build
         "#{@base_dir}/SIMP_ISO*"
       )
 
+      if $simp6_build_dirs
+        ::CLEAN.include($simp6_clean_dirs)
+      end
+
       ::CLOBBER.include(
         @dist_dir,
         "#{@build_dir}/build_keys/dev",
@@ -35,7 +39,7 @@ module Simp::Rake::Build
 
         mock_dirs = Dir.glob("/var/lib/mock/*").map{|x| x = File.basename(x) }
 
-        if not mock_dirs.empty? and not args.chroot then
+        unless ( mock_dirs.empty? or args.chroot )
           $stderr.puts "Notice: You must pass a Mock chroot to erase a specified build root."
         end
 

data/lib/simp/rake/build/code.rb

@@ -16,14 +16,13 @@ module Simp::Rake::Build
 
     def define_tasks
       namespace :code do
-
         desc "Show some basic stats. Uses git to figure out what has changed.
        * :since - Do not include any stats before this date.
        * :until - Do not include any stats after this date."
         task :stats,[:since,:until] do |t,args|
           cur_branch = %x{git rev-parse --abbrev-ref HEAD}.chomp
 
-          if cur_branch.empty? then
+          if cur_branch.empty?
             fail "Error: Could not find branch ID!"
           end
 
@@ -33,15 +32,15 @@ module Simp::Rake::Build
 
           cmd = "git log --shortstat --reverse --pretty=oneline"
 
-          if args.since then
+          if args.since
             cmd = cmd + " --since=#{args.since}"
           end
-          if args.until then
+          if args.until
             cmd = cmd + " --until=#{args.until}"
           end
 
           %x{#{cmd}}.each_line do |line|
-            if encode_line(line) =~ /(\d+) files changed, (\d+) insertions\(\+\), (\d+) del.*/ then
+            if encode_line(line) =~ /(\d+) files changed, (\d+) insertions\(\+\), (\d+) del.*/
               changed = changed + $1.to_i
               new = new + $2.to_i
               removed = removed + $3.to_i
@@ -50,15 +49,15 @@ module Simp::Rake::Build
 
           cmd = "git submodule foreach git log --shortstat --reverse --pretty=oneline"
 
-          if args.since then
+          if args.since
             cmd = cmd + " --since=#{args.since}"
           end
-          if args.until then
+          if args.until
             cmd = cmd + " --until=#{args.until}"
           end
 
           %x{#{cmd}}.each_line do |line|
-            if encode_line(line) =~ /(\d+) files changed, (\d+) insertions\(\+\), (\d+) del.*/ then
+            if encode_line(line) =~ /(\d+) files changed, (\d+) insertions\(\+\), (\d+) del.*/
               changed = changed + $1.to_i
               new = new + $2.to_i
               removed = removed + $3.to_i
@@ -88,7 +87,7 @@ module Simp::Rake::Build
           loc["other"] = 0
 
           File.open("#{SRC_DIR}/../Rakefile","r").each do |line|
-            if encode_line(line) !~ /^\s*$/ then
+            if encode_line(line) !~ /^\s*$/
               loc["rake"] = loc["rake"] + 1
             end
           end.close
@@ -96,26 +95,27 @@ module Simp::Rake::Build
           other_ext = Array.new
 
           Find.find(SRC_DIR) do |path|
-              if (
-                ( File.basename(path)[0] == ?. ) or
-                ( path =~ /src\/rsync/ ) or
-                ( path[-3..-1] =~ /\.gz|pem|pub/ ) or
-                ( path =~ /developers_guide\/rdoc/ )
-              ) then
-                Find.prune
-              else
-                next if FileTest.symlink?(path) or FileTest.directory?(path)
-              end
+            if (
+              ( File.basename(path)[0] == ?. ) or
+              ( path =~ /src\/rsync/ ) or
+              ( path[-3..-1] =~ /\.gz|pem|pub/ ) or
+              ( path =~ /developers_guide\/rdoc/ )
+            )
+              Find.prune
+            else
+              next if FileTest.symlink?(path) or FileTest.directory?(path)
+            end
 
             ext = File.extname(path)[1..-1]
-            if not ext then ext = 'none' end
-            if not loc[ext] then
-              other_ext.push(ext) if not other_ext.include?(ext)
+            ext ||= 'none'
+
+            unless loc[ext]
+              other_ext.push(ext) unless other_ext.include?(ext)
               ext = 'other'
             end
 
             File.open(path,'r').each do |line|
-              if encode_line(line) !~ /^\s*$/ then
+              if encode_line(line) !~ /^\s*$/
                 loc[ext] = loc[ext] + 1
               end
             end
@@ -136,7 +136,7 @@ module Simp::Rake::Build
           puts
           puts "Unknown Extension Count: #{other_ext.length}"
 
-          if args.show_unknown then
+          if args.show_unknown
             puts "Unknown Extensions:"
             other_ext.sort.each do |ext|
               puts "  #{ext}"

data/lib/simp/rake/build/constants.rb

@@ -3,21 +3,100 @@ require 'rake/tasklib'
 module Simp::Rake; end
 module Simp::Rake::Build; end
 module Simp::Rake::Build::Constants
+  def os_build_metadata(distro=nil, version=nil, arch=nil)
+    unless @member_vars_initialized
+      init_member_vars(Dir.pwd)
+    end
+
+    metadata = nil
+
+    build_metadata_file = File.join(@distro_build_dir, 'build_metadata.yaml')
+
+    if File.exist?(build_metadata_file)
+      build_metadata = YAML.load_file(build_metadata_file)
+
+      if distro
+        begin
+          metadata_init = { 'distributions' => {} }
+          metadata = metadata_init.dup
+
+          if build_metadata['distributions'][distro]
+            if version && build_metadata['distributions'][distro][version]
+              metadata['distributions'][distro] = {}
+              metadata['distributions'][distro][version] = build_metadata['distributions'][distro][version].dup
+
+              if arch && build_metadata['distributions'][distro][version]['arch'].include?(arch)
+                metadata['distributions'][distro][version]['arch'] = Array(arch)
+              else
+                raise(NoMethodError)
+              end
+            else
+              metadata['distributions'][distro] = build_metadata['distributions'][distro].dup
+            end
+
+            # Build everything that we've selected
+            metadata['distributions'][distro].keys.each do |d|
+              metadata['distributions'][distro][d]['build'] = true
+            end
+          end
+
+          if metadata['distributions'].empty?
+            raise(NoMethodError)
+          end
+        rescue NoMethodError
+          $stderr.puts(%(Error: Could not find distribution for '#{ENV['SIMP_BUILD_distro']}'))
+          $stderr.puts(%(  Check #{File.expand_path(build_metadata_file)}))
+          exit(1)
+        end
+      else
+        metadata = build_metadata
+      end
+    end
+
+    return metadata
+  end
+
   def init_member_vars( base_dir )
-    @run_dir   = Dir.pwd
-    @base_dir  = base_dir
-    @build_arch = ENV['buld_arch'] || %x{#{:facter} hardwaremodel 2>/dev/null}.chomp
-    @build_dir  = "#{@base_dir}/build"
-    @dist_dir   = "#{@build_dir}/dist"
-    @dvd_dir    = "#{@build_dir}/DVD_Overlay"
-    @src_dir    = "#{@base_dir}/src"
-    @dvd_src       = "#{@src_dir}/DVD"
-    @spec_dir      = "#{@src_dir}/build"
-    @spec_file    = FileList["#{@spec_dir}/*.spec"]
-    @target_dists  = ['CentOS','RHEL']  # The first item is the default build...
-    @simp_version  = Simp::RPM.get_info("#{@spec_dir}/simp.spec")[:full_version]
-    @rhel_version  = ENV['rhel_version'] || '7'
-    @simp_dvd_dirs = ["SIMP","ks","Config"]
+    return if @member_vars_initialized
+
+    @run_dir           = Dir.pwd
+    @base_dir          = base_dir
+    @build_arch        = ENV['buld_arch'] || %x{#{:facter} hardwaremodel 2>/dev/null}.chomp
+    @build_dir         = File.join(@base_dir, 'build')
+    @dvd_dir           = File.join(@build_dir, 'DVD_Overlay')
+    @target_dists      = ['CentOS', 'RedHat']
+    @dist_dir          = File.join(@build_dir, 'dist')
+    @src_dir           = File.join(@base_dir, 'src')
+    @dvd_src           = File.join(@src_dir, 'DVD')
+    @spec_dir          = File.join(@src_dir, 'build')
+    @spec_file         = FileList[File.join(@spec_dir, '*.spec')]
+    @simp_version      = Simp::RPM.get_info(File.join(@spec_dir, 'simp.spec'))[:full_version]
+    @simp_dvd_dirs     = ["SIMP","ks","Config"]
+    @distro_build_dir  = File.join(@build_dir,'distributions')
+    @os_build_metadata = nil
+    @member_vars_initialized = true
+
+    if ENV['SIMP_BUILD_distro']
+      distro, version, arch = ENV['SIMP_BUILD_distro'].split(/,|\//)
+
+      @os_build_metadata = os_build_metadata(distro, version, arch)
+    else
+      @os_build_metadata = os_build_metadata()
+    end
+
+    if @os_build_metadata && !@os_build_metadata.empty?
+      $simp6 = true
+      $simp6_clean_dirs = []
+
+      @os_build_metadata['distributions'].keys.sort.each do |d|
+        @os_build_metadata['distributions'][d].keys.sort.each do |v|
+          next unless @os_build_metadata['distributions'][d][v]['build']
+          @os_build_metadata['distributions'][d][v]['arch'].sort.each do |a|
+            $simp6_clean_dirs << File.join(@distro_build_dir, d, v, a, 'SIMP')
+            $simp6_clean_dirs << File.join(@distro_build_dir, d, v, a, 'SIMP_ISO*')
+          end
+        end
+      end
+    end
   end
 end
-

data/lib/simp/rake/build/deps.rb

@@ -128,6 +128,8 @@ class R10KHelper
     @modules.each do |mod|
       module_dir = mod[:path].split(@basedir.to_s).last.split('/')[1..-2].join('/')
 
+      next unless mod[:r10k_module]
+
       if last_module_dir != module_dir
         pupfile << "moduledir '#{module_dir}'\n"
         last_module_dir = module_dir

data/lib/simp/rake/build/helpers.rb

@@ -13,7 +13,6 @@ class Simp::Rake::Build::Helpers
     end
     Simp::Rake::Build::Auto.new( dir )
     Simp::Rake::Build::Build.new( dir )
-    Simp::Rake::Build::Clean.new( dir )
     Simp::Rake::Build::Code.new( dir )
     Simp::Rake::Build::Deps.new( dir )
     Simp::Rake::Build::Iso.new( dir )
@@ -22,5 +21,6 @@ class Simp::Rake::Build::Helpers
     Simp::Rake::Build::Tar.new( dir )
     Simp::Rake::Build::Upload.new( dir )
     Simp::Rake::Build::Unpack.new( dir )
+    Simp::Rake::Build::Clean.new( dir )
   end
 end

data/lib/simp/rake/build/iso.rb

@@ -11,6 +11,7 @@ module Simp::Rake::Build
 
     def initialize( base_dir )
       init_member_vars( base_dir )
+
       @mock = ENV['mock'] || '/usr/bin/mock'
       define_tasks
     end
@@ -20,6 +21,11 @@ module Simp::Rake::Build
       File.umask(0007)
 
       namespace :iso do
+        task :prep do
+          if $simp6
+            @build_dir = $simp6_build_dir
+          end
+        end
 
         # Remove packages from the given directory. The goal of this method is to help
         # get the distro to be as small as possible.
@@ -98,13 +104,14 @@ module Simp::Rake::Build
        ENV vars:
          - Set `SIMP_ISO_verbose=yes` to report file operations as they happen.
            EOM
-        task :build,[:tarball,:unpacked_dvds,:prune] do |t,args|
+        task :build,[:tarball,:unpacked_dvds,:prune] => [:prep] do |t,args|
           args.with_defaults(:unpacked_dvds => "#{@run_dir}", :prune => 'true')
 
           if args.tarball.nil?
             fail("Error: You must specify a source tarball or tarball directory!")
           else
             tarball = File.expand_path(args.tarball)
+
             unless File.exist?(tarball)
               fail("Error: Could not find tarball at '#{tarball}'!")
             end
@@ -116,10 +123,17 @@ module Simp::Rake::Build
             Dir.glob("#{tarball}/*.tar.gz") : [tarball]
           vermap = YAML::load_file( File.join( File.dirname(__FILE__), 'vermap.yaml'))
 
-          tarfiles.each do |tarball|
+          tarfiles.each do |tball|
             namepieces = File.basename(tarball,".tar.gz").split('-')
-            simpver = namepieces[3..-1].join('-')
-            baseos = namepieces[2]
+
+            # SIMP 6
+            if namepieces[1] =~ /^\d/
+              simpver = namepieces[1..2].join('-')
+              baseos  = namepieces[3]
+            else
+              simpver = namepieces[3..-1].join('-')
+              baseos  = namepieces[2]
+            end
 
             iso_dirs = Dir.glob("#{File.expand_path(args.unpacked_dvds)}/#{baseos}*")
             if iso_dirs.empty?
@@ -173,14 +187,16 @@ module Simp::Rake::Build
 
               # Prune unwanted packages
               begin
-                system("tar --no-same-permissions -C #{dir} -xzf #{tarball} *simp_pkglist.txt")
+                system("tar --no-same-permissions -C #{dir} -xzf #{tball} *simp_pkglist.txt")
               rescue
                 # Does not matter if the command fails
               end
+
               pkglist_file = ENV.fetch(
-                                       'SIMP_PKGLIST_FILE',
-                                        File.join(dir,"#{baseosver.split('.').first}-simp_pkglist.txt")
-                                      )
+                'SIMP_PKGLIST_FILE',
+                File.join(dir,"#{baseosver.split('.').first}-simp_pkglist.txt")
+              )
+
               puts
               puts '-'*80
               puts "### Pruning packages not in file '#{pkglist_file}'"
@@ -200,13 +216,18 @@ module Simp::Rake::Build
               end
 
               # Add the SIMP code
-              system("tar --no-same-permissions -C #{dir} -xzf #{tarball}")
+              system("tar --no-same-permissions -C #{dir} -xzf #{tball}")
 
               Dir.chdir("#{dir}/SIMP") do
                 # Add the SIMP Dependencies
                 simp_base_ver = simpver.split('-').first
-                simp_dep_src = %(SIMP#{simp_base_ver}_#{baseos}#{baseosver}_#{arch})
-                yum_dep_location = File.join(@build_dir,'yum_data',simp_dep_src,'packages')
+
+                if $simp6
+                  yum_dep_location = File.join(@build_dir,'yum_data','packages')
+                else
+                  simp_dep_src = %(SIMP#{simp_base_ver}_#{baseos}#{baseosver}_#{arch})
+                  yum_dep_location = File.join(@build_dir,'yum_data',simp_dep_src,'packages')
+                end
 
                 unless File.directory?(yum_dep_location)
                   fail("Could not find dependency directory at #{yum_dep_location}")
@@ -220,7 +241,7 @@ module Simp::Rake::Build
                 # Add any one-off RPMs that you might want to add to your own build
                 # These are *not* checked to make sure that they actually match your
                 # environment
-                aux_packages = File.join(@build_dir,'yum_data',simp_dep_src,'aux_packages')
+                aux_packages = File.join(File.dirname(yum_dep_location),'aux_packages')
                 if File.directory?(aux_packages)
                   yum_dep_rpms += Dir.glob(File.join(aux_packages,'*.rpm'))
                 end
@@ -307,7 +328,7 @@ module Simp::Rake::Build
             * :key - The GPG key to sign the RPMs with. Defaults to 'prod'.
             * :chroot - An optional Mock Chroot. If this is passed, the tar:build task will be called.
         EOM
-        task :src,[:key,:chroot] do |t,args|
+        task :src,[:prep, :key,:chroot] do |t,args|
           args.with_defaults(:key => 'prod')
 
           if Dir.glob("#{@dvd_dir}/*.gz").empty? && !args.chroot