simp-rake-helpers 5.3.0 → 5.4.1

data/spec/acceptance/files/mock_packages/simp-adapter/usr/local/sbin/simp_rpm_helper

@@ -0,0 +1,406 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+#
+# Purpose
+# -------
+#
+# This script is meant to be called by the %preun and %post sections of the
+# various SIMP Puppet module RPMs.
+#
+# The purpose of the script is to provide helper methods that correctly
+# scaffold the system in such a way that all SIMP Puppet Modules can be
+# installed via RPM to a single location and subsequently can be copied into
+# the standard SIMP installation location based on the version of Puppet that
+# is installed.
+#
+# Care is taken that, should the target directory be managed via 'git', this
+# script will do nothing to harm the managed installation in any way. This
+# ensures that the SIMP modules have maximum compatibility with the widest
+# possible range of Puppet module best practices for management while ensuring
+# that the RPM installations and upgrades can proceed in a seamless fashion
+# over time.
+#
+# Should the 'simp' environment not be found, the script will simply exit
+# without copying any files.
+#
+# Configuration
+# -------------
+#
+# A configuration file may be placed at /etc/simp/adapter_config.yaml. The
+# file must consist of proper YAML as demonstrated in the example below which
+# lists the default options.
+#
+# Any configuration options that are not understood will be ignored.
+#
+# ```yaml
+#   ---
+#   # Target directory
+#   # May be set to a fully qualified path or 'auto'
+#   # If set to 'auto', the directory will be determined from puppet itself
+#
+#   target_directory : 'auto'
+#
+#   # Copy the RPM data to the target directory
+#
+#   copy_rpm_data : false
+#
+# ```
+#
+
+require 'facter'
+require 'fileutils'
+require 'yaml'
+require 'optparse'
+require 'ostruct'
+require 'find'
+
+# Make sure we can find the Puppet executables
+ENV['PATH'] += ':/opt/puppetlabs/bin'
+
+# Get the Puppet configuration parameters currently in use
+def get_puppet_config
+  system_config = %x{puppet config --section master print}
+
+  config_hash = Hash.new
+
+  system_config.each_line do |line|
+    k,v = line.split('=')
+    config_hash[k.strip] = v.strip
+  end
+
+  return config_hash
+end
+
+# A more friendly failure message
+def fail(msg)
+  $stderr.puts(msg)
+  exit(1)
+end
+
+# Determine whether the passed path is under management by git or svn
+def is_managed?(path)
+  # Short circuit if the directory is not present
+  return false unless File.directory?(path)
+
+  git = Facter::Core::Execution.which('git')
+  svn = Facter::Core::Execution.which('svn')
+
+  Dir.chdir(path) do
+    if git
+      %x{#{git} ls-files . --error-unmatch &> /dev/null}
+
+      return true if $?.success?
+    end
+
+    if svn
+      %x{#{svn} info &> /dev/null}
+
+      return true if $?.success?
+    end
+  end
+
+  return false
+end
+
+def parse_options
+  options = OpenStruct.new
+  options.config_file = '/etc/simp/adapter_config.yaml'
+  options.preserve = false
+  # These are not settable, but it's nice to have all this material in one place
+  options.puppet_user = @puppet_config['user']
+  options.puppet_group = @puppet_config['group']
+
+  all_opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+
+    opts.separator ""
+
+    opts.on(
+      "--rpm_dir PATH",
+      "The directory into which the RPM source material is installed"
+    ) do |arg|
+      options.rpm_dir = arg.strip
+      options.module_name = File.basename(options.rpm_dir)
+    end
+
+    opts.on(
+      "--rpm_section SECTION",
+      "The section of the RPM from which the script is being called.",
+      "    Must be one of 'pre', 'post', 'preun', 'postun'"
+    ) do |arg|
+      options.rpm_section = arg.strip
+    end
+
+    opts.on(
+      "--rpm_status STATUS",
+      "The status code passed to the RPM section"
+    ) do |arg|
+      options.rpm_status = arg.strip
+    end
+
+    opts.on(
+      "-f CONFIG_FILE",
+      "--config CONFIG_FILE",
+      "The configuration file to use",
+      "    Default: #{options.config_file}"
+    ) do |arg|
+      options.config_file = arg.strip
+    end
+
+    opts.on(
+      "-p",
+      "--preserve",
+      "Preserve material in 'target_dir' that is not in 'rpm_dir'"
+    ) do |arg|
+      options.preserve = true
+    end
+
+    opts.on(
+      "-e",
+      "--enforce",
+      "If set, enforce the copy, regardless of the setting in the config file",
+      "    Default: false"
+    ) do |arg|
+      options.copy_rpm_data = true
+    end
+
+    opts.on(
+      "-t DIR",
+      "--target_dir DIR",
+      "The subdirectory of #{simp_target_dir('')}",
+      "into which to copy the materials.",
+      "    Default: #{simp_target_dir.gsub(/#{simp_target_dir('')}/,'')}"
+    ) do |arg|
+      options.target_dir = simp_target_dir(arg.strip)
+    end
+
+    opts.on(
+      "-h",
+      "--help",
+      "Help Message"
+    ) do
+      puts opts
+      exit(0)
+    end
+  end
+
+  begin
+    all_opts.parse!(ARGV)
+  rescue OptionParser::ParseError => e
+    puts e
+    puts all_opts
+    exit 1
+  end
+
+  validate_options(options, all_opts.to_s)
+
+  return options
+end
+
+# Process the config, validate the entries and do some munging
+# Return an options hash
+def process_config(config_file, options=OpenStruct.new)
+  # Defaults
+  config = {
+    'target_directory' => 'auto',
+    'copy_rpm_data'    => false
+  }
+
+  if File.exist?(config_file)
+    begin
+      system_config = YAML.load_file(config_file)
+      if system_config
+        config.merge!(system_config)
+      end
+    rescue
+      fail("Error: Config file '#{config_file}' could not be processed")
+    end
+  end
+
+  if options.copy_rpm_data.nil?
+    options.copy_rpm_data = (config['copy_rpm_data'].to_s == 'true')
+  end
+
+  if options.target_dir.nil? && config['target_directory']
+    if config['target_directory'] == 'auto'
+      options.target_dir = simp_target_dir
+    else
+      unless config['target_directory'][0].chr == '/'
+        fail("Error: 'target_directory' in '#{config_file}' must be an absolute path")
+      end
+
+      options.target_dir = config['target_directory'].strip
+    end
+  end
+
+  return options
+end
+
+def puppet_codedir
+  # Figure out where the Puppet code should go
+  # Puppet 4+
+  code_dir = @puppet_config['codedir']
+  if !code_dir || code_dir.empty?
+    code_dir = @puppet_config['confdir']
+  end
+
+  return code_dir
+end
+
+# Return the target installation directory
+def simp_target_dir(subdir=File.join('simp','modules'))
+  install_target = puppet_codedir
+
+  if install_target.empty?
+    fail('Error: Could not find a Puppet code directory for installation')
+  end
+
+  install_target = File.join(install_target,'environments', subdir)
+
+  return install_target
+end
+
+# Input Validation
+def validate_options(options, usage)
+
+  unless options.rpm_dir
+    fail("Error: 'rpm_dir' is required\n#{usage}")
+  end
+
+  unless options.rpm_status
+    fail("Error: 'rpm_status' is required\n#{usage}")
+  end
+
+  unless options.rpm_section
+    fail("Error: 'rpm_section' is required\n#{usage}")
+  end
+
+  valid_rpm_sections = ['pre','post','preun','postun']
+
+  unless valid_rpm_sections.include?(options.rpm_section)
+    fail("Error: 'rpm_section' must be one of '#{valid_rpm_sections.join("', '")}'\n#{usage}")
+  end
+
+  if (options.rpm_section == 'post') || (options.rpm_section == 'preun')
+    unless File.directory?(options.rpm_dir)
+      fail("Error: Could not find 'rpm_dir': '#{options.rpm_dir}'")
+    end
+  end
+
+  unless options.rpm_status =~ /^\d+$/
+    fail("Error: 'rpm_status' must be an integer\n#{usage}")
+  end
+
+  fail('Error: Could not determine puppet user') if options.puppet_user.empty?
+  fail('Error: Could not determine puppet group') if options.puppet_group.empty?
+end
+
+### Main Code
+
+@puppet_config = get_puppet_config
+options = parse_options
+options = process_config(options.config_file, options)
+
+# A list of modules that should never be touched once installed
+safe_modules = ['site']
+
+# If the target directory is managed, we're done
+unless is_managed?(File.join(options.target_dir, options.module_name)) || !options.copy_rpm_data
+
+  rsync = Facter::Core::Execution.which('rsync')
+  fail("Error: Could not find 'rsync' command!") unless rsync
+
+  # A regular installation or upgrade
+  if options.rpm_section == 'post'
+    if safe_modules.include?(options.module_name)
+      # Make sure that we preserve anything in the safe modules on installation
+      options.preserve = true
+
+      if options.rpm_status == '2'
+        # Short circuit on upgrading safe modules, just don't touch them!
+        if File.directory?(File.join(options.target_dir, options.module_name))
+          exit(0)
+        end
+      end
+    end
+
+    # Create the directories, with the proper mode, all the way down
+    dir_paths = options.target_dir.split(File::SEPARATOR).reject(&:empty?)
+    top_dir = File::SEPARATOR + dir_paths.shift
+    unless File.directory?(top_dir)
+      FileUtils.mkdir(top_dir, :mode => 0750)
+      FileUtils.chown('root', options.puppet_group, top_dir)
+    end
+
+    orig_dir = Dir.pwd
+    Dir.chdir(top_dir)
+    dir_paths.each do |dir|
+      unless File.directory?(dir)
+        FileUtils.mkdir(dir, :mode => 0750)
+        FileUtils.chown('root', options.puppet_group, dir)
+      end
+
+      Dir.chdir(dir)
+    end
+    Dir.chdir(orig_dir)
+
+    cmd = %(#{rsync} -a --force)
+
+    if options.preserve
+      cmd += %( --ignore-existing)
+    else
+     cmd += %( --delete)
+    end
+
+    cmd += %( #{options.rpm_dir} #{options.target_dir})
+    cmd += %( 2>&1)
+
+    output = %x{#{cmd}}
+    unless $?.success?
+      fail(%(Error: Copy of '#{options.module_name}' into '#{options.target_dir}' using '#{cmd}' failed with the following error:\n    #{output.gsub("\n","\n    ")}))
+    end
+
+    FileUtils.chown_R(nil, "#{options.puppet_group}", options.target_dir)
+  end
+
+  # A regular uninstall or downgrade
+  # This needs to happen *before* we uninstall since we need to compare with what's on disk
+  if options.rpm_section == 'preun' && options.rpm_status == '0'
+    # Play it safe, this needs to have at least 'environments/simp' in it!
+    if options.target_dir.split(File::SEPARATOR).reject(&:empty?).size < 3
+      fail("Error: Not removing directory '#{options.target_dir}' for safety")
+    else
+      unless safe_modules.include?(options.module_name)
+        # Find out what we have
+        ref_list = []
+        Dir.chdir(options.rpm_dir) do
+          Find.find('.').each do |file|
+            if File.symlink?(file)
+              ref_list << file
+              Find.prune
+            end
+
+            ref_list << file
+          end
+        end
+
+        # Delete from the bottom up to clear out the directories first
+        ref_list.reverse!
+        ref_list.map{|x| x.sub!(/^./, options.module_name)}
+
+        # Only delete items that are in the reference repo
+        Dir.chdir(options.target_dir) do
+          ref_list.each do |to_rm|
+            if File.symlink?(to_rm)
+              FileUtils.rm_f(to_rm)
+            elsif File.directory?(to_rm) && (Dir.entries(to_rm).delete_if {|dir|
+                dir == '.' || dir == '..'}.size == 0)
+              FileUtils.rmdir(to_rm)
+            elsif File.exist?(to_rm)
+              FileUtils.rm_f(to_rm)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/acceptance/files/mock_packages/simp-adapter.spec

@@ -0,0 +1,43 @@
+Name:           simp-adapter
+Version:        0.9.9
+Release:        9
+Summary:        mock simplib test package
+BuildArch:      noarch
+
+License:        Apache-2.0
+URL:            http://foo.bar
+
+Source0: files.tar.gz
+
+%description
+A mock RPM package used for acceptance tests
+
+%prep
+echo ================ PWD: $PWD
+%setup -c
+
+%build
+
+
+%install
+[ "%{buildroot}" != "/" ] && rm -rf %{buildroot}
+mkdir -p %{buildroot}/opt/test/puppet/code
+cp -r $PWD/* %{buildroot}/
+rm -rf %{buildroot}/**/.*.swp
+rm -rf %{buildroot}/opt/puppetlabs
+
+%clean
+
+%files
+%defattr(0640,root,root,0755)
+
+%attr(0750,root,root)/usr/local/sbin/simp_rpm_helper
+#%attr(0755,root,root)/opt/puppetlabs/puppet/bin/ruby
+#%attr(0755,root,root)/opt/puppetlabs/bin/puppet
+%attr(0644,root,root)/etc/simp/adapter_config.yaml
+%dir  /opt/test/puppet/code
+
+
+%changelog
+* Fri Feb 23 2018 nobody
+- some comment

data/spec/acceptance/files/testpackage_custom_scriptlet/CHANGELOG

@@ -0,0 +1,2 @@
+* Sat Dec 13 2014 Richard Stallman <rms@gnu.org> - 0.0.1-0
+- Behold, there was a brave GNU world....

data/spec/acceptance/files/testpackage_custom_scriptlet/Rakefile

@@ -0,0 +1,3 @@
+require 'simp/rake/pkg'
+
+Simp::Rake::Pkg.new( File.dirname( __FILE__ ) )

data/spec/acceptance/files/testpackage_custom_scriptlet/build/rpm_metadata/custom/overrides

@@ -0,0 +1,18 @@
+%pretrans -p <lua>
+-- Custom scriptlet
+
+%triggerun -- bar
+# Custom trigger
+echo "This trigger runs just before the 'bar' package's %%preun"
+
+%triggerun -- foo
+# Custom trigger
+echo "The 'foo' package is great; why would you uninstall it?"
+
+%pre
+# Custom scriptlet override
+echo 'I override the default %%pre section provided by the spec file.'
+
+%triggerun -- foo
+# Custom trigger
+echo "This is a duplicate definition and won't be included"

data/spec/acceptance/files/testpackage_custom_scriptlet/build/rpm_metadata/requires

@@ -0,0 +1 @@
+Requires: pupmod-simp-foo >= 1.2.3

data/spec/acceptance/files/testpackage_custom_scriptlet/metadata.json

@@ -0,0 +1,33 @@
+{
+  "name":    "simp-testpackage",
+  "version": "0.0.1",
+  "author":  "simp",
+  "summary": "A SIMP Test Package for Testing Test Things",
+  "license": "Apache-2.0",
+  "source":  "https://github.com/simp/pupmod-imatest",
+  "project_page": "https://github.com/simp/pupmod-imatest",
+  "issues_url":   "https://simp/project.atlassian.net",
+  "tags": [ "simp", "test"],
+  "dependencies": [
+    {
+      "name": "simp/simplib",
+      "version_requirement": ">= 1.1.0"
+    }
+  ],
+  "operatingsystem_support": [
+    {
+      "operatingsystem": "CentOS",
+      "operatingsystemrelease": [
+        "6",
+        "7"
+      ]
+    },
+    {
+      "operatingsystem": "RedHat",
+      "operatingsystemrelease": [
+        "6",
+        "7"
+      ]
+    }
+  ]
+}

data/spec/acceptance/nodesets/default.yml

@@ -15,8 +15,7 @@ HOSTS:
       - 'useradd -b /home -m -c "Build User" -s /bin/bash -U build_user'
       - 'yum install -y facter'
       # simp build-deps
-      - 'yum install -y rpm-build augeas-devel createrepo genisoimage git gnupg2 libicu-devel libxml2 libxml2-devel libxslt libxslt-devel mock rpmdevtools clamav which'
-      - 'usermod -a -G mock build_user'
+      - 'yum install -y rpm-build augeas-devel createrepo genisoimage git gnupg2 libicu-devel libxml2 libxml2-devel libxslt libxslt-devel rpmdevtools which'
       # rvm build-deps
       - 'yum install -y libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel readline-devel libffi-devel openssl-devel automake libtool bison sqlite-devel'
       - 'runuser build_user -l -c "gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3"'
@@ -51,8 +50,7 @@ HOSTS:
       - 'useradd -b /home -m -c "Build User" -s /bin/bash -U build_user'
       - 'yum install -y facter'
       # simp build-deps
-      - 'yum install -y rpm-build augeas-devel createrepo genisoimage git gnupg2 libicu-devel libxml2 libxml2-devel libxslt libxslt-devel mock rpmdevtools clamav clamav-update which'
-      - 'usermod -a -G mock build_user'
+      - 'yum install -y rpm-build augeas-devel createrepo genisoimage git gnupg2 libicu-devel libxml2 libxml2-devel libxslt libxslt-devel rpmdevtools clamav-update which'
       # rvm build-deps
       - 'yum install -y libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel readline-devel libffi-devel openssl-devel automake libtool bison sqlite-devel'
       - 'runuser build_user -l -c "gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3"'

data/spec/acceptance/support/pkg_rpm_helpers.rb

@@ -0,0 +1,81 @@
+module Simp::BeakerHelpers::SimpRakeHelpers::PkgRpmHelpers
+
+  # rake command string to run on hosts
+  # passes on useful troubleshooting env vars
+  def rake_cmd
+    cmd = 'bundle exec rake'
+    %w(
+      SIMP_RPM_LUA_debug
+      SIMP_RAKE_PKG_verbose
+      SIMP_RPM_verbose
+    ).each do |env_var|
+      if value = ENV[env_var]
+         cmd = "#{env_var}=#{value} #{cmd}"
+      end
+    end
+    cmd
+  end
+
+
+  def copy_host_files_into_build_user_homedir(
+    hosts,
+    root_dir=File.expand_path('../../../',__FILE__)
+  )
+    # I've added the `ch* -R` on the SUT-side, which seems to work on a fresh checkout
+    on hosts, 'cp -a /host_files /home/build_user/; ' +
+             'chmod -R go=u-w /home/build_user/host_files/{dist,**/dist,junit,log}; ' +
+             'chown -R build_user:build_user /home/build_user/host_files; '
+  end
+
+
+  # key   = what `rpm -q --scripts` calls each scriptlet
+  # value = the label passed to `simp_rpm_helper`
+  def scriptlet_label_map
+    {
+      'pretrans'      => nil,
+      'preinstall'    => 'pre',
+      'postinstall'   => 'post',
+      'preuninstall'  => 'preun',
+      'postuninstall' => 'postun',
+      'posttrans'     => nil,
+    }
+  end
+
+
+  # returns a Hash of information about an RPM file's scriptlets
+  def rpm_scriptlets_for( host, rpm_file )
+    _labels  = scriptlet_label_map.keys.join('|')
+    rx_scriptlet_blocks = /^(?<block>(?<scriptlet>#{_labels}) scriptlet.*?(\r|\n)(?<content>.*?))(?=\n#{_labels}|\Z)/m
+
+    result = on host, %Q(rpm -qp --scripts #{rpm_file})
+
+    scriptlets = {}
+    result.stdout.to_s.scan(rx_scriptlet_blocks) do
+      scriptlet = scriptlets[$~[:scriptlet]] ||= { :count => 0 }
+      scriptlet[:count]       += 1
+      scriptlet[:content]      = $~[:content].strip
+      scriptlet[:full_block]   = $~[:block]
+      scriptlet[:bare_content] = scriptlet[:content].gsub(/^((--|#).*?[\r\n]+)/,'')
+    end
+    scriptlets
+  end
+
+
+  # returns a Hash of information about an RPM file's triggers
+  def rpm_triggers_for( host, rpm_file )
+    _trigger          = 'trigger\\w+ scriptlet \\(using [\\/a-z0-9]+\\) --(!?\\p{Graph}|\\s)*?'
+    rx_trigger_blocks = /^(?<block>(?<trigger>#{_trigger})[\r\n](?<content>.*?)(?=\n#{_trigger}|\Z))/m
+
+    result = on host, %Q(rpm -qp --triggers #{rpm_file})
+
+    triggers = {}
+    result.stdout.to_s.scan(rx_trigger_blocks) do
+      trigger=  triggers[$~[:trigger]] ||= { :count => 0 }
+      trigger[:count]       += 1
+      trigger[:content]      = $~[:content].strip
+      trigger[:full_block]   = $~[:block]
+      trigger[:bare_content] = trigger[:content].gsub(/^((--|#).*?[\r\n]+)/,'')
+    end
+    triggers
+  end
+end

data/spec/lib/simp/rake/build/files/unmanaged_mod/build/rpm_metadata/requires

@@ -1,4 +1,5 @@
-Obsoletes: pupmod-unmanaged_mod-test >= 0.0.1
+Obsoletes: pupmod-unmanaged_mod-test < 0.0.1
+Provides: pupmod-unmanaged_mod-test = 0.0.1
 Requires: pupmod-puppetlabs-inifile < 2.0.0-0
 Requires: pupmod-puppetlabs-inifile >= 1.6.0-0
 Requires: pupmod-puppetlabs-postgresql < 5.0.0-0

data/spec/lib/simp/rake/build/rpmdeps_spec.rb

@@ -63,7 +63,8 @@ describe 'Simp::Rake::Build::RpmDeps#generate_rpm_meta_files' do
       requires_file = File.join(mod_dir, 'build', 'rpm_metadata', 'requires')
       expect(File.exist?(requires_file)).to be true
       expected = <<EOM
-Obsoletes: pupmod-oldowner-changed_name_mod >= 2.5.0-2016.1
+Obsoletes: pupmod-oldowner-changed_name_mod < 2.5.0-2016.obsolete
+Provides: pupmod-oldowner-changed_name_mod = 2.5.0-2016.obsolete
 Requires: pupmod-foo1-bar1 = 1.0.0
 Requires: pupmod-foo2-bar2 > 2.0.0
 Requires: pupmod-foo3-bar3 < 3.0.0

data/spec/spec_helper_acceptance.rb

@@ -1,9 +1,28 @@
 require 'beaker-rspec'
-require 'tmpdir'
 require 'simp/beaker_helpers'
 include Simp::BeakerHelpers
+require 'tmpdir'
+require 'pry' if ENV['PRY'] == 'yes'
+
+$LOAD_PATH.unshift(File.expand_path('../acceptance/support',__FILE__))
+
+module Simp::BeakerHelpers::SimpRakeHelpers
+  # Add RSpec log comments within examples ("it blocks")
+  def comment(msg, indent=10)
+    logger.optionally_color(Beaker::Logger::MAGENTA, " "*indent + msg)
+  end
+
+  # basic command + arguments for executing `runuser` within an SUT
+  def run_cmd
+    @run_cmd ||= 'runuser build_user -l -c '
+  end
+end
 
 RSpec.configure do |c|
+  # provide helper methods to individual examples AND example groups
+  c.include Simp::BeakerHelpers::SimpRakeHelpers
+  c.extend Simp::BeakerHelpers::SimpRakeHelpers
+
   # Readable test descriptions
   c.formatter = :documentation