simp-beaker-helpers 1.19.3 → 1.21.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9eede146380997a5256ef8fc379350b7b2b44e5a4a1156e5a14f13a5be0d396d
-  data.tar.gz: 31345b9736832c517f96e327dd18ddaca39c9eb72b0d94b9b56e83344f83d645
+  metadata.gz: 876c4f6b1c6dfe6c9230d0c127e6dabdd3cc77c372e0d6def7e42606ecd35cc6
+  data.tar.gz: 062201bba85a2199e442d907b30aae6e2674db29de2c21c7bc49e5ef5dd7c117
 SHA512:
-  metadata.gz: 89e30957c049a514d97f58e99f2668682fa956e7a43a7414e69dff18d58f9af2f7ac2bf715ba72af97ffe6cdf93630dc3eeb68a45096b07adad069a907abaa93
-  data.tar.gz: 004cf046d60282beeecc534f39b45d4398cf4b35bccc0b9e46a6cdddf260cd2fd222ebb796e66e10f6cd9e518b7b806584716ccc0eb8734ff2c00181a3a53748
+  metadata.gz: 8469baf544e003f9450a6c0b5639a2112c36961846e06faf6222878b5a1f0d6eb5663fd3c921d6b2117174cfdcf74863da67c40364c34a3818a5eb1204a94901
+  data.tar.gz: 2810e22a457f9daf3d91e355d2ead18e1b3dd5aaecbebc8f516930968cc937b1a9322750d305591db47d3f2d0d25cd8895c9189c16ee71f9505756eb447015b0

data/.gitignore

@@ -1,3 +1,4 @@
+.fips_fixtures.yml
 *.gem
 *.swp
 Gemfile.lock

data/.gitlab-ci.yml

@@ -97,28 +97,25 @@ variables:
   changes:
     - .gitlab-ci.yml
     - .fixtures.yml
+    - .rspec
+    - metadata.json
     - "spec/spec_helper.rb"
-    - "spec/{classes,unit,defines,type_aliases,types,hosts}/**/*.rb"
-    - "{manifests,files,types}/**/*"
-    - "templates/*.{erb,epp}"
-    - "lib/**/*"
+    - "spec/{classes,unit,defines,type_aliases,types,hosts,lib}/**/*.rb"
+    - "{SIMP,data,manifests,files,types,lib}/**/*"
+    - "templates/**/*.{erb,epp}"
     - "Gemfile"
-    - "SIMP/**/*"
-    - "data/**/*"
   exists:
-    - "spec/{classes,unit,defines,type_aliases,types,hosts}/**/*_spec.rb"
+    - "spec/{classes,unit,defines,type_aliases,types,hosts,lib}/**/*_spec.rb"
 
 .relevant_file_conditions_trigger_acceptance_tests: &relevant_file_conditions_trigger_acceptance_tests
   changes:
     - .gitlab-ci.yml
+    - .fixtures.yml
     - "spec/spec_helper_acceptance.rb"
     - "spec/acceptance/**/*"
-    - "{manifests,files,types}/**/*"
-    - "templates/*.{erb,epp}"
-    - "lib/**/*"
+    - "{SIMP,data,manifests,files,types,lib}/**/*"
+    - "templates/**/*.{erb,epp}"
     - "Gemfile"
-    - "SIMP/**/*"
-    - "data/**/*"
   exists:
     - "spec/acceptance/**/*_spec.rb"
 
@@ -286,21 +283,14 @@ variables:
 pup5-unit:
   <<: *pup_5
   <<: *unit_tests
-  <<: *with_SIMP_SPEC_MATRIX_LEVEL_2
 
 pup6-unit:
   <<: *pup_6
   <<: *unit_tests
-  <<: *with_SIMP_SPEC_MATRIX_LEVEL_2
-
-pup6.18.0-unit:
-  <<: *pup_6_18_0
-  <<: *unit_tests
 
 pup7-unit:
   <<: *pup_7
   <<: *unit_tests
-  <<: *with_SIMP_SPEC_MATRIX_LEVEL_2
 
 # ------------------------------------------------------------------------------
 #             NOTICE: **This file is maintained with puppetsync**
@@ -312,6 +302,26 @@ pup7-unit:
 # Repo-specific content
 # ==============================================================================
 
+#=======================================================================
+# Packaging test
+
+pup5-pkg:
+  <<: *pup_5
+  <<: *unit_tests
+  script:
+    'bundle exec rake pkg:gem'
+
+pup6-pkg:
+  <<: *pup_6
+  <<: *unit_tests
+  script:
+    'bundle exec rake pkg:gem'
+
+pup7-pkg:
+  <<: *pup_7
+  <<: *unit_tests
+  script:
+    'bundle exec rake pkg:gem'
 
 #=======================================================================
 # Acceptance tests

data/CHANGELOG.md

@@ -1,3 +1,54 @@
+### 1.21.3 / 2021-01-20
+* Fixed:
+  * Allow all methods that can safely take SUT arrays to do so
+  * Ensure that pfact_on returns a Hash if appropriate
+  * Fix container support in copy_to
+* Added:
+  * Explicitly support podman local and remote in copy_to
+
+### 1.21.2 / 2021-01-15
+* Fixed version mismatch.  1.21.1 was tagged with an incorrect version
+  in version.rb.
+
+### 1.21.1 / 2021-01-13
+* Added:
+  * update_package_from_centos_stream method
+  * install_latest_package_on method
+* Fixed:
+  * Removed some of the extraneous calls to facter
+  * Automatically pull the CentOS 8 kernel to the latest version in
+    CentOS-Stream to work around issues on FIPS systems
+
+### 1.20.1 / 2021-01-08
+* Fixed:
+  * Ensure that yum calls commands appropriately depending on whether or not
+    packages are already installed.
+  * Also change all HostKeyAlgorithms settings for SSH connections
+
+### 1.20.0 / 2021-01-05
+* Added:
+  * A `enable_epel_on` function that follows the instructions on the EPEL
+    website to properly enable EPEL on hosts. May be disabled using
+    `BEAKER_enable_epel=no`.
+  * An Ubuntu nodeset to make sure our default settings don't destroy other
+    Linux systems.
+  * Added has_crypto_policies method for determining if crypto policies are
+    present on the SUT
+  * Added munge_ssh_crypto_policies to allow vagrant to SSH back into systems
+    with restrictive crypto policies (usually FIPS)
+* Fixed:
+  * Modify all crypto-policy backend files to support ssh-rsa keys
+  * Try harder when doing yum installations
+
+### 1.19.4 / 2021-01-05
+* Fixed:
+  * Only return a default empty string when `pfact_on` finds a `nil` value
+    * Added an acceptance test to validate this
+  * Ensure that we start with `facter -p` for `facter` < 4.0 and continue to
+      `puppet facts` otherwise
+  * Updated the Rakefile to skip symlinks in chmods which fixes the ability to
+    build gems
+
 ### 1.19.3 / 2021-01-01
 * Fixed:
   * Ensure that `pfact_on` can handle fact dot notation

data/Rakefile

@@ -30,7 +30,7 @@ task :chmod do
   gemspec = File.expand_path( "#{@package}.gemspec", @rakefile_dir ).strip
   spec = Gem::Specification::load( gemspec )
   spec.files.each do |file|
-    FileUtils.chmod 'go=r', file
+    FileUtils.chmod 'go=r', file unless File.symlink?(file)
   end
 end
 

data/lib/simp/beaker_helpers.rb

@@ -18,6 +18,33 @@ module Simp::BeakerHelpers
     "simp-beaker-helpers-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}.tmp"
   end
 
+  def install_latest_package_on(suts, package_name, package_source=nil, opts={})
+    default_opts = {
+      max_retries: 3,
+      retry_interval: 10
+    }
+
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      package_source = package_name unless package_source
+
+      if sut.check_for_package(package_name)
+        sut.upgrade_package(
+          package_source,
+          '',
+          default_opts.merge(opts)
+        )
+      else
+        sut.install_package(
+          package_source,
+          '',
+          nil,
+          default_opts.merge(opts)
+        )
+      end
+    end
+  end
+
   def is_windows?(sut)
     sut[:platform] =~ /windows/i
   end
@@ -80,7 +107,30 @@ module Simp::BeakerHelpers
       else
         container_id = sut.host_hash[:docker_container_id]
       end
-      %x(tar #{exclude_list.join(' ')} -hcf - -C "#{File.dirname(src)}" "#{File.basename(src)}" | docker exec -i "#{container_id}" tar -C "#{dest}" -xf -)
+
+      if ENV['BEAKER_docker_cmd']
+        docker_cmd = ENV['BEAKER_docker_cmd']
+      else
+        docker_cmd = 'docker'
+
+        if ::Docker.version['Components'].any?{|x| x['Name'] =~ /podman/i}
+          docker_cmd = 'podman'
+
+          if ENV['CONTAINER_HOST']
+            docker_cmd = 'podman --remote'
+          elsif ENV['DOCKER_HOST']
+            docker_cmd = "podman --remote --url=#{ENV['DOCKER_HOST']}"
+          end
+        end
+      end
+
+      unless directory_exists_on(sut, dest)
+        dest = File.dirname(dest)
+        sut.mkdir_p(dest)
+      end
+
+      %x(tar #{exclude_list.join(' ')} -hcf - -C "#{File.dirname(src)}" "#{File.basename(src)}" | #{docker_cmd} exec -i "#{container_id}" tar -C "#{dest}" -xf -)
+
     elsif rsync_functional_on?(sut)
       # This makes rsync_to work like beaker and scp usually do
       exclude_hack = %(__-__' -L --exclude '__-__)
@@ -118,13 +168,35 @@ module Simp::BeakerHelpers
   def pfact_on(sut, fact_name)
     require 'ostruct'
 
+    found_fact = nil
+    # If puppet is not installed, there are no puppet facts to fetch
     if sut.which('puppet').empty?
-      fact_on(sut, fact_name, :silent => true)
+      found_fact = fact_on(sut, fact_name)
     else
-      facts_json = on(sut,'puppet facts find garbage_xxx', :silent => true).stdout
-      facts      = JSON.parse(facts_json, object_class: OpenStruct).values
-      facts.dig(*(fact_name.split('.'))) || ''
+      facts_json = nil
+      begin
+        cmd_output = on(sut, 'facter -p --json', :silent => true)
+
+        # Facter 4+
+        raise('skip facter -p') if (cmd_output.stderr =~ /no longer supported/)
+
+        facts = JSON.parse(cmd_output.stdout, object_class: OpenStruct)
+      rescue StandardError
+        # If *anything* fails, we need to fall back to `puppet facts`
+
+        facts_json = on(sut, 'puppet facts find garbage_xxx', :silent => true).stdout
+        facts = JSON.parse(facts_json, object_class: OpenStruct).values
+      end
+
+      found_fact = facts.dig(*(fact_name.split('.')))
+
+      # If we did not find a fact, we should use the upstream function since
+      # puppet may be installed via a gem or through some other means.
+      found_fact = fact_on(sut, fact_name) if found_fact.nil?
     end
+
+    # Ensure that Hashes return as Hash objects
+    found_fact.is_a?(OpenStruct) ? found_fact.marshal_dump : found_fact
   end
 
   # Returns the modulepath on the SUT, as an Array
@@ -302,6 +374,22 @@ module Simp::BeakerHelpers
     pluginsync_on(suts) if opts[:pluginsync]
   end
 
+  def has_crypto_policies(sut)
+    file_exists_on(sut, '/etc/crypto-policies/config')
+  end
+
+  def munge_ssh_crypto_policies(suts, key_types=['ssh-rsa'])
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      if has_crypto_policies(sut)
+        install_latest_package_on(sut, 'crypto-policies', nil, :accept_all_exit_codes => true)
+
+        # Since we may be doing this prior to having a box flip into FIPS mode, we
+        # need to find and modify *all* of the affected policies
+        on( sut, %{sed --follow-symlinks -i 's/\\(HostKeyAlgorithms\\|PubkeyAcceptedKeyTypes\\)\\(.\\)/\\1\\2#{key_types.join(',')},/g' $( grep -L ssh-rsa $( find /etc/crypto-policies /usr/share/crypto-policies -type f -a \\( -name '*.txt' -o -name '*.config' \\) -exec grep -l PubkeyAcceptedKeyTypes {} \\; ) ) })
+      end
+    end
+  end
 
   # Configure and reboot SUTs into FIPS mode
   def enable_fips_mode_on( suts = hosts )
@@ -309,7 +397,10 @@ module Simp::BeakerHelpers
     puts '  -- (use BEAKER_fips=no to disable)'
     parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
 
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
     block_on(suts, :run_in_parallel => parallel) do |sut|
+      next if sut[:hypervisor] == 'docker'
+
       if is_windows?(sut)
         puts "  -- SKIPPING #{sut} because it is windows"
         next
@@ -355,17 +446,14 @@ module Simp::BeakerHelpers
         on(sut, module_install_cmd)
       end
 
-      # Enable FIPS and then reboot to finish.
-      on(sut, %(puppet apply --verbose #{fips_enable_modulepath} -e "class { 'fips': enabled => true }"))
-
       # Work around Vagrant and cipher restrictions in EL8+
       #
       # Hopefully, Vagrant will update the used ciphers at some point but who
       # knows when that will be
-      opensshserver_config = '/etc/crypto-policies/back-ends/opensshserver.config'
-      if file_exists_on(sut, opensshserver_config)
-        on(sut, "sed --follow-symlinks -i 's/PubkeyAcceptedKeyTypes=/PubkeyAcceptedKeyTypes=ssh-rsa,/' #{opensshserver_config}")
-      end
+      munge_ssh_crypto_policies(sut)
+
+      # Enable FIPS and then reboot to finish.
+      on(sut, %(puppet apply --verbose #{fips_enable_modulepath} -e "class { 'fips': enabled => true }"))
 
       sut.reboot
     end
@@ -458,102 +546,171 @@ module Simp::BeakerHelpers
       repo_manifest = repo_manifest + %(\n#{repo_manifest_opts.join(",\n")}) + "\n}\n"
   end
 
-  def linux_errata( sut )
-    # We need to be able to flip between server and client without issue
-    on sut, 'puppet resource group puppet gid=52'
-    on sut, 'puppet resource user puppet comment="Puppet" gid="52" uid="52" home="/var/lib/puppet" managehome=true'
-
-    # Make sure we have a domain on our host
-    current_domain = fact_on(sut, 'domain').strip
-    hostname = fact_on(sut, 'hostname').strip
-
-    if current_domain.empty?
-      new_fqdn = hostname + '.beaker.test'
+  # Enable EPEL if appropriate to do so and the system is online
+  #
+  # Can be disabled by setting BEAKER_enable_epel=no
+  def enable_epel_on(suts)
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      if ONLINE && (ENV['BEAKER_stringify_facts'] != 'no')
+        os_info = fact_on(sut, 'os')
+        os_maj_rel = os_info['release']['major']
+
+        # This is based on the official EPEL docs https://fedoraproject.org/wiki/EPEL
+        if ['RedHat', 'CentOS'].include?(os_info['name'])
+          install_latest_package_on(
+            sut,
+            'epel-release',
+            "https://dl.fedoraproject.org/pub/epel/epel-release-latest-#{os_maj_rel}.noarch.rpm",
+          )
+
+          if os_info['name'] == 'RedHat'
+            if os_maj_rel == '7'
+              on sut, %{subscription-manager repos --enable "rhel-*-optional-rpms"}
+              on sut, %{subscription-manager repos --enable "rhel-*-extras-rpms"}
+              on sut, %{subscription-manager repos --enable "rhel-ha-for-rhel-*-server-rpms"}
+            end
 
-      on(sut, "sed -i 's/#{hostname}.*/#{new_fqdn} #{hostname}/' /etc/hosts")
-      on(sut, "echo '#{new_fqdn}' > /etc/hostname", :accept_all_exit_codes => true)
-      on(sut, "hostname #{new_fqdn}", :accept_all_exit_codes => true)
+            if os_maj_rel == '8'
+              on sut, %{subscription-manager repos --enable "codeready-builder-for-rhel-8-#{os_info['architecture']}-rpms"}
+            end
+          end
 
-      if sut.file_exist?('/etc/sysconfig/network')
-        on(sut, "sed -s '/HOSTNAME=/d' /etc/sysconfig/network")
-        on(sut, "echo 'HOSTNAME=#{new_fqdn}' >> /etc/sysconfig/network")
+          if os_info['name'] == 'CentOS'
+            if os_maj_rel == '8'
+              # 8.0 fallback
+              install_latest_package_on(sut, 'dnf-plugins-core')
+              on sut, %{dnf config-manager --set-enabled powertools || dnf config-manager --set-enabled PowerTools}
+            end
+          end
+        end
       end
     end
+  end
 
-    if fact_on(sut, 'domain').strip.empty?
-      fail("Error: hosts must have an FQDN, got domain='#{current_domain}'")
+  def update_package_from_centos_stream(suts, package_name)
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      sut.install_package('centos-release-stream') unless sut.check_for_package('centos-release-stream')
+      install_latest_package_on(sut, package_name)
+      sut.uninstall_package('centos-release-stream')
     end
+  end
 
-    # This may not exist in docker so just skip the whole thing
-    if sut.file_exist?('/etc/ssh')
-      # SIMP uses a central ssh key location so we prep that spot in case we
-      # flip to the SIMP SSH module.
-      on(sut, 'mkdir -p /etc/ssh/local_keys')
-      on(sut, 'chown -R root:root /etc/ssh/local_keys')
-      on(sut, 'chmod 755 /etc/ssh/local_keys')
-
-      user_info = on(sut, 'getent passwd').stdout.lines
-
-      # Hash of user => home_dir
-      # Exclude silly directories
-      #   * /
-      #   * /dev/*
-      #   * /s?bin
-      #   * /proc
-      user_info = Hash[
-        user_info.map do |u|
-          u.strip!
-          u = u.split(':')
-          u[5] =~ %r{^(/|/dev/.*|/s?bin/?.*|/proc/?.*)$} ? [nil] : [u[0], u[5]]
-        end
-      ]
+  def linux_errata( suts )
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      # We need to be able to flip between server and client without issue
+      on sut, 'puppet resource group puppet gid=52'
+      on sut, 'puppet resource user puppet comment="Puppet" gid="52" uid="52" home="/var/lib/puppet" managehome=true'
 
-      user_info.keys.each do |user|
-        src_file = "#{user_info[user]}/.ssh/authorized_keys"
-        tgt_file = "/etc/ssh/local_keys/#{user}"
+      os_info = fact_on(sut, 'os')
 
-        on(sut, %{if [ -f "#{src_file}" ]; then cp -a -f "#{src_file}" "#{tgt_file}" && chmod 644 "#{tgt_file}"; fi}, :silent => true)
-      end
-    end
+      # Make sure we have a domain on our host
+      current_domain = fact_on(sut, 'domain').strip
+      hostname = fact_on(sut, 'hostname').strip
 
-    # SIMP uses structured facts, therefore stringify_facts must be disabled
-    unless ENV['BEAKER_stringify_facts'] == 'yes'
-      on sut, 'puppet config set stringify_facts false'
-    end
+      if current_domain.empty?
+        new_fqdn = hostname + '.beaker.test'
 
-    # Occasionally we run across something similar to BKR-561, so to ensure we
-    # at least have the host defaults:
-    #
-    # :hieradatadir is used as a canary here; it isn't the only missing key
-    unless sut.host_hash.key? :hieradatadir
-      configure_type_defaults_on(sut)
-    end
+        on(sut, "sed -i 's/#{hostname}.*/#{new_fqdn} #{hostname}/' /etc/hosts")
+        on(sut, "echo '#{new_fqdn}' > /etc/hostname", :accept_all_exit_codes => true)
+        on(sut, "hostname #{new_fqdn}", :accept_all_exit_codes => true)
 
-    if fact_on(sut, 'osfamily') == 'RedHat'
-      if fact_on(sut, 'operatingsystem') == 'RedHat'
-        RSpec.configure do |c|
-          c.before(:all) do
-            rhel_rhsm_subscribe(sut)
-          end
+        if sut.file_exist?('/etc/sysconfig/network')
+          on(sut, "sed -s '/HOSTNAME=/d' /etc/sysconfig/network")
+          on(sut, "echo 'HOSTNAME=#{new_fqdn}' >> /etc/sysconfig/network")
+        end
+      end
 
-          c.after(:all) do
-            rhel_rhsm_unsubscribe(sut)
+      if fact_on(sut, 'domain').strip.empty?
+        fail("Error: hosts must have an FQDN, got domain='#{current_domain}'")
+      end
+
+      # This may not exist in docker so just skip the whole thing
+      if sut.file_exist?('/etc/ssh')
+        # SIMP uses a central ssh key location so we prep that spot in case we
+        # flip to the SIMP SSH module.
+        on(sut, 'mkdir -p /etc/ssh/local_keys')
+        on(sut, 'chown -R root:root /etc/ssh/local_keys')
+        on(sut, 'chmod 755 /etc/ssh/local_keys')
+
+        user_info = on(sut, 'getent passwd').stdout.lines
+
+        # Hash of user => home_dir
+        # Exclude silly directories
+        #   * /
+        #   * /dev/*
+        #   * /s?bin
+        #   * /proc
+        user_info = Hash[
+          user_info.map do |u|
+            u.strip!
+            u = u.split(':')
+            u[5] =~ %r{^(/|/dev/.*|/s?bin/?.*|/proc/?.*)$} ? [nil] : [u[0], u[5]]
           end
+        ]
+
+        user_info.keys.each do |user|
+          src_file = "#{user_info[user]}/.ssh/authorized_keys"
+          tgt_file = "/etc/ssh/local_keys/#{user}"
+
+          on(sut, %{if [ -f "#{src_file}" ]; then cp -a -f "#{src_file}" "#{tgt_file}" && chmod 644 "#{tgt_file}"; fi}, :silent => true)
         end
       end
 
-      enable_yum_repos_on(sut)
+      # SIMP uses structured facts, therefore stringify_facts must be disabled
+      unless ENV['BEAKER_stringify_facts'] == 'yes'
+        on sut, 'puppet config set stringify_facts false'
+      end
 
-      # net-tools required for netstat utility being used by be_listening
-      if fact_on(sut, 'operatingsystemmajrelease') == '7'
-        pp = <<-EOS
-          package { 'net-tools': ensure => installed }
-        EOS
-        apply_manifest_on(sut, pp, :catch_failures => false)
+      # Occasionally we run across something similar to BKR-561, so to ensure we
+      # at least have the host defaults:
+      #
+      # :hieradatadir is used as a canary here; it isn't the only missing key
+      unless sut.host_hash.key? :hieradatadir
+        configure_type_defaults_on(sut)
       end
 
-      # Clean up YUM prior to starting our test runs.
-      on(sut, 'yum clean all')
+      if os_info['family'] == 'RedHat'
+        # OS-specific items
+        if os_info['name'] == 'RedHat'
+          RSpec.configure do |c|
+            c.before(:all) do
+              rhel_rhsm_subscribe(sut)
+            end
+
+            c.after(:all) do
+              rhel_rhsm_unsubscribe(sut)
+            end
+          end
+        end
+
+        if ['CentOS','RedHat','OracleLinux'].include?(os_info['name'])
+          enable_yum_repos_on(sut)
+          enable_epel_on(sut)
+
+          # net-tools required for netstat utility being used by be_listening
+          if os_info['release']['major'].to_i >= 7
+            pp = <<-EOS
+              package { 'net-tools': ensure => installed }
+            EOS
+            apply_manifest_on(sut, pp, :catch_failures => false)
+          end
+
+          unless sut[:hypervisor] == 'docker'
+            if (os_info['name'] == 'CentOS') && (os_info['release']['major'].to_i >= 8)
+              if os_info['release']['minor'].to_i == 3
+                update_package_from_centos_stream(sut, 'kernel')
+                sut.reboot
+              end
+            end
+          end
+
+          # Clean up YUM prior to starting our test runs.
+          on(sut, 'yum clean all')
+        end
+      end
     end
   end
 
@@ -561,85 +718,100 @@ module Simp::BeakerHelpers
   #
   # Must set BEAKER_RHSM_USER and BEAKER_RHSM_PASS environment variables or pass them in as
   # parameters
-  def rhel_rhsm_subscribe(sut, *opts)
+  def rhel_rhsm_subscribe(suts, *opts)
     require 'securerandom'
 
-    rhsm_opts = {
-      :username => ENV['BEAKER_RHSM_USER'],
-      :password => ENV['BEAKER_RHSM_PASS'],
-      :system_name => "#{sut}_beaker_#{Time.now.to_i}_#{SecureRandom.uuid}",
-      :repo_list => {
-        '7' => [
-          'rhel-7-server-extras-rpms',
-          'rhel-7-server-optional-rpms',
-          'rhel-7-server-rh-common-rpms',
-          'rhel-7-server-rpms',
-          'rhel-7-server-supplementary-rpms'
-        ],
-        '8' => [
-          'rhel-8-for-x86_64-baseos-rpms',
-          'rhel-8-for-x86_64-supplementary-rpms'
-        ]
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      rhsm_opts = {
+        :username => ENV['BEAKER_RHSM_USER'],
+        :password => ENV['BEAKER_RHSM_PASS'],
+        :system_name => "#{sut}_beaker_#{Time.now.to_i}_#{SecureRandom.uuid}",
+        :repo_list => {
+          '7' => [
+            'rhel-7-server-extras-rpms',
+            'rhel-7-server-optional-rpms',
+            'rhel-7-server-rh-common-rpms',
+            'rhel-7-server-rpms',
+            'rhel-7-server-supplementary-rpms'
+          ],
+          '8' => [
+            'rhel-8-for-x86_64-baseos-rpms',
+            'rhel-8-for-x86_64-supplementary-rpms'
+          ]
+        }
       }
-    }
 
-    if opts && opts.is_a?(Hash)
-      rhsm_opts.merge!(opts)
-    end
+      if opts && opts.is_a?(Hash)
+        rhsm_opts.merge!(opts)
+      end
 
-    os = fact_on(sut, 'operatingsystem').strip
-    os_release = fact_on(sut, 'operatingsystemmajrelease').strip
+      os = fact_on(sut, 'operatingsystem').strip
+      os_release = fact_on(sut, 'operatingsystemmajrelease').strip
 
-    if os == 'RedHat'
-      unless rhsm_opts[:username] && rhsm_opts[:password]
-        fail("You must set BEAKER_RHSM_USER and BEAKER_RHSM_PASS environment variables to register RHEL systems")
-      end
+      if os == 'RedHat'
+        unless rhsm_opts[:username] && rhsm_opts[:password]
+          fail("You must set BEAKER_RHSM_USER and BEAKER_RHSM_PASS environment variables to register RHEL systems")
+        end
 
-      sub_status = on(sut, 'subscription-manager status', :accept_all_exit_codes => true)
-      unless sub_status.exit_code == 0
-        logger.info("Registering #{sut} via subscription-manager")
-        on(sut, %{subscription-manager register --auto-attach --name='#{rhsm_opts[:system_name]}' --username='#{rhsm_opts[:username]}' --password='#{rhsm_opts[:password]}'}, :silent => true)
-      end
+        sub_status = on(sut, 'subscription-manager status', :accept_all_exit_codes => true)
+        unless sub_status.exit_code == 0
+          logger.info("Registering #{sut} via subscription-manager")
+          on(sut, %{subscription-manager register --auto-attach --name='#{rhsm_opts[:system_name]}' --username='#{rhsm_opts[:username]}' --password='#{rhsm_opts[:password]}'}, :silent => true)
+        end
 
-      if rhsm_opts[:repo_list][os_release]
-        rhel_repo_enable(sut, rhsm_opts[:repo_list][os_release])
-      else
-        logger.warn("simp-beaker-helpers:#{__method__} => Default repos for RHEL '#{os_release}' not found")
-      end
+        if rhsm_opts[:repo_list][os_release]
+          rhel_repo_enable(sut, rhsm_opts[:repo_list][os_release])
+        else
+          logger.warn("simp-beaker-helpers:#{__method__} => Default repos for RHEL '#{os_release}' not found")
+        end
 
-      # Ensure that all users can access the entitlements since we don't know
-      # who we'll be running jobs as (often not root)
-      on(sut, 'chmod -R ugo+rX /etc/pki/entitlement', :accept_all_exit_codes => true)
+        # Ensure that all users can access the entitlements since we don't know
+        # who we'll be running jobs as (often not root)
+        on(sut, 'chmod -R ugo+rX /etc/pki/entitlement', :accept_all_exit_codes => true)
+      end
     end
   end
 
-  def sosreport(sut, dest='sosreports')
-    on(sut, 'puppet resource package sos ensure=latest')
-    on(sut, 'sosreport --batch')
+  def sosreport(suts, dest='sosreports')
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      install_latest_package_on(sut, 'sos')
+      on(sut, 'sosreport --batch')
 
-    files = on(sut, 'ls /var/tmp/sosreport* /tmp/sosreport* 2>/dev/null', :accept_all_exit_codes => true).output.lines.map(&:strip)
+      files = on(sut, 'ls /var/tmp/sosreport* /tmp/sosreport* 2>/dev/null', :accept_all_exit_codes => true).output.lines.map(&:strip)
 
-    FileUtils.mkdir_p(dest)
+      FileUtils.mkdir_p(dest)
 
-    files.each do |file|
-      scp_from(sut, file, File.absolute_path(dest))
+      files.each do |file|
+        scp_from(sut, file, File.absolute_path(dest))
+      end
     end
   end
 
-  def rhel_repo_enable(sut, repos)
-    Array(repos).each do |repo|
-      on(sut, %{subscription-manager repos --enable #{repo}})
+  def rhel_repo_enable(suts, repos)
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      Array(repos).each do |repo|
+        on(sut, %{subscription-manager repos --enable #{repo}})
+      end
     end
   end
 
-  def rhel_repo_disable(sut, repos)
-    Array(repos).each do |repo|
-      on(sut, %{subscription-manager repos --disable #{repo}}, :accept_all_exit_codes => true)
+  def rhel_repo_disable(suts, repos)
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      Array(repos).each do |repo|
+        on(sut, %{subscription-manager repos --disable #{repo}}, :accept_all_exit_codes => true)
+      end
     end
   end
 
-  def rhel_rhsm_unsubscribe(sut)
-    on(sut, %{subscription-manager unregister}, :accept_all_exit_codes => true)
+  def rhel_rhsm_unsubscribe(suts)
+    parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
+    block_on(suts, :run_in_parallel => parallel) do |sut|
+      on(sut, %{subscription-manager unregister}, :accept_all_exit_codes => true)
+    end
   end
 
   # Apply known OS fixes we need to run Beaker on each SUT
@@ -713,6 +885,9 @@ module Simp::BeakerHelpers
 
       host_entry = { fqdn => [] }
 
+      # Add the short name because containers can't change the hostname
+      host_entry[fqdn] << host.name if (host[:hypervisor] == 'docker')
+
       # Ensure that all interfaces are active prior to collecting data
       activate_interfaces(host) unless ENV['BEAKER_no_fix_interfaces']
 
@@ -726,7 +901,7 @@ module Simp::BeakerHelpers
         host_entry[fqdn] << ipaddress.strip
 
         unless host_entry[fqdn].empty?
-          suts_network_info[fqdn] = host_entry[fqdn]
+          suts_network_info[fqdn] = host_entry[fqdn].sort.uniq
         end
       end
     end
@@ -755,6 +930,7 @@ module Simp::BeakerHelpers
       end
 
       copy_to(ca_sut, pki_hosts_file, host_dir)
+
       # generate certs
       on(ca_sut, "cd #{host_dir}; cat #{host_dir}/pki.hosts | xargs bash make.sh")
     end
@@ -789,8 +965,8 @@ module Simp::BeakerHelpers
       sut.mkdir_p("#{sut_pki_dir}/public")
       sut.mkdir_p("#{sut_pki_dir}/private")
       sut.mkdir_p("#{sut_pki_dir}/cacerts")
-      copy_to(sut, "#{local_host_pki_tree}/#{fqdn}.pem",   "#{sut_pki_dir}/private/")
-      copy_to(sut, "#{local_host_pki_tree}/#{fqdn}.pub",   "#{sut_pki_dir}/public/")
+      copy_to(sut, "#{local_host_pki_tree}/#{fqdn}.pem", "#{sut_pki_dir}/private/")
+      copy_to(sut, "#{local_host_pki_tree}/#{fqdn}.pub", "#{sut_pki_dir}/public/")
 
       copy_to(sut, local_cacert, "#{sut_pki_dir}/cacerts/simp_auto_ca.pem")
 
@@ -800,18 +976,19 @@ module Simp::BeakerHelpers
       # Need to hash all of the CA certificates so that apps can use them
       # properly! This must happen on the host itself since it needs to match
       # the native hashing algorithms.
-      hash_cmd = <<-EOM.strip
-cd #{sut_pki_dir}/cacerts; \
-for x in *; do \
-  if [ ! -h "$x" ]; then \
-    `openssl x509 -in $x >/dev/null 2>&1`; \
-    if [ $? -eq 0 ]; then \
-      hash=`openssl x509 -in $x -hash | head -1`; \
-      ln -sf $x $hash.0; \
-    fi; \
-   fi; \
-done
-      EOM
+      hash_cmd = <<~EOM.strip
+        PATH=/opt/puppetlabs/puppet/bin:$PATH; \
+        cd #{sut_pki_dir}/cacerts; \
+        for x in *; do \
+          if [ ! -h "$x" ]; then \
+            `openssl x509 -in $x >/dev/null 2>&1`; \
+            if [ $? -eq 0 ]; then \
+              hash=`openssl x509 -in $x -hash | head -1`; \
+              ln -sf $x $hash.0; \
+            fi; \
+           fi; \
+        done
+        EOM
 
       on(sut, hash_cmd)
   end
@@ -1226,13 +1403,12 @@ done
   def install_simp_repos(sut, disable = [])
     # NOTE: Do *NOT* use puppet in this method since it may not be available yet
 
-    if on(sut, 'rpm -q yum-utils', :accept_all_exit_codes => true).exit_code != 0
-      on(sut, 'yum -y install yum-utils')
-    end
-
-    if on(sut, 'rpm -q simp-release-community', :accept_all_exit_codes => true).exit_code != 0
-      on(sut, 'yum -y install "https://download.simp-project.com/simp-release-community.rpm"')
-    end
+    install_latest_package_on(sut, 'yum-utils')
+    install_latest_package_on(
+      sut,
+      'simp-release-community',
+      "https://download.simp-project.com/simp-release-community.rpm",
+    )
 
     to_disable = disable.dup
 

data/lib/simp/beaker_helpers/constants.rb

@@ -17,7 +17,11 @@ module Simp::BeakerHelpers
     require 'open-uri'
 
     begin
-      ONLINE = true if open('http://google.com')
+      if URI.respond_to?(:open)
+        ONLINE = true if URI.open('http://google.com')
+      else
+        ONLINE = true if open('http://google.com')
+      end
     rescue
       ONLINE = false
     end

data/lib/simp/beaker_helpers/version.rb

@@ -1,5 +1,5 @@
 module Simp; end
 
 module Simp::BeakerHelpers
-  VERSION = '1.19.3'
+  VERSION = '1.21.3'
 end

data/spec/acceptance/nodesets/default.yml

@@ -6,21 +6,27 @@
   end
 -%>
 HOSTS:
-  server-el7:
+  el7:
     roles:
-      - server
-      - master
-      - default
       - el7
+      - master
     platform: el-7-x86_64
     box: centos/7
     hypervisor: <%= hypervisor %>
 
-  server-el8:
+  el8:
+    roles:
+      - el8
+    platform: el-8-x86_64
+    box: centos/8
+    hypervisor: <%= hypervisor %>
+
+  el8-0:
     roles:
       - el8
     platform: el-8-x86_64
     box: centos/8
+    box_version: "1905.1"
     hypervisor: <%= hypervisor %>
 
 CONFIG:
@@ -30,3 +36,14 @@ CONFIG:
 <% if ENV['BEAKER_PUPPET_COLLECTION'] -%>
   puppet_collection: <%= ENV['BEAKER_PUPPET_COLLECTION'] %>
 <% end -%>
+  ssh:
+    keepalive: true
+    keepalive_interval: 10
+    host_key:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].join("\n#{' '*6}- ") %>
+    kex:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].join("\n#{' '*6}- ") %>
+    encryption:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:encryption].join("\n#{' '*6}- ") %>
+    hmac:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:hmac].join("\n#{' '*6}- ") %>

data/spec/acceptance/nodesets/docker.yml

@@ -0,0 +1,36 @@
+HOSTS:
+  el7:
+    roles:
+      - el7
+      - master
+    platform: el-7-x86_64
+    hypervisor: docker
+    image: simpproject/simp_build_centos7
+    docker_cmd: '/usr/sbin/sshd -D -E /var/log/sshd.log'
+
+  el8:
+    roles:
+      - el8
+    platform: el-8-x86_64
+    hypervisor: docker
+    image: simpproject/simp_build_centos8
+    docker_cmd: '["/sbin/init"]'
+
+CONFIG:
+  docker_preserve_image: true
+  log_level: verbose
+  type: aio
+<% if ENV['BEAKER_PUPPET_COLLECTION'] -%>
+  puppet_collection: <%= ENV['BEAKER_PUPPET_COLLECTION'] %>
+<% end -%>
+  ssh:
+    keepalive: true
+    keepalive_interval: 10
+    host_key:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:host_key].join("\n#{' '*6}- ") %>
+    kex:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:kex].join("\n#{' '*6}- ") %>
+    encryption:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:encryption].join("\n#{' '*6}- ") %>
+    hmac:
+      - <%= Net::SSH::Transport::Algorithms::ALGORITHMS[:hmac].join("\n#{' '*6}- ") %>

data/spec/acceptance/nodesets/ubuntu.yml

@@ -0,0 +1,20 @@
+<%
+  if ENV['BEAKER_HYPERVISOR']
+    hypervisor = ENV['BEAKER_HYPERVISOR']
+  else
+    hypervisor = 'vagrant'
+  end
+-%>
+HOSTS:
+  focal:
+    platform: ubuntu-20.04-x86_64
+    box: ubuntu/focal64
+    hypervisor: <%= hypervisor %>
+
+CONFIG:
+  log_level: verbose
+  type: aio
+  vagrant_memsize: 256
+<% if ENV['BEAKER_PUPPET_COLLECTION'] -%>
+  puppet_collection: <%= ENV['BEAKER_PUPPET_COLLECTION'] %>
+<% end -%>

data/spec/acceptance/suites/default/check_puppet_version_spec.rb

@@ -5,7 +5,7 @@ hosts.each do |host|
     context "on #{host}" do
       puppet_collection = host.options[:puppet_collection]
 
-      client_puppet_version = on(host, 'puppet --version').output.strip
+      client_puppet_version = on(host, 'puppet --version').stdout.strip
 
       if puppet_collection =~ /puppet(\d+)/
         puppet_collection_version = $1

data/spec/acceptance/suites/default/fixture_modules_spec.rb

@@ -29,4 +29,18 @@ context 'after copy_fixture_modules_to( hosts )' do
       expect(pfact_on(master, 'os.release.foo')).to eq ''
     end
   end
+
+  describe "pfact_on(master,'fips_enabled')" do
+    expected = (ENV['BEAKER_fips'] == 'yes')
+
+    it 'should return false' do
+      expect(pfact_on(master, 'fips_enabled')).to eq expected
+    end
+  end
+
+  describe "pfact_on returns a hash" do
+    it 'should return a Hash' do
+      expect(pfact_on(master, 'os')).to be_a(Hash)
+    end
+  end
 end

data/spec/acceptance/suites/default/nodesets

@@ -0,0 +1 @@
+spec/acceptance/suites/default/../../nodesets

data/spec/acceptance/suites/fips_from_fixtures/nodesets

@@ -0,0 +1 @@
+spec/acceptance/suites/fips_from_fixtures/../../nodesets

data/spec/acceptance/suites/snapshot/nodesets

@@ -0,0 +1 @@
+spec/acceptance/suites/snapshot/../../nodesets

data/spec/lib/simp/beaker_helpers_spec.rb

@@ -103,8 +103,21 @@ describe 'Simp::BeakerHelpers' do
     end
 
     it 'uses defaults when no environment variables are set' do
-      expect( @helper.get_puppet_install_info[:puppet_install_version] ).to match(/^6\./)
-      expect( @helper.get_puppet_install_info[:puppet_collection] ).to eq('puppet6')
+
+      # Prevent namespace pollution
+      pipe_out,pipe_in = IO.pipe
+      fork do
+        pipe_out.close
+        require 'puppet'
+        pipe_in.write(Puppet.version)
+      end
+      pipe_in.close
+
+      expected_version = pipe_out.gets
+      expected_major_version = expected_version.split('.').first
+
+      expect( @helper.get_puppet_install_info[:puppet_install_version] ).to match(expected_version)
+      expect( @helper.get_puppet_install_info[:puppet_collection] ).to eq("puppet#{expected_major_version}")
       expect( @helper.get_puppet_install_info[:puppet_install_type] ).to eq('agent')
     end
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: simp-beaker-helpers
 version: !ruby/object:Gem::Version
-  version: 1.19.3
+  version: 1.21.3
 platform: ruby
 authors:
 - Chris Tessmer
 - Trevor Vaughan
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-04 00:00:00.000000000 Z
+date: 2021-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaker
@@ -188,15 +188,19 @@ files:
 - lib/simp/rake/beaker.rb
 - simp-beaker-helpers.gemspec
 - spec/acceptance/nodesets/default.yml
+- spec/acceptance/nodesets/docker.yml
+- spec/acceptance/nodesets/ubuntu.yml
 - spec/acceptance/suites/default/check_puppet_version_spec.rb
 - spec/acceptance/suites/default/enable_fips_spec.rb
 - spec/acceptance/suites/default/fixture_modules_spec.rb
 - spec/acceptance/suites/default/install_simp_deps_repo_spec.rb
+- spec/acceptance/suites/default/nodesets
 - spec/acceptance/suites/default/pki_tests_spec.rb
 - spec/acceptance/suites/default/set_hieradata_on_spec.rb
 - spec/acceptance/suites/default/write_hieradata_to_spec.rb
 - spec/acceptance/suites/fips_from_fixtures/00_default_spec.rb
 - spec/acceptance/suites/fips_from_fixtures/metadata.yml
+- spec/acceptance/suites/fips_from_fixtures/nodesets
 - spec/acceptance/suites/offline/00_default_spec.rb
 - spec/acceptance/suites/offline/README
 - spec/acceptance/suites/offline/nodesets/default.yml
@@ -204,6 +208,7 @@ files:
 - spec/acceptance/suites/puppet_collections/metadata.yml
 - spec/acceptance/suites/snapshot/00_snapshot_test_spec.rb
 - spec/acceptance/suites/snapshot/10_general_usage_spec.rb
+- spec/acceptance/suites/snapshot/nodesets
 - spec/acceptance/suites/windows/00_default_spec.rb
 - spec/acceptance/suites/windows/metadata.yml
 - spec/acceptance/suites/windows/nodesets/default.yml
@@ -217,7 +222,7 @@ licenses:
 - Apache-2.0
 metadata:
   issue_tracker: https://simp-project.atlassian.net
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -232,21 +237,26 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.10
+signing_key: 
 specification_version: 4
 summary: beaker helper methods for SIMP
 test_files:
 - spec/acceptance/nodesets/default.yml
+- spec/acceptance/nodesets/docker.yml
+- spec/acceptance/nodesets/ubuntu.yml
 - spec/acceptance/suites/default/check_puppet_version_spec.rb
 - spec/acceptance/suites/default/enable_fips_spec.rb
 - spec/acceptance/suites/default/fixture_modules_spec.rb
 - spec/acceptance/suites/default/install_simp_deps_repo_spec.rb
+- spec/acceptance/suites/default/nodesets
 - spec/acceptance/suites/default/pki_tests_spec.rb
 - spec/acceptance/suites/default/set_hieradata_on_spec.rb
 - spec/acceptance/suites/default/write_hieradata_to_spec.rb
 - spec/acceptance/suites/fips_from_fixtures/00_default_spec.rb
 - spec/acceptance/suites/fips_from_fixtures/metadata.yml
+- spec/acceptance/suites/fips_from_fixtures/nodesets
 - spec/acceptance/suites/offline/00_default_spec.rb
 - spec/acceptance/suites/offline/README
 - spec/acceptance/suites/offline/nodesets/default.yml
@@ -254,6 +264,7 @@ test_files:
 - spec/acceptance/suites/puppet_collections/metadata.yml
 - spec/acceptance/suites/snapshot/00_snapshot_test_spec.rb
 - spec/acceptance/suites/snapshot/10_general_usage_spec.rb
+- spec/acceptance/suites/snapshot/nodesets
 - spec/acceptance/suites/windows/00_default_spec.rb
 - spec/acceptance/suites/windows/metadata.yml
 - spec/acceptance/suites/windows/nodesets/default.yml