beaker-puppet 0.10.0 → 0.11.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZTBjMzQ0ZjEzZTNkMDI3ODE1NGIwYWU2YzllZDA3NjFhNmU1YmM3ZQ==
-  data.tar.gz: !binary |-
-    MzY0ZWZiNTEzNDk0M2YwZTdlODcxNWQyM2E3MWRiYWE5YWNkNzE4YQ==
+SHA1:
+  metadata.gz: 02ec1da775402ec80c54b00f614d7b18f90c0a27
+  data.tar.gz: eb6d32c7c507cd87055c287d99e44b8a7b69bedc
 SHA512:
-  metadata.gz: !binary |-
-    YmRiOTAwYjU1NDU5ODBhZjdiMmFmNTIwODcxNGRhNDhhZWYzODA1YWJlYWZi
-    NTFlYzU5NjMwMTY3ZWZiODc4YjUxMjE0MGVkMmE0YzdkNTE2Njg0NDVhMmZm
-    MjEyMTliNTRhNGNmNmVhZGMxNjYzZGFhNWZhYjExOTk2MmNmZTU=
-  data.tar.gz: !binary |-
-    ODczODMyZjcwZWNhNTgwMzg5YTljYTBiNTM4MWIzYmQ4MTkwNmVkYTE5ZGQ2
-    YWE0NGFhNDNiY2EwMGYwNWYyZDU0YmRhNmM2NzJjODYzNzc4MWM2OWZkMjg1
-    YjBlNDAxM2I0YWIwNzVhNjI1ZjNiN2MzMTBiNDFkODY2N2EyZTU=
+  metadata.gz: 8ae33a0aba67b868c412302688b813524b84ee72f512ae8425c884c276f7608ddadda199847b4444b7f8edc886186cd16f594e3778f11790bd1bb122fc74f758
+  data.tar.gz: 424bdbafc891afb87f12f95456f4c4bf9d15bb2c8233173d829baa6142f8451b8349f1d3bc02b8a700d1fc95c0b50f7eabd10ee7ae2a0a3d2e9d54f95c73655b

data/lib/beaker-puppet.rb

@@ -3,6 +3,8 @@ require 'in_parallel'
 require 'beaker-puppet/version'
 require 'beaker-puppet/wrappers'
 
+require 'beaker-puppet/helpers/rake_helpers'
+
 [ 'aio', 'foss' ].each do |lib|
   require "beaker-puppet/install_utils/#{lib}_defaults"
 end

data/lib/beaker-puppet/helpers/rake_helpers.rb

@@ -0,0 +1,21 @@
+module Beaker
+  module DSL
+    module Helpers
+      # Methods that help you interact with rake during ci setup
+      module RakeHelpers
+        class << self
+          def load_tasks(beaker_root = File.expand_path("#{__dir__}/../../.."))
+            task_dir = File.join(beaker_root, 'tasks')
+            tasks = [
+              'ci.rake'
+            ]
+
+            tasks.each do |task|
+              load File.join(task_dir, task)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/beaker-puppet/install_utils/foss_utils.rb

@@ -1353,6 +1353,83 @@ module Beaker
 
           end
         end
+
+        # Installs packages on the hosts.
+        #
+        # @param hosts [Array<Host>] Array of hosts to install packages to.
+        # @param package_hash [Hash{Symbol=>Array<String,Array<String,String>>}]
+        #   Keys should be a symbol for a platform in PLATFORM_PATTERNS.  Values
+        #   should be an array of package names to install, or of two element
+        #   arrays where a[0] is the command we expect to find on the platform
+        #   and a[1] is the package name (when they are different).
+        # @param options [Hash{Symbol=>Boolean}]
+        # @option options [Boolean] :check_if_exists First check to see if
+        #   command is present before installing package.  (Default false)
+        # @return true
+        def install_packages_on(hosts, package_hash, options = {})
+          platform_patterns = {
+            :redhat        => /fedora|el-|centos/,
+            :debian        => /debian|ubuntu|cumulus/,
+            :debian_ruby18 => /debian|ubuntu-lucid|ubuntu-precise/,
+            :solaris_10    => /solaris-10/,
+            :solaris_11    => /solaris-11/,
+            :windows       => /windows/,
+            :eos           => /^eos-/,
+          }.freeze
+
+          check_if_exists = options[:check_if_exists]
+          Array(hosts).each do |host|
+            package_hash.each do |platform_key,package_list|
+              if pattern = platform_patterns[platform_key]
+                if pattern.match(host['platform'])
+                  package_list.each do |cmd_pkg|
+                    if cmd_pkg.kind_of?(Array)
+                      command, package = cmd_pkg
+                    else
+                      command = package = cmd_pkg
+                    end
+                    if !check_if_exists || !host.check_for_package(command)
+                      host.logger.notify("Installing #{package}")
+                      additional_switches = '--allow-unauthenticated' if platform_key == :debian
+                      host.install_package(package, additional_switches)
+                    end
+                  end
+                end
+              else
+                raise("Unknown platform '#{platform_key}' in package_hash")
+              end
+            end
+          end
+          return true
+        end
+
+        def ruby_command(host)
+          "env PATH=\"#{host['privatebindir']}:${PATH}\" ruby"
+        end
+
+        def gem_command(host, type = 'aio')
+          if type == 'aio'
+            if host['platform'] =~ /windows/
+              "env PATH=\"#{host['privatebindir']}:${PATH}\" cmd /c gem"
+            else
+              "env PATH=\"#{host['privatebindir']}:${PATH}\" gem"
+            end
+          else
+            on(host, 'which gem').stdout.chomp
+          end
+        end
+
+        # Configures gem sources on hosts to use a mirror, if specified
+        # This is a duplicate of the Gemfile logic.
+        def configure_gem_mirror(hosts)
+          gem_source = ENV['GEM_SOURCE'] || 'https://rubygems.org'
+
+          Array(hosts).each do |host|
+            gem = gem_command(host)
+            on host, "#{gem} source --clear-all"
+            on host, "#{gem} source --add #{gem_source}"
+          end
+        end
       end
     end
   end

data/lib/beaker-puppet/version.rb

@@ -1,3 +1,3 @@
 module BeakerPuppet
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'
 end

data/setup/aio/010_Install.rb

@@ -0,0 +1,50 @@
+extend BeakerPuppet::Install::Puppet5
+extend Beaker::DSL::InstallUtils::FOSSUtils
+
+test_name "Install Packages"
+
+dev_builds_url  = ENV['DEV_BUILDS_URL'] || 'http://builds.delivery.puppetlabs.net'
+nightly_builds_url = ENV['NIGHTLY_BUILDS_URL'] || "http://ravi.puppetlabs.com"
+
+step "Install puppet-agent..." do
+  sha = ENV['SHA']
+  install_from_build_data_url('puppet-agent', "#{dev_builds_url}/puppet-agent/#{sha}/artifacts/#{sha}.yaml", hosts)
+end
+
+step "Install puppetserver..." do
+  if ENV['SERVER_VERSION'].nil? || ENV['SERVER_VERSION'] == 'latest'
+    install_puppetlabs_dev_repo(master, 'puppetserver', 'latest', nil, :dev_builds_url => nightly_builds_url)
+    master.install_package('puppetserver')
+  else
+    server_version = ENV['SERVER_VERSION']
+    install_from_build_data_url('puppetserver', "#{dev_builds_url}/puppetserver/#{server_version}/artifacts/#{server_version}.yaml", master)
+  end
+end
+
+# make sure install is sane, beaker has already added puppet and ruby
+# to PATH in ~/.ssh/environment
+agents.each do |agent|
+  on agent, puppet('--version')
+  ruby = ruby_command(agent)
+  on agent, "#{ruby} --version"
+end
+
+# Get a rough estimate of clock skew among hosts
+times = []
+hosts.each do |host|
+  ruby = ruby_command(host)
+  on(host, "#{ruby} -e 'puts Time.now.strftime(\"%Y-%m-%d %T.%L %z\")'") do |result|
+    times << result.stdout.chomp
+  end
+end
+times.map! do |time|
+  (Time.strptime(time, "%Y-%m-%d %T.%L %z").to_f * 1000.0).to_i
+end
+diff = times.max - times.min
+if diff < 60000
+  logger.info "Host times vary #{diff} ms"
+else
+  logger.warn "Host times vary #{diff} ms, tests may fail"
+end
+
+configure_gem_mirror(hosts)

data/setup/aio/020_InstallCumulusModules.rb

@@ -0,0 +1,12 @@
+platforms = hosts.map{|val| val[:platform]}
+skip_test "No cumulus hosts present" unless platforms.any? { |val| /cumulus/ =~ val }
+confine :to, {}, hosts.select { |host| host[:roles].include?('master') }
+
+step 'install Cumulus Modules on masters' do
+  hosts.each do |node|
+    on(node, puppet('module','install','cumuluslinux-cumulus_license'))
+    on(node, puppet('module','install','cumuluslinux-cumulus_interfaces'))
+    on(node, puppet('module','install','cumuluslinux-cumulus_interface_policy'))
+    on(node, puppet('module','install','cumuluslinux-cumulus_ports'))
+  end
+end

data/setup/aio/021_InstallAristaModule.rb

@@ -0,0 +1,20 @@
+platforms = hosts.map{|val| val[:platform]}
+skip_test "No arista hosts present" unless platforms.any? { |val| /^eos-/ =~ val }
+test_name 'Arista Switch Pre-suite' do
+  masters = select_hosts({:roles => ['master', 'compile_master']})
+  switchs = select_hosts({:platform => ['eos-4-i386']})
+
+  step 'install Arista Module on masters' do
+    masters.each do |node|
+      on(node, puppet('module','install','aristanetworks-netdev_stdlib_eos'))
+    end
+  end
+
+  step 'add puppet user to switch' do
+    switchs.each do |switch|
+      on(switch, "useradd -U puppet")
+      on(switch, "/opt/puppetlabs/bin/puppet config --confdir /etc/puppetlabs/puppet set user root")
+      on(switch, "/opt/puppetlabs/bin/puppet config --confdir /etc/puppetlabs/puppet set group root")
+    end
+  end
+end

data/setup/aio/045_EnsureMasterStarted.rb

@@ -0,0 +1 @@
+on(master, puppet('resource', 'service', master['puppetservice'], "ensure=running"))

data/setup/common/000-delete-puppet-when-none.rb

@@ -0,0 +1,11 @@
+test_name "Expunge puppet bits if hypervisor is none"
+
+# Ensure that the any previous installations of puppet
+# are removed from the host if it is not managed by a
+# provisioning hypervisor.
+
+hosts.each do |host|
+  if host[:hypervisor] == "none"
+    remove_puppet_on(host)
+  end
+end

data/setup/common/025_StopFirewall.rb

@@ -0,0 +1,16 @@
+test_name "Stop firewall" do
+  hosts.each do |host|
+    case host['platform']
+    when /debian/
+      on host, 'iptables -F'
+    when /fedora|el-7/
+      on host, puppet('resource', 'service', 'firewalld', 'ensure=stopped')
+    when /el-|centos/
+      on host, puppet('resource', 'service', 'iptables', 'ensure=stopped')
+    when /ubuntu/
+      on host, puppet('resource', 'service', 'ufw', 'ensure=stopped')
+    else
+      logger.notify("Not sure how to clear firewall on #{host['platform']}")
+    end
+  end
+end

data/setup/common/030_StopSssd.rb

@@ -0,0 +1,9 @@
+test_name "Stop sssd" do
+  # The sssd service causes local users/groups to be cached,
+  # which can cause unexpected results when tests are trying
+  # to restore state. We ensure that it is not running to
+  # prevent such caching from occurring.
+  hosts.each do |host|
+    on(host, puppet('resource', 'service', 'sssd', 'ensure=stopped'), :accept_all_exit_codes => true)
+  end
+end

data/setup/common/040_ValidateSignCert.rb

@@ -0,0 +1,46 @@
+test_name "Validate Sign Cert" do
+  hostname = on(master, 'facter hostname').stdout.strip
+  fqdn = on(master, 'facter fqdn').stdout.strip
+
+  if master.use_service_scripts?
+    step "Ensure puppet is stopped"
+    # Passenger, in particular, must be shutdown for the cert setup steps to work,
+    # but any running puppet master will interfere with webrick starting up and
+    # potentially ignore the puppet.conf changes.
+    on(master, puppet('resource', 'service', master['puppetservice'], "ensure=stopped"))
+  end
+
+  step "Clear SSL on all hosts"
+  hosts.each do |host|
+    ssldir = on(host, puppet('agent --configprint ssldir')).stdout.chomp
+    on(host, "rm -rf '#{ssldir}'")
+  end
+
+  step "Master: Start Puppet Master" do
+    master_opts = {
+      :main => {
+        :dns_alt_names => "puppet,#{hostname},#{fqdn}",
+      },
+      :__service_args__ => {
+        # apache2 service scripts can't restart if we've removed the ssl dir
+        :bypass_service_script => true,
+      },
+    }
+    with_puppet_running_on(master, master_opts) do
+
+      hosts.each do |host|
+        next if host['roles'].include? 'master'
+
+        step "Agents: Run agent --test first time to gen CSR"
+        on host, puppet("agent --test --server #{master}"), :acceptable_exit_codes => [1]
+      end
+
+      # Sign all waiting certs
+      step "Master: sign all certs"
+      on master, puppet("cert --sign --all"), :acceptable_exit_codes => [0,24]
+
+      step "Agents: Run agent --test second time to obtain signed cert"
+      on agents, puppet("agent --test --server #{master}"), :acceptable_exit_codes => [0,2]
+    end
+  end
+end

data/setup/gem/010_GemInstall.rb

@@ -0,0 +1,40 @@
+extend Beaker::DSL::InstallUtils::FOSSUtils
+
+test_name "Install puppet gem"
+
+agents.each do |agent|
+  sha = ENV['SHA']
+  base_url = "http://builds.delivery.puppetlabs.net/puppet/#{sha}/artifacts"
+
+  ruby_command = ruby_command(agent)
+  gem_command = gem_command(agent)
+
+  # retrieve the build data, since the gem version is based on the short git
+  # describe, not the full git SHA
+  on(agent, "curl -s -o build_data.yaml #{base_url}/#{sha}.yaml")
+  gem_version = on(agent, "#{ruby_command} -ryaml -e 'puts YAML.load_file(\"build_data.yaml\")[:gemversion]'").stdout.chomp
+
+  if agent['platform'] =~ /windows/
+    # wipe existing gems first
+    default_dir = on(agent, "#{ruby_command} -rrbconfig -e 'puts Gem.default_dir'").stdout.chomp
+    on(agent, "rm -rf '#{default_dir}'")
+
+    arch = agent[:ruby_arch] || 'x86'
+    gem_arch = arch == 'x64' ? 'x64-mingw32' : 'x86-mingw32'
+    url = "#{base_url}/puppet-#{gem_version}-#{gem_arch}.gem"
+  else
+    url = "#{base_url}/puppet-#{gem_version}.gem"
+  end
+
+  step "Download puppet gem from #{url}"
+  on(agent, "curl -s -o puppet.gem #{url}")
+
+  step "Install puppet.gem"
+  on(agent, "#{gem_command} install puppet.gem")
+
+  step "Verify it's sane"
+  on(agent, puppet('--version'))
+  on(agent, puppet('apply', "-e \"notify { 'hello': }\"")) do |result|
+    assert_match(/defined 'message' as 'hello'/, result.stdout)
+  end
+end

data/setup/git/000_EnvSetup.rb

@@ -0,0 +1,144 @@
+test_name "Setup environment"
+
+step "Ensure Git and Ruby"
+
+extend Beaker::DSL::InstallUtils
+
+PACKAGES = {
+  :redhat => [
+    'git',
+    'ruby',
+    'rubygem-json',       # invalid on RHEL6
+    'rubygem-io-console', # required for Fedora25 to bundle install
+    'rubygem-rdoc'        # required for Fedora25 to install gems
+  ],
+  :debian => [
+    ['git', 'git-core'],
+    'ruby',
+  ],
+  :debian_ruby18 => [
+    'libjson-ruby',
+  ],
+  :solaris_11 => [
+    ['git', 'developer/versioning/git'],
+  ],
+  :solaris_10 => [
+    'coreutils',
+    'curl', # update curl to fix "CURLOPT_SSL_VERIFYHOST no longer supports 1 as value!" issue
+    'git',
+    'ruby19',
+    'ruby19_dev',
+    'gcc4core',
+  ],
+  :windows => [
+    'git',
+    # there isn't a need for json on windows because it is bundled in ruby 1.9
+  ],
+}
+
+# override incorrect FOSS (git) defaults from Beaker with AIO applicable ones
+#
+# Remove after PUP-4867 breaks distmoduledir and sitemoduledir into individual
+# settings from modulepath and Beaker can properly introspect these settings
+hosts.each do |host|
+  platform = host['platform'] =~ /windows/ ? 'windows' : 'unix'
+
+  host['puppetbindir'] = '/usr/bin' if platform == 'windows'
+
+  # Beakers add_aio_defaults_on helper is not appropriate here as it
+  # also alters puppetbindir / privatebindir to use package installed
+  # paths rather than git installed paths
+  host['distmoduledir'] = AIO_DEFAULTS[platform]['distmoduledir']
+  host['sitemoduledir'] = AIO_DEFAULTS[platform]['sitemoduledir']
+end
+
+hosts.each do |host|
+  case host['platform']
+  when  /solaris-10/
+    on host, 'mkdir -p /var/lib'
+    on host, 'ln -sf /opt/csw/bin/pkgutil /usr/bin/pkgutil'
+    on host, 'ln -sf /opt/csw/bin/gem19 /usr/bin/gem'
+    on host, 'ln -sf /opt/csw/bin/git /usr/bin/git'
+    on host, 'ln -sf /opt/csw/bin/ruby19 /usr/bin/ruby'
+    on host, 'ln -sf /opt/csw/bin/gstat /usr/bin/stat'
+    on host, 'ln -sf /opt/csw/bin/greadlink /usr/bin/readlink'
+  when /solaris-11/
+    step "#{host} jump through hoops to install ruby19; switch back to runtime/ruby-19 after template upgrade to sol11.2"
+    create_remote_file host, "/root/shutupsolaris", <<END
+mail=
+# Overwrite already installed instances
+instance=overwrite
+# Do not bother checking for partially installed packages
+partial=nocheck
+# Do not bother checking the runlevel
+runlevel=nocheck
+# Do not bother checking package dependencies (We take care of this)
+idepend=nocheck
+rdepend=nocheck
+# DO check for available free space and abort if there isn't enough
+space=quit
+# Do not check for setuid files.
+setuid=nocheck
+# Do not check if files conflict with other packages
+conflict=nocheck
+# We have no action scripts.  Do not check for them.
+action=nocheck
+# Install to the default base directory.
+basedir=default
+END
+    on host, 'pkgadd -a /root/shutupsolaris -d http://get.opencsw.org/now all'
+    on host, '/opt/csw/bin/pkgutil -U all'
+    on host, '/opt/csw/bin/pkgutil -i -y ruby19_dev'
+    on host, '/opt/csw/bin/pkgutil -i -y ruby19'
+    on host, 'ln -sf /opt/csw/bin/gem19 /usr/bin/gem'
+    on host, 'ln -sf /opt/csw/bin/ruby19 /usr/bin/ruby'
+  end
+end
+
+install_packages_on(hosts, PACKAGES, :check_if_exists => true)
+
+hosts.each do |host|
+  case host['platform']
+  when /windows/
+    arch = host[:ruby_arch] || 'x86'
+    step "#{host} Selected architecture #{arch}"
+
+    revision = if arch == 'x64'
+                 '2.1.x-x64'
+               else
+                 '2.1.x-x86'
+               end
+
+    step "#{host} Install ruby from git using revision #{revision}"
+    # TODO remove this step once we are installing puppet from msi packages
+    win_path = on(host, 'cygpath -m /opt/puppet-git-repos').stdout.chomp
+    install_from_git_on(host, win_path,
+                     :name => 'puppet-win32-ruby',
+                     :path => build_git_url('puppet-win32-ruby'),
+                     :rev  => revision)
+    on host, 'cd /opt/puppet-git-repos/puppet-win32-ruby; cp -r ruby/* /'
+    on host, 'cd /lib; icacls ruby /grant "Everyone:(OI)(CI)(RX)"'
+    on host, 'cd /lib; icacls ruby /reset /T'
+    on host, 'cd /; icacls bin /grant "Everyone:(OI)(CI)(RX)"'
+    on host, 'cd /; icacls bin /reset /T'
+    on host, 'ruby --version'
+    on host, 'cmd /c gem list'
+  end
+end
+
+# Only configure gem mirror after Ruby has been installed, but before any gems are installed.
+configure_gem_mirror(hosts)
+
+hosts.each do |host|
+  case host['platform']
+  when /solaris/
+    step "#{host} Install json from rubygems"
+    on host, 'gem install json_pure --no-ri --no-rdoc --version 1.8.3' # json_pure 2.0 requires ruby 2
+    on host, 'gem install bundler --no-ri --no-rdoc'
+    on host, "ln -sf /opt/csw/bin/bundle #{host['puppetbindir']}/bundle"
+  when /windows/
+    on host, 'cmd /c gem install bundler --no-ri --no-rdoc'
+  else
+    on host, 'gem install bundler --no-ri --no-rdoc'
+  end
+end