beaker 1.16.0 → 1.17.0

data/lib/beaker/dsl/install_utils.rb

@@ -172,6 +172,17 @@ module Beaker
         end
       end
 
+      #Create the Higgs install command string based upon the host and options settings.  Installation command will be run as a 
+      #background process.  The output of the command will be stored in the provided host['higgs_file'].
+      # @param [Host] host The host that Higgs is to be installed on
+      #                    The host object must have the 'working_dir', 'dist' and 'pe_installer' field set correctly.
+      # @api private
+      def higgs_installer_cmd host
+
+        "cd #{host['working_dir']}/#{host['dist']} ; nohup ./#{host['pe_installer']} <<<Y > #{host['higgs_file']} 2>&1 &"
+
+      end
+
       #Determine is a given URL is accessible
       #@param [String] link The URL to examine
       #@return [Boolean] true if the URL has a '200' HTTP response code, false otherwise
@@ -420,8 +431,7 @@ module Beaker
             version = host['pe_ver'] || opts[:pe_ver]
             host['dist'] = "puppet-enterprise-#{version}-#{host['platform']}"
           end
-          host['working_dir'] = "/tmp/" + Time.new.strftime("%Y-%m-%d_%H.%M.%S") #unique working dirs make me happy
-          on host, "mkdir #{host['working_dir']}"
+          host['working_dir'] = host.tmpdir(Time.new.strftime("%Y-%m-%d_%H.%M.%S"))
         end
 
         fetch_puppet(hosts, opts)
@@ -493,6 +503,57 @@ module Beaker
         on install_hosts, puppet_agent('-t'), :acceptable_exit_codes => [0,2]
       end
 
+      #Perform a Puppet Enterprise Higgs install up until web browser interaction is required, runs on linux hosts only.
+      # @param [Host] host The host to install higgs on
+      # @param  [Hash{Symbol=>Symbol, String}] opts The options
+      # @option opts [String] :pe_dir Default directory or URL to pull PE package from
+      #                  (Otherwise uses individual hosts pe_dir)
+      # @option opts [String] :pe_ver Default PE version to install 
+      #                  (Otherwise uses individual hosts pe_ver)
+      # @raise [StandardError] When installation times out
+      #
+      # @example
+      #  do_higgs_install(master, {:pe_dir => path, :pe_ver => version})
+      #
+      # @api private
+      #
+      def do_higgs_install host, opts
+        use_all_tar = ENV['PE_USE_ALL_TAR'] == 'true'
+        platform = use_all_tar ? 'all' : host['platform']
+        version = host['pe_ver'] || opts[:pe_ver]
+        host['dist'] = "puppet-enterprise-#{version}-#{platform}"
+
+        use_all_tar = ENV['PE_USE_ALL_TAR'] == 'true'
+        host['pe_installer'] ||= 'puppet-enterprise-installer'
+        host['working_dir'] = host.tmpdir(Time.new.strftime("%Y-%m-%d_%H.%M.%S"))
+
+        fetch_puppet([host], opts)
+
+        host['higgs_file'] = "higgs_#{File.basename(host['working_dir'])}.log"
+        on host, higgs_installer_cmd(host), opts
+
+        #wait for output to host['higgs_file']
+        #we're all done when we find this line in the PE installation log
+        higgs_re = /Please\s+go\s+to\s+https:\/\/.*\s+in\s+your\s+browser\s+to\s+continue\s+installation/m
+        res = Result.new(host, 'tmp cmd')
+        tries = 10
+        attempts = 0
+        prev_sleep = 0
+        cur_sleep = 1
+        while (res.stdout !~ higgs_re) and (attempts < tries)
+          res = on host, "cd #{host['working_dir']}/#{host['dist']} && cat #{host['higgs_file']}", :acceptable_exit_codes => (0..255)
+          attempts += 1
+          sleep( cur_sleep )
+          prev_sleep = cur_sleep
+          cur_sleep = cur_sleep + prev_sleep
+        end
+
+        if attempts >= tries
+          raise "Failed to kick off PE (Higgs) web installation"
+        end
+
+      end
+
       #Sort array of hosts so that it has the correct order for PE installation based upon each host's role
       # @example
       #  h = sorted_hosts
@@ -551,6 +612,10 @@ module Beaker
               raise "install_puppet() called for unsupported platform '#{host['platform']}' on '#{host.name}'"
             end
           end
+
+          # Certain install paths may not create the config dirs/files needed
+          on host, "mkdir -p #{host['puppetpath']}"
+          on host, "echo '' >> #{host['hieraconf']}"
         end
         nil
       end
@@ -688,20 +753,51 @@ module Beaker
       # @raise [StandardError] if gem does not exist on target host
       # @api private
       def install_puppet_from_gem( host, opts )
-        if host.check_for_command( 'gem' )
-          if opts[:facter_version]
-            on host, "gem install facter -v#{opts[:facter_version]}"
-          end
-          if opts[:hiera_version]
-            on host, "gem install hiera -v#{opts[:hiera_version]}"
-          end
-          ver_cmd = opts[:version] ? "-v#{opts[:version]}" : ''
-          on host, "gem install puppet #{ver_cmd}"
-        else
-          raise "install_puppet() called with default_action 'gem_install' but program `gem' not installed on #{host.name}"
+        # Hosts may be provisioned with csw but pkgutil won't be in the
+        # PATH by default to avoid changing the behavior for Puppet's tests
+        if host['platform'] =~ /solaris-10/
+          on host, 'ln -s /opt/csw/bin/pkgutil /usr/bin/pkgutil'
+        end
+
+        # Solaris doesn't necessarily have this, but gem needs it
+        if host['platform'] =~ /solaris/
+          on host, 'mkdir -p /var/lib' 
+        end
+
+        unless host.check_for_command( 'gem' )
+          gempkg = case host['platform']
+                   when /solaris-11/                   then 'ruby-18'
+                   when /ubuntu-14/                    then 'ruby'
+                   when /solaris-10|ubuntu|debian|el-/ then 'rubygems'
+                   else
+                     raise "install_puppet() called with default_action " +
+                           "'gem_install' but program `gem' is " +
+                           "not installed on #{host.name}"
+                   end
+
+          host.install_package gempkg
+        end
+
+        if host['platform'] =~ /debian|ubuntu|solaris/
+          gem_env = YAML.load( on( host, 'gem environment' ).stdout )
+          gem_paths_array = gem_env['RubyGems Environment'].find {|h| h['GEM PATHS'] != nil }['GEM PATHS']
+          path_with_gem = 'export PATH=' + gem_paths_array.join(':') + ':${PATH}'
+          on host, "echo '#{path_with_gem}' >> ~/.bashrc"
+        end
+
+        if opts[:facter_version]
+          on host, "gem install facter -v#{opts[:facter_version]} --no-ri --no-rdoc"
+        end
+
+        if opts[:hiera_version]
+          on host, "gem install hiera -v#{opts[:hiera_version]} --no-ri --no-rdoc"
         end
+
+        ver_cmd = opts[:version] ? "-v#{opts[:version]}" : ''
+        on host, "gem install puppet #{ver_cmd} --no-ri --no-rdoc"
       end
 
+
       #Install PE based upon host configuration and options
       # @example
       #  install_pe
@@ -777,7 +873,7 @@ module Beaker
           on host, "rpm -ivh --force #{rpm}"
 
         when /^(debian|ubuntu)$/
-          deb = options[:release_apt_repo_url] + "puppetlabs-release-%s.deb" % codename
+          deb = URI.join(options[:release_apt_repo_url],  "puppetlabs-release-%s.deb" % codename)
 
           on host, "wget -O /tmp/puppet.deb #{deb}"
           on host, "dpkg -i --force-all /tmp/puppet.deb"
@@ -888,8 +984,8 @@ module Beaker
           scp_to host, list, config_dir
           scp_to host, repo_dir, "/root/#{package_name}"
 
-          search = "'deb\\s\\+http:\\/\\/#{hostname}.*$"
-          replace = "'deb file:\\/\\/\\/root\\/#{package_name}\\/#{codename} #{codename} main'"
+          search = "deb\\s\\+http:\\/\\/#{hostname}.*$"
+          replace = "deb file:\\/\\/\\/root\\/#{package_name}\\/#{codename} #{codename} main"
           sed_command = "sed -i 's/#{search}/#{replace}/'"
           find_and_sed = "find #{config_dir} -name \"*.list\" -exec #{sed_command} {} \\;"
 
@@ -900,6 +996,29 @@ module Beaker
           raise "No repository installation step for #{variant} yet..."
         end
       end
+
+      #Install Higgs up till the point where you need to continue installation in a web browser, defaults to execution
+      #on the master node.
+      #@param [Host] higgs_host The host to install Higgs on (supported on linux platform only)
+      # @example
+      #  install_higgs
+      #
+      # @note Either pe_ver and pe_dir should be set in the ENV or each host should have pe_ver and pe_dir set individually.
+      #       Install file names are assumed to be of the format puppet-enterprise-VERSION-PLATFORM.(tar)|(tar.gz).
+      #
+      # @api dsl
+      def install_higgs( higgs_host = master )
+        #process the version files if necessary
+        master['pe_dir'] ||= options[:pe_dir]
+        master['pe_ver'] = master['pe_ver'] || options['pe_ver'] ||
+          Beaker::Options::PEVersionScraper.load_pe_version(master[:pe_dir] || options[:pe_dir], options[:pe_version_file])
+        if higgs_host['platform'] =~ /osx|windows/
+          raise "Attempting higgs installation on host #{higgs_host.name} with unsupported platform #{higgs_host['platform']}"
+        end
+        #send in the global options hash
+        do_higgs_install higgs_host, options
+      end
+
     end
   end
 end

data/lib/beaker/dsl/patterns.rb

@@ -0,0 +1,37 @@
+module Beaker
+  module DSL
+    # These are simple patterns that appear frequently in beaker test
+    # code, and are provided to simplify test construction.
+    #
+    #
+    # It requires the class it is mixed into to provide the attribute
+    # `hosts` which contain the hosts to search, these should implement
+    # {Beaker::Host}'s interface. They, at least, must have #[]
+    # and #to_s available and provide an array when #[]('roles') is called.
+    #
+    module Patterns
+
+      #Execute a block selecting the hosts that match with the provided criteria
+      #@param [Array<Host>, Host, String, Symbol] hosts_or_filter A host role as a String or Symbol that can be
+      #                                                used to search for a set of Hosts,  a host name
+      #                                                as a String that can be used to search for
+      #                                                a set of Hosts, or a {Host}
+      #                                                or Array<{Host}> to run the block against
+      #@param [Block] block This method will yield to a block of code passed by the caller
+      def block_on hosts_or_filter, &block
+        block_hosts = nil
+        if defined? hosts
+          block_hosts = hosts
+        end
+        filter = nil
+        if hosts_or_filter.is_a? String or hosts_or_filter.is_a? Symbol
+          filter = hosts_or_filter
+        else
+          block_hosts = hosts_or_filter
+        end
+        run_block_on block_hosts, filter, &block
+      end
+
+    end
+  end
+end

data/lib/beaker/dsl/roles.rb

@@ -88,6 +88,35 @@ module Beaker
         find_only_one :default
       end
 
+      #Create a new role method for a given arbitrary role name.  Makes it possible to be able to run
+      #commands without having to refer to role by String or Symbol.  Will not add a new method 
+      #definition if the name is already in use.
+      # @param [String, Symbol, Array[String,Symbol]] role The role that you wish to create a definition for, either a String
+      # Symbol or an Array of Strings or Symbols.
+      # @example Basic usage
+      #  add_role_def('myrole')
+      #  on myrole, "run command"
+      def add_role_def role
+        if role.kind_of?(Array)
+          role.each do |r|
+            add_role_def r
+          end
+        else
+          if not respond_to? role
+            if role !~ /\A[[:alpha:]]+[a-zA-Z0-9_]*[!?=]?\Z/
+              raise "Role name format error for '#{role}'.  Allowed characters are: \na-Z\n0-9 (as long as not at the beginning of name)\n'_'\n'?', '!' and '=' (only as individual last character at end of name)"
+            end
+            self.class.send :define_method, role.to_s do
+              hosts_with_role = hosts_as role.to_sym
+              if hosts_with_role.length == 1
+                hosts_with_role = hosts_with_role.pop
+              end
+              hosts_with_role
+            end
+          end
+        end
+      end
+
       # Determine if there is a host or hosts with the given role defined
       # @return [Boolean] True if there is a host with role, false otherwise
       #

data/lib/beaker/dsl.rb

@@ -1,4 +1,4 @@
-[ 'install_utils', 'roles', 'outcomes', 'assertions',
+[ 'install_utils', 'roles', 'outcomes', 'assertions', 'patterns',
   'structure', 'helpers', 'ezbake_utils', 'wrappers' ].each do |lib|
   require "beaker/dsl/#{lib}"
 end
@@ -78,5 +78,6 @@ module Beaker
     include Beaker::DSL::Helpers
     include Beaker::DSL::EZBakeUtils
     include Beaker::DSL::InstallUtils
+    include Beaker::DSL::Patterns
   end
 end

data/lib/beaker/host/unix.rb

@@ -20,16 +20,18 @@ module Unix
         'user'          => 'root',
         'group'         => 'pe-puppet',
         'master-start-curl-retries' => 120,
-        'puppetservice' => 'pe-httpd',
-        'puppetpath'    => '/etc/puppetlabs/puppet',
-        'puppetbin'     => '/opt/puppet/bin/puppet',
-        'puppetbindir'  => '/opt/puppet/bin',
-        'puppetvardir'  => '/var/opt/lib/pe-puppet',
-        'hieradatadir'  => '/var/lib/hiera',
-        'hieraconf'     => '/etc/puppetlabs/puppet/hiera.yaml',
-        'distmoduledir' => '/etc/puppetlabs/puppet/modules',
-        'sitemoduledir' => '/opt/puppet/share/puppet/modules',
-        'pathseparator' => ':',
+        'jvm-puppet-confdir' => '/etc/puppetlabs/jvm-puppet/conf.d',
+        'puppetservice'    => 'pe-httpd',
+        'puppetpath'       => '/etc/puppetlabs/puppet',
+        'puppetbin'        => '/opt/puppet/bin/puppet',
+        'puppetbindir'     => '/opt/puppet/bin',
+        'puppetsbindir'    => '/opt/puppet/sbin',
+        'puppetvardir'     => '/var/opt/lib/pe-puppet',
+        'hieradatadir'     => '/var/lib/hiera',
+        'hieraconf'        => '/etc/puppetlabs/puppet/hiera.yaml',
+        'distmoduledir'    => '/etc/puppetlabs/puppet/modules',
+        'sitemoduledir'    => '/opt/puppet/share/puppet/modules',
+        'pathseparator'    => ':',
       })
     end
 
@@ -39,6 +41,8 @@ module Unix
         'user'              => 'root',
         'group'             => 'puppet',
         'master-start-curl-retries' => 120,
+        'jvm-puppet-confdir' => '/etc/jvm-puppet/conf.d',
+        'puppetservice'     => 'puppetmaster',
         'puppetpath'        => '/etc/puppet',
         'puppetvardir'      => '/var/lib/puppet',
         'puppetbin'         => '/usr/bin/puppet',

data/lib/beaker/host/windows.rb

@@ -21,6 +21,7 @@ module Windows
         'group'         => 'Administrators',
         'puppetservice' => 'pe-httpd',
         'puppetpath'    => '`cygpath -smF 35`/PuppetLabs/puppet/etc',
+        'hieraconf'     => '`cygpath -smF 35`/Puppetlabs/puppet/etc/hiera.yaml',
         'puppetvardir'  => '`cygpath -smF 35`/PuppetLabs/puppet/var',
         'distmoduledir' => '`cygpath -smF 35`/PuppetLabs/puppet/etc/modules',
         'sitemoduledir' => 'C:/usr/share/puppet/modules',
@@ -36,6 +37,7 @@ module Windows
         'user'              => 'Administrator',
         'group'             => 'Administrators',
         'puppetpath'        => '`cygpath -smF 35`/PuppetLabs/puppet/etc',
+        'hieraconf'         => '`cygpath -smF 35`/Puppetlabs/puppet/etc/hiera.yaml',
         'puppetvardir'      => '`cygpath -smF 35`/PuppetLabs/puppet/var',
         'distmoduledir'     => '`cygpath -smF 35`/PuppetLabs/puppet/etc/modules',
         'sitemoduledir'     => 'C:/usr/share/puppet/modules',

data/lib/beaker/host.rb

@@ -137,6 +137,43 @@ module Beaker
       @options.is_pe?
     end
 
+    # True if this is a pe run, or if the host has had a 'use-service' property set.
+    def use_service_scripts?
+      is_pe? || self['use-service']
+    end
+
+    # Mirrors the true/false value of the host's 'graceful-restarts' property,
+    # or falls back to the value of +is_using_passenger?+ if
+    # 'graceful-restarts' is nil, but only if this is not a PE run (foss only).
+    def graceful_restarts?
+      graceful =
+      if !self['graceful-restarts'].nil?
+        self['graceful-restarts']
+      else
+        !is_pe? && is_using_passenger?
+      end
+      graceful
+    end
+
+    # Modifies the host settings to indicate that it will be using passenger service scripts,
+    # (apache2) by default.  Does nothing if this is a PE host, since it is already using
+    # passenger.
+    # @param [String] puppetservice Name of the service script that should be
+    #   called to stop/startPuppet on this host.  Defaults to 'apache2'.
+    def uses_passenger!(puppetservice = 'apache2')
+      if !is_pe?
+        self['passenger'] = true
+        self['puppetservice'] = puppetservice
+        self['use-service'] = true
+      end
+      return true
+    end
+
+    # True if this is a PE run, or if the host's 'passenger' property has been set.
+    def is_using_passenger?
+      is_pe? || self['passenger']
+    end
+
     def log_prefix
       if @defaults['vmhostname']
         "#{self} (#{@name})"
@@ -206,9 +243,66 @@ module Beaker
       end
     end
 
+    # Create the provided directory structure on the host
+    # @param [String] dir The directory structure to create on the host
+    # @return [Boolean] True, if directory construction succeeded, otherwise False
+    def mkdir_p dir
+      result = exec(Beaker::Command.new("mkdir -p #{dir}"), :acceptable_exit_codes => [0, 1])
+      result.exit_code == 0
+    end
+
+    # scp files from the localhost to this test host
+    # @param source [String] The path to the file/dir to upload
+    # @param target [String] The destination path on the host
+    # @param [Hash{Symbol=>String}] options Options to alter execution
+    # @option options [Boolean] :recursive Should we copy recursively?  Defaults to 'True' in case of a directory source.
+    # @option options [Array<String>] :ignore An array of file/dir paths that will not be copied to the host
     def do_scp_to source, target, options
       @logger.debug "localhost $ scp #{source} #{@name}:#{target}"
-      result = connection.scp_to(source, target, options, $dry_run)
+
+      result = Result.new(@name, [source, target])
+      has_ignore = options[:ignore] and not options[:ignore].empty?
+      # construct the regex for matching ignored files/dirs
+      ignore_re = nil
+      if has_ignore
+        ignore_arr = Array(options[:ignore]).map do |entry|
+          "((\/|\\A)#{entry}(\/|\\z))".sub(/\./, "\.")
+        end
+        ignore_re = Regexp.new(ignore_arr.join('|'))
+      end
+
+      # either a single file, or a directory with no ignores
+      if File.file?(source) or (File.directory?(source) and not has_ignore)
+        source_file = source
+        if has_ignore and (source =~ ignore_re)
+          @logger.debug "After rejecting ignored files/dirs, there is no file to copy"
+          source_file = nil
+          result.stdout = "No files to copy"
+          result.exit_code = 1
+        end
+        if source_file
+          result = connection.scp_to(source_file, target, options, $dry_run)
+        end
+      else # a directory with ignores
+        dir_source = Dir.glob("#{source}/**/*").reject do |f|
+          f =~ ignore_re
+        end
+        @logger.debug "After rejecting ignored files/dirs, going to scp [#{dir_source.join(", ")}]"
+
+        # create necessary directory structure on host
+        required_dirs = (dir_source.map{ | dir | File.dirname(dir) }).uniq
+        required_dirs.each do |dir|
+          mkdir_p( File.join(target, dir) )
+        end
+
+        # copy each file to the host
+        dir_source.each do |s|
+          file_path = File.join(target, s)
+          result = connection.scp_to(s, file_path, options, $dry_run)
+        end
+      end
+
+      @logger.debug result.stdout
       return result
     end
 
@@ -216,6 +310,7 @@ module Beaker
 
       @logger.debug "localhost $ scp #{@name}:#{source} #{target}"
       result = connection.scp_from(source, target, options, $dry_run)
+      @logger.debug result.stdout
       return result
     end