beaker 1.13.1 → 1.14.0

data/lib/beaker/dsl/install_utils.rb

@@ -143,26 +143,32 @@ module Beaker
       # @param [Hash{Symbol=>String}] opts The options
       # @option opts [String] :pe_ver_win Default PE version to install or upgrade to on Windows hosts
       #                          (Othersie uses individual Windows hosts pe_ver)
-      # @option opts [String  :pe_ver Default PE version to install or upgrade to
+      # @option opts [String]  :pe_ver Default PE version to install or upgrade to
       #                          (Otherwise uses individual hosts pe_ver)
+      # @option opts [Boolean] :pe_debug (false) Should we run the installer in debug mode?
       # @example
       #      on host, "#{installer_cmd(host, opts)} -a #{host['working_dir']}/answers"
       # @api private
       def installer_cmd(host, opts)
         version = host['pe_ver'] || opts[:pe_ver]
         if host['platform'] =~ /windows/
-          version = opts[:pe_ver_win] || host['pe_ver']
-          "cd #{host['working_dir']} && cmd /C 'start /w msiexec.exe /qn /i puppet-enterprise-#{version}.msi PUPPET_MASTER_SERVER=#{master} PUPPET_AGENT_CERTNAME=#{host}'"
+          version = host[:pe_ver] || opts['pe_ver_win']
+          log_file = "#{File.basename(host['working_dir'])}.log"
+          pe_debug = host[:pe_debug] || opts[:pe_debug] ? " && cat #{log_file}" : ''
+          "cd #{host['working_dir']} && cmd /C 'start /w msiexec.exe /qn /L*V #{log_file} /i puppet-enterprise-#{version}.msi PUPPET_MASTER_SERVER=#{master} PUPPET_AGENT_CERTNAME=#{host}'#{pe_debug}"
         elsif host['platform'] =~ /osx/
           version = host['pe_ver'] || opts[:pe_ver]
-          "cd #{host['working_dir']} && hdiutil attach #{host['dist']}.dmg && installer -pkg /Volumes/puppet-enterprise-#{version}/puppet-enterprise-installer-#{version}.pkg -target /"
+          pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -verboseR' : ''
+          "cd #{host['working_dir']} && hdiutil attach #{host['dist']}.dmg && installer#{pe_debug} -pkg /Volumes/puppet-enterprise-#{version}/puppet-enterprise-installer-#{version}.pkg -target /"
 
         # Frictionless install didn't exist pre-3.2.0, so in that case we fall
         # through and do a regular install.
         elsif host['roles'].include? 'frictionless' and ! version_is_less(version, '3.2.0')
-          "cd #{host['working_dir']} && curl -kO https://#{master}:8140/packages/#{version}/install.bash && bash install.bash"
+          pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -x' : ''
+          "cd #{host['working_dir']} && curl -kO https://#{master}:8140/packages/#{version}/install.bash && bash#{pe_debug} install.bash"
         else
-          "cd #{host['working_dir']}/#{host['dist']} && ./#{host['pe_installer']} -a #{host['working_dir']}/answers"
+          pe_debug = host[:pe_debug] || opts[:pe_debug]  ? ' -D' : ''
+          "cd #{host['working_dir']}/#{host['dist']} && ./#{host['pe_installer']}#{pe_debug} -a #{host['working_dir']}/answers"
         end
       end
 
@@ -184,6 +190,74 @@ module Beaker
         end
       end
 
+      # Fetch file_name from the given base_url into dst_dir.
+      #
+      # @param [String] base_url The base url from which to recursively download
+      #                          files.
+      # @param [String] file_name The trailing name compnent of both the source url
+      #                           and the destination file.
+      # @param [String] dst_dir The local destination directory.
+      #
+      # @return [String] dst The name of the newly-created file.
+      #
+      # @!visibility private
+      def fetch_http_file(base_url, file_name, dst_dir)
+        FileUtils.makedirs(dst_dir)
+        src = "#{base_url}/#{file_name}"
+        dst = File.join(dst_dir, file_name)
+        if File.exists?(dst)
+          logger.notify "Already fetched #{dst}"
+        else
+          logger.notify "Fetching: #{src}"
+          logger.notify "  and saving to #{dst}"
+          open(src) do |remote|
+            File.open(dst, "w") do |file|
+              FileUtils.copy_stream(remote, file)
+            end
+          end
+        end
+        return dst
+      end
+
+      # Recursively fetch the contents of the given http url, ignoring
+      # `index.html` and `*.gif` files.
+      #
+      # @param [String] url The base http url from which to recursively download
+      #                     files.
+      # @param [String] dst_dir The local destination directory.
+      #
+      # @return [String] dst The name of the newly-created subdirectory of
+      #                      dst_dir.
+      #
+      # @!visibility private
+      def fetch_http_dir(url, dst_dir)
+        logger.notify "fetch_http_dir (url: #{url}, dst_dir #{dst_dir})"
+        if url[-1, 1] !~ /\//
+          url += '/'
+        end
+        url = URI.parse(url)
+        chunks = url.path.split('/')
+        dst = File.join(dst_dir, chunks.last)
+        #determine directory structure to cut
+        #only want to keep the last directory, thus cut total number of dirs - 2 (hostname + last dir name)
+        cut = chunks.length - 2
+        wget_command = "wget -nv -P #{dst_dir} --reject \"index.html*\",\"*.gif\" --cut-dirs=#{cut} -np -nH --no-check-certificate -r #{url}"
+
+        logger.notify "Fetching remote directory: #{url}"
+        logger.notify "  and saving to #{dst}"
+        logger.notify "  using command: #{wget_command}"
+
+        #in ruby 1.9+ we can upgrade this to popen3 to gain access to the subprocess pid
+        result = `#{wget_command} 2>&1`
+        result.each_line do |line|
+          logger.debug(line)
+        end
+        if $?.to_i != 0
+          raise "Failed to fetch_remote_dir '#{url}' (exit code #{$?}"
+        end
+        dst
+      end
+
       #Determine the PE package to download/upload on a mac host, download/upload that package onto the host.
       # Assumed file name format: puppet-enterprise-3.3.0-rc1-559-g97f0833-osx-10.9-x86_64.dmg.
       # @param [Host] host The mac host to download/upload and unpack PE onto
@@ -545,8 +619,12 @@ module Beaker
           on host, "apt-get install -y hiera=#{opts[:hiera_version]}-1puppetlabs1"
         end
 
-        puppet_pkg = opts[:version] ? "puppet=#{opts[:version]}-1puppetlabs1" : 'puppet'
-        on host, "apt-get install -y #{puppet_pkg}"
+        if opts[:version]
+          on host, "apt-get install -y puppet-common=#{opts[:version]}-1puppetlabs1"
+          on host, "apt-get install -y puppet=#{opts[:version]}-1puppetlabs1"
+        else
+          on host, 'apt-get install -y puppet'
+        end
       end
 
       # Installs Puppet and dependencies from msi
@@ -559,7 +637,7 @@ module Beaker
       # @api private
       def install_puppet_from_msi( host, opts )
         on host, "curl -O http://downloads.puppetlabs.com/windows/puppet-#{opts[:version]}.msi"
-        on host, "msiexec /qn /i puppet-#{relver}.msi"
+        on host, "msiexec /qn /i puppet-#{opts[:version]}.msi"
 
         #Because the msi installer doesn't add Puppet to the environment path
         if fact_on(host, 'architecture').eql?('x86_64')
@@ -678,6 +756,148 @@ module Beaker
         do_install(sorted_hosts, options.merge({:type => :upgrade}))
         options['upgrade'] = true
       end
+
+      # Install official puppetlabs release repository configuration on host.
+      #
+      # @param [Host] host An object implementing {Beaker::Hosts}'s
+      #                    interface.
+      #
+      # @note This method only works on redhat-like and debian-like hosts.
+      #
+      def install_puppetlabs_release_repo ( host )
+        variant, version, arch, codename = host['platform'].to_array
+
+        case variant
+        when /^(fedora|el|centos)$/
+          variant = (($1 == 'centos') ? 'el' : $1)
+
+          rpm = options[:release_yum_repo_url] +
+            "/puppetlabs-release-%s-%s.noarch.rpm" % [variant, version]
+
+          on host, "rpm -ivh --force #{rpm}"
+
+        when /^(debian|ubuntu)$/
+          deb = options[:release_apt_repo_url] + "puppetlabs-release-%s.deb" % codename
+
+          on host, "wget -O /tmp/puppet.deb #{deb}"
+          on host, "dpkg -i --force-all /tmp/puppet.deb"
+          on host, "apt-get update"
+        else
+          raise "No repository installation step for #{variant} yet..."
+        end
+      end
+
+      # Install development repository on the given host. This method pushes all
+      # repository information including package files for the specified
+      # package_name to the host and modifies the repository configuration file
+      # to point at the new repository. This is particularly useful for
+      # installing development packages on hosts that can't access the builds
+      # server.
+      #
+      # @param [Host] host An object implementing {Beaker::Hosts}'s
+      #                    interface.
+      # @param [String] package_name The name of the package whose repository is
+      #                              being installed.
+      # @param [String] build_version A string identifying the output of a
+      #                               packaging job for use in looking up
+      #                               repository directory information
+      # @param [String] repo_configs_dir A local directory where repository files will be
+      #                                  stored as an intermediate step before
+      #                                  pushing them to the given host.
+      #
+      # @note This method only works on redhat-like and debian-like hosts.
+      #
+      def install_puppetlabs_dev_repo ( host, package_name, build_version,
+                                repo_configs_dir = 'tmp/repo_configs' )
+        variant, version, arch, codename = host['platform'].to_array
+        platform_configs_dir = File.join(repo_configs_dir, variant)
+
+        # some of the uses of dev_builds_url below can't include protocol info,
+        # pluse this opens up possibility of switching the behavior on provided
+        # url type
+        _, protocol, hostname = options[:dev_builds_url].partition /.*:\/\//
+        dev_builds_url = protocol + hostname
+
+        case variant
+        when /^(fedora|el|centos)$/
+          variant = (($1 == 'centos') ? 'el' : $1)
+          fedora_prefix = ((variant == 'fedora') ? 'f' : '')
+
+          if host.is_pe?
+            pattern = "pl-%s-%s-repos-pe-%s-%s%s-%s.repo"
+          else
+            pattern = "pl-%s-%s-%s-%s%s-%s.repo"
+          end
+
+          repo_filename = pattern % [
+            package_name,
+            build_version,
+            variant,
+            fedora_prefix,
+            version,
+            arch
+          ]
+
+          repo = fetch_http_file( "%s/%s/%s/repo_configs/rpm/" %
+                       [ dev_builds_url, package_name, build_version ],
+                        repo_filename,
+                        platform_configs_dir)
+
+          link = "%s/%s/%s/repos/%s/%s%s/products/%s/" %
+            [ dev_builds_url, package_name, build_version, variant,
+              fedora_prefix, version, arch ]
+
+          if not link_exists?( link )
+            link = "%s/%s/%s/repos/%s/%s%s/devel/%s/" %
+              [ dev_builds_url, package_name, build_version, variant,
+                fedora_prefix, version, arch ]
+          end
+
+          if not link_exists?( link )
+            raise "Unable to reach a repo directory at #{link}"
+          end
+
+          repo_dir = fetch_http_dir( link, platform_configs_dir )
+
+          config_dir = '/etc/yum.repos.d/'
+          scp_to host, repo, config_dir
+          scp_to host, repo_dir, '/root'
+
+          search = "baseurl\\s*=\\s*http:\\/\\/#{hostname}.*$"
+          replace = "baseurl=file:\\/\\/\\/root\\/#{arch}"
+          sed_command = "sed -i 's/#{search}/#{replace}/'"
+          find_and_sed = "find #{config_dir} -name \"*.repo\" -exec #{sed_command} {} \\;"
+
+          on host, find_and_sed
+
+        when /^(debian|ubuntu)$/
+          list = fetch_http_file( "%s/%s/%s/repo_configs/deb/" %
+                         [ dev_builds_url, package_name, build_version ],
+                        "pl-%s-%s-%s.list" %
+                         [ package_name, build_version, codename ],
+                        platform_configs_dir )
+
+          repo_dir = fetch_http_dir( "%s/%s/%s/repos/apt/%s" %
+                                      [ dev_builds_url, package_name,
+                                        build_version, codename ],
+                                       platform_configs_dir )
+
+          config_dir = '/etc/apt/sources.list.d'
+          scp_to host, list, config_dir
+          scp_to host, repo_dir, '/root'
+
+          search = "'deb\\s\\+http:\\/\\/#{hostname}.*$"
+          replace = "'deb file:\\/\\/\\/root\\/#{codename} #{codename} main'"
+          sed_command = "sed -i 's/#{search}/#{replace}/'"
+          find_and_sed = "find #{config_dir} -name \"*.list\" -exec #{sed_command} {} \\;"
+
+          on host, find_and_sed
+          on host, "apt-get update"
+
+        else
+          raise "No repository installation step for #{variant} yet..."
+        end
+      end
     end
   end
 end

data/lib/beaker/host/unix.rb

@@ -19,8 +19,8 @@ module Unix
       h.merge({
         'user'          => 'root',
         'group'         => 'pe-puppet',
-        'service-wait'  => false,
         'service-prefix'=> '/etc/init.d/',
+        'master-start-curl-retries' => 120,
         'puppetservice' => 'pe-httpd',
         'puppetpath'    => '/etc/puppetlabs/puppet',
         'puppetbin'     => '/opt/puppet/bin/puppet',
@@ -39,7 +39,7 @@ module Unix
       h.merge({
         'user'              => 'root',
         'group'             => 'puppet',
-        'service-wait'      => false,
+        'master-start-curl-retries' => 120,
         'puppetpath'        => '/etc/puppet',
         'puppetvardir'      => '/var/lib/puppet',
         'puppetbin'         => '/usr/bin/puppet',

data/lib/beaker/host/unix/pkg.rb

@@ -49,17 +49,23 @@ module Unix::Pkg
     end
   end
 
-  def install_package(name, cmdline_args = '')
+  def install_package(name, cmdline_args = '', version = nil)
     case self['platform']
       when /sles-/
         execute("zypper --non-interactive in #{name}")
       when /el-4/
         @logger.debug("Package installation not supported on rhel4")
       when /fedora|centos|el-/
+        if version
+          name = "#{name}-#{version}"
+        end
         execute("yum -y #{cmdline_args} install #{name}")
       when /ubuntu|debian/
+        if version
+          name = "#{name}=#{version}"
+        end
         update_apt_if_needed
-        execute("apt-get install #{cmdline_args} -y #{name}")
+        execute("apt-get install --force-yes #{cmdline_args} -y #{name}")
       when /solaris-11/
         execute("pkg #{cmdline_args} install #{name}")
       when /solaris-10/

data/lib/beaker/host/windows.rb

@@ -19,7 +19,6 @@ module Windows
       h.merge({
         'user'          => 'Administrator',
         'group'         => 'Administrators',
-        'service-wait'  => false,
         'puppetservice' => 'pe-httpd',
         'puppetpath'    => '`cygpath -smF 35`/PuppetLabs/puppet/etc',
         'puppetvardir'  => '`cygpath -smF 35`/PuppetLabs/puppet/var',

data/lib/beaker/host_prebuilt_steps.rb

@@ -91,7 +91,7 @@ module Beaker
               host.install_package pkg
             end
           end
-        when host['platform'] !~ /aix|solaris|windows|sles-/
+        when host['platform'] !~ /aix|solaris|windows|sles-|osx-/
           UNIX_PACKAGES.each do |pkg|
             if not host.check_for_package pkg
               host.install_package pkg
@@ -103,48 +103,6 @@ module Beaker
       report_and_raise(logger, e, "validate")
     end
 
-    #Update /etc/hosts on the master node to include a rule for lookup of the master by name/ip.
-    # @param [Host, Array<Host>, String, Symbol] hosts One or more hosts to act upon
-    # @param [Hash{Symbol=>String}] opts Options to alter execution.
-    # @option opts [Beaker::Logger] :logger A {Beaker::Logger} object
-    def add_master_entry hosts, opts
-      logger = opts[:logger]
-      master = only_host_with_role(hosts, :master)
-      logger.notify "Add Master entry to /etc/hosts on #{master.name}"
-      if master["hypervisor"] and master["hypervisor"] =~ /docker/
-        # skip on docker because as of 0.8.1 /etc/hosts isn't modifiable
-        # https://github.com/dotcloud/docker/issues/2267
-        logger.debug "Don't update master entry on docker masters"
-        return
-      end
-      if master["hypervisor"] and master["hypervisor"] =~ /vagrant/
-        logger.debug "Don't update master entry on vagrant masters"
-        return
-      end
-      logger.debug "Get ip address of Master #{master}"
-      if master['platform'].include? 'solaris'
-        stdout = master.exec(Command.new("ifconfig -a inet| awk '/broadcast/ {print $2}' | cut -d/ -f1 | head -1")).stdout
-      else
-        stdout = master.exec(Command.new("ip a|awk '/global/{print$2}' | cut -d/ -f1 | head -1")).stdout
-      end
-      ip=stdout.chomp
-
-      path = ETC_HOSTS_PATH
-      if master['platform'].include? 'solaris'
-        path = ETC_HOSTS_PATH_SOLARIS
-      end
-
-      logger.debug "Update %s on #{master}" % path
-      # Preserve the mode the easy way...
-      master.exec(Command.new("cp %s %s.old" % [path, path]))
-      master.exec(Command.new("cp %s %s.new" % [path, path]))
-      master.exec(Command.new("grep -v '#{ip} #{master}' %s > %s.new" % [path, path]))
-      master.exec(Command.new("echo '#{ip} #{master}' >> %s.new" % path))
-      master.exec(Command.new("mv %s.new %s" % [path, path]))
-    rescue => e
-      report_and_raise(logger, e, "add_master_entry")
-    end
-
     #Install a set of authorized keys using {HostPrebuiltSteps::ROOT_KEYS_SCRIPT}.  This is a
     #convenience method to allow for easy login to hosts after they have been provisioned with
     #Beaker.

data/lib/beaker/hypervisor.rb

@@ -88,9 +88,6 @@ module Beaker
       if @options[:add_el_extras]
         add_el_extras(@hosts, @options)
       end
-      if @options[:add_master_entry]
-        add_master_entry(@hosts, @options)
-      end
       if @options[:package_proxy]
         package_proxy(@hosts, @options)
       end

data/lib/beaker/hypervisor/docker.rb

@@ -35,7 +35,12 @@ module Beaker
         container.start({"PublishAllPorts" => true, "Privileged" => true})
 
         # Find out where the ssh port is from the container
-        ip   = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostIp"]
+        if ENV['DOCKER_HOST']
+          ip = URI.parse(ENV['DOCKER_HOST']).host
+          @logger.info("Using docker server at #{ip}")
+        else
+          ip = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostIp"]
+        end
         port = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostPort"]
 
         # Update host metadata

data/lib/beaker/logger.rb

@@ -133,6 +133,25 @@ module Beaker
       LOG_LEVELS[@log_level] >= LOG_LEVELS[:notify]
     end
 
+    # Remove invalid UTF-8 codes from provided string(s)
+    # @param [String, Array<String>] string The string(s) to remove invalid codes from
+    def convert string
+      if string.kind_of?(Array)
+        string.map do |s|
+          convert s
+        end
+      else
+        if string.respond_to?( :encode )
+          # We're running in >= 1.9 and we'll need to convert
+          # Remove invalide and undefined UTF-8 character encodings
+          string.encode('UTF-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
+        else
+          # We're running 1.8, do nothing
+          string
+        end
+      end
+    end
+
     # Custom reporting for messages generated by host SUTs.
     # Will not print unless we are at {LOG_LEVELS} 'verbose' or higher.
     # Strips any color codes already in the provided messages, then adds logger color codes before reporting
@@ -196,8 +215,8 @@ module Beaker
     # @param [String] lines A single or array of lines to removed color codes from
     # @return [Array<String>] An array of strings that do not have color codes
     def strip_colors_from lines
-      Array(lines).map do |line|
-        line.gsub /\e\[(\d+;)?\d+m/, ''
+      Array( lines ).map do |line|
+        convert(line).gsub(/(\e|\^\[)\[(\d*;)*\d*m/, '')
       end
     end
 
@@ -209,7 +228,7 @@ module Beaker
       print_statement = add_newline ? :puts : :print
       @destinations.each do |to|
         to.print color_code if @color
-        to.send print_statement, msg
+        to.send print_statement, convert( msg )
         to.print NORMAL if @color
       end
     end