beaker 1.21.0 → 2.0.0

data/Rakefile

@@ -19,11 +19,15 @@ task :travis do
 end
 
 namespace :test do
-  desc 'Run specs (with coverage on 1.9), alias `spec` & the default'
+  desc 'Run specs and check for deprecation warnings'
   task :spec do
     original_dir = Dir.pwd
     Dir.chdir( File.expand_path(File.dirname(__FILE__)) )
-    sh 'export COVERAGE=true; bundle exec rspec'
+    output = `export COVERAGE=true; bundle exec rspec`
+    puts output
+    if output =~ /Deprecation Warnings/
+      raise "Deprecation Warnings in spec generation, please fix!"
+    end
     Dir.chdir( original_dir )
   end
 end

data/beaker.gemspec

@@ -18,39 +18,35 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Testing dependencies
-  s.add_development_dependency 'minitest', '~> 4.0'
-  s.add_development_dependency 'rspec', '~> 2.14.0'
-  s.add_development_dependency 'fakefs', '0.4'
-  s.add_development_dependency 'rake', '~> 10.1.0'
-  s.add_development_dependency 'simplecov' unless RUBY_VERSION < '1.9'
-  s.add_development_dependency 'pry', '~> 0.9.12.6'
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rspec-its'
+  s.add_development_dependency 'fakefs', '~> 0.6'
+  s.add_development_dependency 'rake', '~> 10.1'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'pry', '~> 0.10'
 
   # Documentation dependencies
   s.add_development_dependency 'yard'
-  s.add_development_dependency 'markdown' unless RUBY_VERSION < '1.9'
+  s.add_development_dependency 'markdown'
   s.add_development_dependency 'thin'
   s.add_development_dependency 'gitlab-grit'
 
   # Run time dependencies
+  s.add_runtime_dependency 'minitest', '~> 5.4'
   s.add_runtime_dependency 'json', '~> 1.8'
   s.add_runtime_dependency 'hocon', '~> 0.0.4'
-  s.add_runtime_dependency 'net-ssh', '~> 2.6'
-  s.add_runtime_dependency 'net-scp', '~> 1.1'
+  s.add_runtime_dependency 'net-ssh', '~> 2.9'
+  s.add_runtime_dependency 'net-scp', '~> 1.2'
   s.add_runtime_dependency 'inifile', '~> 2.0'
 
   # Optional provisioner specific support
-  s.add_runtime_dependency 'rbvmomi', '1.8.1'
-  s.add_runtime_dependency 'blimpy', '~> 0.6'
+  s.add_runtime_dependency 'rbvmomi', '~> 1.8'
   s.add_runtime_dependency 'fission', '~> 0.4'
-  s.add_runtime_dependency 'google-api-client', '~> 0.7.1'
-  s.add_runtime_dependency 'aws-sdk', '1.42.0'
-  s.add_runtime_dependency 'docker-api' unless RUBY_VERSION < '1.9'
-  s.add_runtime_dependency 'fog', '~> 1.22.1'
+  s.add_runtime_dependency 'google-api-client', '~> 0.7'
+  s.add_runtime_dependency 'aws-sdk', '~> 1.57'
+  s.add_runtime_dependency 'docker-api'
+  s.add_runtime_dependency 'fog', '~> 1.25'
 
-  # These are transitive dependencies that we include or pin to because...
-  # Ruby 1.8 compatibility
-  s.add_runtime_dependency 'nokogiri', '~> 1.5.10'
-  s.add_runtime_dependency 'mime-types', '~> 1.25' # 2.0 won't install on 1.8
   # So fog doesn't always complain of unmet AWS dependencies
   s.add_runtime_dependency 'unf', '~> 0.1'
 end

data/lib/beaker.rb

@@ -33,11 +33,8 @@ module Beaker
   # Shared methods and helpers
   require 'beaker/shared'
 
-  # utf-8 support in ruby 1.8
-  if RUBY_VERSION.to_f < 1.9
-    require 'jcode'
-    $KCODE="u"
-  end
+  # MiniTest, for including MiniTest::Assertions
+  require 'minitest/test'
 
   # Add pry support when available
   begin

data/lib/beaker/answers.rb

@@ -15,6 +15,8 @@ module Beaker
     #   data.
     def self.create version, hosts, options
       case version
+      when /\A3\.7/
+        return Version34.new(version, hosts, options)
       when /\A3\.4/
         return Version34.new(version, hosts, options)
       when /\A3\.[2-3]/

data/lib/beaker/answers/version34.rb

@@ -6,7 +6,43 @@ module Beaker
   # @api private
   class Version34 < Version32
     def generate_answers
-      super
+      dashboard = only_host_with_role(@hosts, 'dashboard')
+      database = only_host_with_role(@hosts, 'database')
+
+      the_answers = super
+
+      classifier_database_user     = answer_for(@options, :q_classifier_database_user, 'DFGhjlkj')
+      classifier_database_name     = answer_for(@options, :q_database_name, 'pe-classifier')
+      classifier_database_password = "'#{@options[:answers][:q_classifier_database_password]}'"
+      activity_database_user       = answer_for(@options, :q_activity_database_user, 'adsfglkj')
+      activity_database_name       = answer_for(@options, :q_activity_database_name, 'pe-activity')
+      activity_database_password   = "'#{@options[:answers][:q_activity_database_password]}'"
+      rbac_database_user           = answer_for(@options, :q_rbac_database_user, 'RbhNBklm')
+      rbac_database_name           = answer_for(@options, :q_rbac_database_name, 'pe-rbac')
+      rbac_database_password       = "'#{@options[:answers][:q_rbac_database_password]}'"
+
+      console_services_hash = {
+        :q_classifier_database_user     => classifier_database_user,
+        :q_classifier_database_name     => classifier_database_name,
+        :q_classifier_database_password => classifier_database_password,
+        :q_activity_database_user       => activity_database_user,
+        :q_activity_database_name       => activity_database_name,
+        :q_activity_database_password   => activity_database_password,
+        :q_rbac_database_user           => rbac_database_user,
+        :q_rbac_database_name           => rbac_database_name,
+        :q_rbac_database_password       => rbac_database_password,
+      }
+
+      # If we're installing or upgrading from a non-RBAC version, set the 'admin' password
+      if @options[:type] == :upgrade && @options[:HOSTS][dashboard.name][:pe_ver] < "3.4.0"
+        dashboard_password = "'#{options[:answers][:q_puppet_enterpriseconsole_auth_password]}'"
+        the_answers[dashboard.name][:q_puppet_enterpriseconsole_auth_password] = dashboard_password
+      end
+
+      the_answers[dashboard.name].merge!(console_services_hash)
+      the_answers[database.name].merge!(console_services_hash)
+
+      return the_answers
     end
   end
 end

data/lib/beaker/cli.rb

@@ -43,6 +43,9 @@ module Beaker
       @options[:helper].each do |helper|
         require File.expand_path(helper)
       end
+
+      @options[:log_dated_dir] = Beaker::Logger.generate_dated_log_folder(@options[:log_dir], @timestamp)
+      @options[:xml_dated_dir] = Beaker::Logger.generate_dated_log_folder(@options[:xml_dir], @timestamp)
     end
 
     #Provision, validate and configure all hosts as defined in the hosts file
@@ -51,6 +54,7 @@ module Beaker
         @hosts =  []
         @network_manager = Beaker::NetworkManager.new(@options, @logger)
         @hosts = @network_manager.provision
+        @network_manager.proxy_package_manager
         @network_manager.validate
         @network_manager.configure
       rescue => e

data/lib/beaker/command.rb

@@ -5,24 +5,6 @@ module Beaker
   # @api public
   class Command
 
-    DEFAULT_GIT_RUBYLIB = {
-      :default => [],
-      :host => %w(hieralibdir hierapuppetlibdir
-                  pluginlibpath puppetlibdir
-                  facterlibdir),
-      :opts => { :additive => true,
-                 :separator => {:host => 'pathseparator' }
-      }
-    }
-
-    DEFAULT_GIT_PATH = {
-      :default => [],
-      :host => %w(puppetbindir facterbindir hierabindir),
-      :opts => { :additive => true, :separator => ':' }
-    }
-
-    DEFAULT_GIT_ENV = { :PATH => DEFAULT_GIT_PATH, :RUBYLIB => DEFAULT_GIT_RUBYLIB }
-
     # A string representing the (possibly) incomplete command
     attr_accessor :command
 
@@ -67,6 +49,7 @@ module Beaker
       @options = options
       @args    = args
       @environment = {}
+      @cmdexe = @options.delete(:cmdexe) || false
 
       # this is deprecated and will not allow you to use a command line
       # option of `--environment`, please use ENV instead.
@@ -87,8 +70,10 @@ module Beaker
     def cmd_line host, cmd = @command, env = @environment
       env_string = env.nil? ? '' : environment_string_for( host, env )
 
+      cygwin = ((host['platform'] =~ /windows/) and @cmdexe) ? 'cmd.exe /c' : nil
+
       # This will cause things like `puppet -t -v agent` which is maybe bad.
-      "#{env_string} #{cmd} #{options_string} #{args_string}"
+      [env_string, cygwin, cmd, options_string, args_string].compact.reject(&:empty?).join(' ')
     end
 
     # @param [Hash] opts These are the options that the command takes
@@ -133,7 +118,7 @@ module Beaker
       args.flatten.compact.join(' ')
     end
 
-    # Determine the appropriate env commands for the given host.
+    # Construct the environment string for this command
     #
     # @param [Host]                 host  A Host object
     # @param [Hash{String=>String}] env   An optional Hash containing
@@ -152,75 +137,22 @@ module Beaker
     #       given a generic Command.
     def environment_string_for host, env
       return '' if env.empty?
-
-      env_array = parse_env_hash_for( host, env ).compact
-
-      # cygwin-ism
-      cmd = host['platform'] =~ /windows/ ? 'cmd.exe /c' : nil
-      env_array << cmd if cmd
-
-      environment_string = env_array.join(' ')
-
-      "env #{environment_string}"
-    end
-
-    # @!visibility private
-    def parse_env_hash_for( host, env = @environment )
-      # I needlessly love inject
-      env.inject([]) do |array_of_parsed_vars, key_and_value|
-        variable, val_in_unknown_format = *key_and_value
-        if val_in_unknown_format.is_a?(Hash)
-          value = val_in_unknown_format
-        elsif val_in_unknown_format.is_a?(Array)
-          value = { :default => val_in_unknown_format }
+      env_array = []
+      env.each_key do |key|
+        val = env[key]
+        if val.is_a?(Array)
+          val = val.join(':')
         else
-          value = { :default => [ val_in_unknown_format.to_s ] }
+          val = val.to_s
         end
-
-        var_settings = ensure_correct_structure_for( value )
-        # any default array of variable values ( like [ '/bin', '/usr/bin' ] for PATH )
-        default_values = var_settings[:default]
-
-        # host specific values, ie :host => [ 'puppetlibdir' ] is evaluated to
-        # an array with whatever host['puppetlibdir'] is
-        host_values = var_settings[:host].map { |attr| host[attr] }
-
-        # the two arrays are combined with host specific values first
-        var_array = ( host_values + default_values ).compact
-
-        # This will add the name of the variable, so :PATH => { ... }
-        # gets '${PATH}' appended to it if the :additive opt is passed
-        var_array << "${#{variable}}" if var_settings[:opts][:additive]
-
-        # This is stupid, but because we're using cygwin we sometimes need to use
-        # ':' and sometimes ';' on windows as a separator
-        attr_string = join_env_vars_for( var_array, host, var_settings[:opts][:separator] )
-        var_string = attr_string.empty? ? nil : %Q[#{variable}="#{attr_string}"]
-
-        # Now we append this to our accumulator array ie [ 'RUBYLIB=....', 'PATH=....' ]
-        array_of_parsed_vars << var_string
-
-        array_of_parsed_vars
+        env_array << "#{key.to_s.upcase}=\"#{val}\""
       end
-    end
 
-    # @!visibility private
-    def ensure_correct_structure_for( settings )
-      structure = { :default => [],
-        :host => [],
-        :opts => {}
-      }.merge( settings )
-      structure[:opts][:separator] ||= ':'
-      structure
-    end
+      environment_string = env_array.join(' ')
 
-    # @!visibility private
-    def join_env_vars_for( array_of_variables, host, separator = ':' )
-      if separator.is_a?( Hash )
-        separator = host[separator[:host]]
-      end
-      array_of_variables.join( separator )
+      "env #{environment_string}"
     end
+
   end
 
   class PuppetCommand < Command
@@ -228,7 +160,7 @@ module Beaker
       command = "puppet #{args.shift}"
       opts = args.last.is_a?(Hash) ? args.pop : Hash.new
       opts['ENV'] ||= Hash.new
-      opts['ENV'] = opts['ENV'].merge( DEFAULT_GIT_ENV )
+      opts[:cmdexe] = true
       super( command, args, opts )
     end
   end

data/lib/beaker/command_factory.rb

@@ -1,8 +1,19 @@
-require 'test/unit/assertions'
+require 'minitest/test'
 
 module Beaker
   module CommandFactory
-    include Test::Unit::Assertions
+    include Minitest::Assertions
+    #Why do we need this accessor?
+    # https://github.com/seattlerb/minitest/blob/master/lib/minitest/assertions.rb#L8-L12
+    # Protocol: Nearly everything here boils up to +assert+, which
+    # expects to be able to increment an instance accessor named
+    # +assertions+. This is not provided by Assertions and must be
+    # provided by the thing including Assertions. See Minitest::Runnable
+    # for an example.
+    attr_accessor :assertions
+    def assertions
+      @assertions || 0
+    end
 
     def execute(command, options={}, &block)
       result = self.exec(Command.new(command), options)

data/lib/beaker/dsl/assertions.rb

@@ -1,4 +1,4 @@
-require 'test/unit/assertions'
+require 'minitest/test'
 
 module Beaker
   module DSL
@@ -15,7 +15,19 @@ module Beaker
     # or direct object for asserting against into your assertion.
     #
     module Assertions
-      include Test::Unit::Assertions
+      include Minitest::Assertions
+
+      #Why do we need this accessor?
+      # https://github.com/seattlerb/minitest/blob/master/lib/minitest/assertions.rb#L8-L12
+      # Protocol: Nearly everything here boils up to +assert+, which
+      # expects to be able to increment an instance accessor named
+      # +assertions+. This is not provided by Assertions and must be
+      # provided by the thing including Assertions. See Minitest::Runnable
+      # for an example.
+      attr_accessor :assertions
+      def assertions
+        @assertions || 0
+      end
 
       # Make assertions about the content of console output.
       #
@@ -82,6 +94,17 @@ module Beaker
         assert_equal our_err,    (result.nil? ? '' : result.stderr),
           'The contents of STDERR did not match expectations'
       end
+
+      # Assert that the provided string does not match the provided regular expression, can pass optional message
+      # @deprecated This is placed her for backwards compatability for tests that used Test::Unit::Assertions,
+      #             http://apidock.com/ruby/Test/Unit/Assertions/assert_no_match
+      #             
+      def assert_no_match(regexp, string, msg=nil)
+        assert_instance_of(Regexp, regexp, "The first argument to assert_no_match should be a Regexp.")
+        msg = message(msg) { "<#{mu_pp(regexp)}> expected to not match\n<#{mu_pp(string)}>" }
+        assert(regexp !~ string, msg)
+      end
+
     end
   end
 end

data/lib/beaker/dsl/ezbake_utils.rb

@@ -106,7 +106,7 @@ module Beaker
             install_package host, package_name, package_version
           end
 
-        when /^(debian|ubuntu)$/
+        when /^(debian|ubuntu|cumulus)$/
           dependency_list = ezbake[:debian][:additional_dependencies]
           dependency_list.each do |dependency|
             package_name, _, package_version = dependency.split
@@ -192,7 +192,7 @@ module Beaker
           when /^(fedora|el|centos)$/
             env += "defaultsdir=/etc/sysconfig "
             on host, cd_to_package_dir + env + "make -e install-rpm-sysv-init"
-          when /^(debian|ubuntu)$/
+          when /^(debian|ubuntu|cumulus)$/
             env += "defaultsdir=/etc/default "
             on host, cd_to_package_dir + env + "make -e install-deb-sysv-init"
           else

data/lib/beaker/dsl/helpers.rb

@@ -187,6 +187,8 @@ module Beaker
       # @note If using {Beaker::Host} for the hosts *scp* is not
       #   required on the system as it uses Ruby's net/scp library.  The
       #   net-scp gem however is required (and specified in the gemspec.
+      #   When using SCP with Windows it will now auto expand path when
+      #   using `cygpath instead of failing or requiring full path
       #
       # @param [Host, #do_scp_to] host One or more hosts (or some object
       #                                that responds like
@@ -198,6 +200,10 @@ module Beaker
       # @return [Result] Returns the result of the SCP operation
       def scp_to host, from_path, to_path, opts = {}
         block_on host do | host |
+          if host['platform'] =~ /windows/ && to_path.match('`cygpath')
+            result = on host, "echo #{to_path}"
+            to_path = result.raw_output.chomp
+          end
           @result = host.do_scp_to(from_path, to_path, opts)
           @result.log logger
           @result
@@ -256,7 +262,7 @@ module Beaker
       # @return [String] Returns the name of the newly-created file.
       def create_tmpdir_for_user(host, name='/tmp/beaker', user=nil)
         if not user
-          result = on(host, "puppet master --configprint user")
+          result = on host, puppet("master --configprint user")
           if not result.exit_code == 0
             raise "`puppet master --configprint` failed, check that puppet is installed on #{host} or explicitly pass in a user name."
           end
@@ -674,7 +680,7 @@ module Beaker
       def dump_puppet_log(host)
         syslogfile = case host['platform']
           when /fedora|centos|el|redhat|scientific/ then '/var/log/messages'
-          when /ubuntu|debian/ then '/var/log/syslog'
+          when /ubuntu|debian|cumulus/ then '/var/log/syslog'
           else return
         end
 
@@ -861,18 +867,25 @@ module Beaker
       #                         path separator character. (The POSIX path separator
       #                         is ‘:’, and the Windows path separator is ‘;’.)
       #
+      # @option opts [String]   :debug (false) If this option exists,
+      #                         the "--debug" command line parameter
+      #                         will be passed to the 'puppet apply' command.
+      #
       # @param [Block] block This method will yield to a block of code passed
       #                      by the caller; this can be used for additional
       #                      validation, etc.
       #
       def apply_manifest_on(host, manifest, opts = {}, &block)
         block_on host do | host |
-
           on_options = {}
           on_options[:acceptable_exit_codes] = Array(opts[:acceptable_exit_codes])
 
           puppet_apply_opts = {}
-          puppet_apply_opts[:verbose] = nil
+          if opts[:debug]
+            puppet_apply_opts[:debug] = nil
+          else
+            puppet_apply_opts[:verbose] = nil
+          end
           puppet_apply_opts[:parseonly] = nil if opts[:parseonly]
           puppet_apply_opts[:trace] = nil if opts[:trace]
           puppet_apply_opts[:parser] = 'future' if opts[:future_parser]
@@ -1115,22 +1128,62 @@ module Beaker
                           host, "https://#{host.node_name}:8081", [35, 60])
       end
 
+      def sleep_until_puppetserver_started(host)
+        curl_with_retries("start puppetserver (ssl)",
+                          host, "https://#{host.node_name}:8140", [35, 60])
+      end
+
+      def sleep_until_nc_started(host)
+        curl_with_retries("start nodeclassifier (ssl)",
+                          host, "https://#{host.node_name}:4433", [35, 60])
+      end
+
       def curl_with_retries(desc, host, url, desired_exit_codes, max_retries = 60, retry_interval = 1)
-        retry_command(desc, host, "curl -m 1 #{url}", desired_exit_codes, max_retries, retry_interval)
+        opts = {
+          :desired_exit_codes => desired_exit_codes,
+          :max_retries => max_retries,
+          :retry_interval => retry_interval
+        }
+        retry_on(host, "curl -m 1 #{url}", opts)
       end
 
-      def retry_command(desc, host, command, desired_exit_codes = 0,
-                        max_retries = 60, retry_interval = 1, verbose = false)
+      # This command will execute repeatedly until success or it runs out with an error
+      #
+      # @param [Host, Array<Host>, String, Symbol] host    One or more hosts to act upon,
+      #                            or a role (String or Symbol) that identifies one or more hosts.
+      # @param [String, Command]   command The command to execute on *host*.
+      # @param [Hash{Symbol=>String}] opts Options to alter execution.
+      # @param [Proc]              block   Additional actions or assertions.
+      #
+      # @option opts [Array<Fixnum>, Fixnum] :desired_exit_codes (0) An array
+      #   or integer exit code(s) that should be considered
+      #   acceptable.  An error will be thrown if the exit code never
+      #   matches one of the values in this list.
+      # @option opts [Fixnum] :max_retries (60) number of times the
+      #   command will be tried before failing
+      # @option opts [Float] :retry_interval (1) number of seconds
+      #   that we'll wait between tries
+      # @option opts [Boolean] :verbose (false)
+      def retry_on(host, command, opts = {}, &block)
+        option_exit_codes     = opts[:desired_exit_codes]
+        option_max_retries    = opts[:max_retries].to_i
+        option_retry_interval = opts[:retry_interval].to_f
+        desired_exit_codes    = option_exit_codes ? [option_exit_codes].flatten : [0]
+        desired_exit_codes    = [0] if desired_exit_codes.empty?
+        max_retries           = option_max_retries == 0 ? 60 : option_max_retries  # nil & "" both return 0
+        retry_interval        = option_retry_interval == 0 ? 1 : option_retry_interval
+        verbose               = true.to_s == opts[:verbose]
+
         log_prefix = host.log_prefix
         logger.debug "\n#{log_prefix} #{Time.new.strftime('%H:%M:%S')}$ #{command}"
         logger.debug "  Trying command #{max_retries} times."
         logger.debug ".", add_newline=false
-        desired_exit_codes = [desired_exit_codes].flatten
-        result = on host, command, {:acceptable_exit_codes => (0...127), :silent => !verbose}
+
+        result = on host, command, {:acceptable_exit_codes => (0...127), :silent => !verbose}, &block
         num_retries = 0
         until desired_exit_codes.include?(result.exit_code)
           sleep retry_interval
-          result = on host, command, {:acceptable_exit_codes => (0...127), :silent => !verbose}
+          result = on host, command, {:acceptable_exit_codes => (0...127), :silent => !verbose}, &block
           num_retries += 1
           logger.debug ".", add_newline=false
           if (num_retries > max_retries)
@@ -1139,6 +1192,7 @@ module Beaker
           end
         end
         logger.debug "\n#{log_prefix} #{Time.new.strftime('%H:%M:%S')}$ #{command} ostensibly successful."
+        result
       end
 
       #Is semver-ish version a less than semver-ish version b
@@ -1219,7 +1273,7 @@ module Beaker
       #wait for a given host to appear in the dashboard
       def wait_for_host_in_dashboard(host)
         hostname = host.node_name
-        retry_command("Wait for #{hostname} to be in the console", dashboard, "! curl --tlsv1 -k -I https://#{dashboard}/nodes/#{hostname} | grep '404 Not Found'")
+        retry_on(dashboard, "! curl --tlsv1 -k -I https://#{dashboard}/nodes/#{hostname} | grep '404 Not Found'")
       end
 
       # Ensure the host has requested a cert, then sign it
@@ -1245,7 +1299,7 @@ module Beaker
             (0..10).each do |i|
               fail_test("Failed to sign cert for #{hostname}") if i == 10
 
-              on master, puppet("cert --sign --all"), :acceptable_exit_codes => [0,24]
+              on master, puppet("cert --sign --all --allow-dns-alt-names"), :acceptable_exit_codes => [0,24]
               break if on(master, puppet("cert --list --all")).stdout =~ /\+ "?#{hostname}"?/
               sleep next_sleep
               (last_sleep, next_sleep) = next_sleep, last_sleep+next_sleep