beaker 0.0.0 → 1.0.0

data/lib/beaker/options/hosts_file_parser.rb

@@ -0,0 +1,39 @@
+module Beaker
+  module Options
+    #A set of functions to parse hosts files
+    module HostsFileParser
+
+      # Read the contents of the hosts.cfg into an OptionsHash, merge the 'CONFIG' section into the OptionsHash, return OptionsHash
+      # @param [String] hosts_file_path The path to the hosts file
+      #
+      # @example 
+      #     hosts_hash = HostsFileParser.parse_hosts_file('sample.cfg')
+      #     hosts_hash == {:HOSTS=>{:"pe-ubuntu-lucid"=>{:roles=>["agent", "dashboard", "database", "master"], ... }
+      #
+      # @return [OptionsHash] The contents of the hosts file as an OptionsHash
+      # @raise [ArgumentError] Raises if hosts_file_path is not a path to a file, or is not a valid YAML file
+      def self.parse_hosts_file(hosts_file_path)
+        hosts_file_path = File.expand_path(hosts_file_path)
+        unless File.exists?(hosts_file_path)
+          raise ArgumentError, "Required host file '#{hosts_file_path}' does not exist!"
+        end
+        host_options = Beaker::Options::OptionsHash.new
+        begin
+          host_options = host_options.merge(YAML.load_file(hosts_file_path))
+        rescue Psych::SyntaxError => e
+          raise ArgumentError, "#{hosts_file_path} is not a valid YAML file\n\t#{e}"
+        end
+
+        # Make sure the roles array is present for all hosts
+        host_options['HOSTS'].each_key do |host|
+          host_options['HOSTS'][host]['roles'] ||= []
+        end
+        if host_options.has_key?('CONFIG')
+          host_options = host_options.merge(host_options.delete('CONFIG'))
+        end
+        host_options
+      end
+
+    end
+  end
+end

data/lib/beaker/options/options_file_parser.rb

@@ -0,0 +1,45 @@
+require 'open-uri'
+module Beaker
+  module Options
+    #A set of functions to read options files
+    module OptionsFileParser
+
+      # Eval the contents of options_file_path, return as an OptionsHash
+      #
+      # Options file is assumed to contain extra options stored in a Hash
+      #
+      # ie, 
+      #   {
+      #     :debug => true,
+      #     :tests => "test.rb",
+      #   }
+      #   
+      # @param [String] options_file_path The path to the options file
+      #
+      # @example 
+      #     options_hash = OptionsFileParser.parse_options_file('sample.cfg')
+      #     options_hash == {:debug=>true, :tests=>"test.rb", :pre_suite=>["pre-suite.rb"], :post_suite=>"post_suite1.rb,post_suite2.rb"}
+      #
+      # @return [OptionsHash] The contents of the options file as an OptionsHash
+      # @raise [ArgumentError] Raises if options_file_path is not a path to a file
+      # @note Since the options_file is Eval'ed, any other Ruby commands will also be executed, this can be used
+      #    to set additional environment variables 
+      def self.parse_options_file(options_file_path)
+        result = Beaker::Options::OptionsHash.new
+        if options_file_path
+          options_file_path = File.expand_path(options_file_path)
+          unless File.exists?(options_file_path)
+            raise ArgumentError, "Specified options file '#{options_file_path}' does not exist!"
+          end
+          # This eval will allow the specified options file to have access to our
+          #  scope.  It is important that the variable 'options_file_path' is
+          #  accessible, because some existing options files (e.g. puppetdb) rely on
+          #  that variable to determine their own location (for use in 'require's, etc.)
+          result = result.merge(eval(File.read(options_file_path)))
+        end
+        result
+      end
+
+    end
+  end
+end

data/lib/beaker/options/options_hash.rb

@@ -0,0 +1,294 @@
+module Beaker
+  module Options
+
+    # A hash that treats Symbol and String keys interchangeably
+    # and recursively merges hashes
+    class OptionsHash < Hash
+
+      # The dividor between elements when OptionsHash is dumped
+      DIV = '    '
+
+      # The end of line when dumping
+      EOL = "\n"
+
+      # Get value for given key, search for both k as String and k as Symbol,
+      # if not present return nil
+      #
+      # @param [Object] k The key to find, searches for both k as String
+      #                   and k as Symbol
+      #
+      # @example Use this method to return the value for a given key
+      #     a['key'] = 'value'
+      #     a['key'] == a[:key] == 'value'
+      #
+      # @return [nil, Object] Return the Object found at given key,
+      #                       or nil if no Object found
+      def [] k
+        super(k.to_s) || super(k.to_sym)
+      end
+
+      # Set Symbol key to Object value
+      # @param [Object] k The key to associated with the value,
+      #                   converted to Symbol key
+      # @param [Object] v The value to store in the ObjectHash
+      #
+      # @example Use this method to set the value for a key
+      #     a['key'] = 'value'
+      #
+      # @return [Object] Return the Object value just stored
+      def []=k,v
+        super(k.to_sym, v)
+      end
+
+      # Determine if type of ObjectHash is pe, defaults to true
+      #
+      # @example Use this method to return the value for a given key
+      #     a['type'] = 'pe'
+      #     a.is_pe? == true
+      #
+      # @return [Boolean]
+      def is_pe?
+        self[:type] ? self[:type] =~ /pe/ : true
+      end
+
+      # Determine if key is stored in ObjectHash
+      # @param [Object] k The key to find in ObjectHash, searches for
+      #                   both k as String and k as Symbol
+      #
+      # @example Use this method to set the value for a key
+      #     a['key'] = 'value'
+      #     a.has_key[:key] == true
+      #
+      # @return [Boolean]
+      def has_key? k
+        super(k.to_s) || super(k.to_sym)
+      end
+
+      # Determine key=>value entry in OptionsHash, remove both value at
+      # String key and value at Symbol key
+      #
+      # @param [Object] k The key to delete in ObjectHash,
+      # deletes both k as String and k as Symbol
+      #
+      # @example Use this method to set the value for a key
+      #     a['key'] = 'value'
+      #     a.delete[:key] == 'value'
+      #
+      # @return [Object, nil] The Object deleted at value,
+      # nil if no Object deleted
+      def delete k
+        super(k.to_s) || super(k.to_sym)
+      end
+
+      # Recursively merge and OptionsHash with an OptionsHash or Hash
+      #
+      # @param [OptionsHash]       base The hash to merge into
+      # @param [OptionsHash, Hash] hash The hash to merge from
+      #
+      # @example
+      #   base = { :key => { :subkey1 => 'subval', :subkey2 => 'subval' } }
+      #   hash = { :key => { :subkey1 => 'newval'} }
+      #
+      #   rmerge(base, hash)
+      #   #=>  {:key =>
+      #           {:subkey1 => 'newval',
+      #            :subkey2 => 'subval'}}
+      #
+      # @return [OptionsHash] The combined bash and hash
+      def rmerge base, hash
+        return base unless hash.is_a?(Hash) || hash.is_a?(OptionsHash)
+        hash.each do |key, v|
+          if (base[key].is_a?(Hash) || base[key].is_a?(OptionsHash)) && (hash[key].is_a?(Hash) || has[key].is_a?(OptionsHash))
+            rmerge(base[key], hash[key])
+          elsif hash[key].is_a?(Hash)
+            base[key] = OptionsHash.new.merge(hash[key])
+          else
+            base[key]= hash[key]
+          end
+        end
+        base
+      end
+
+      # Recursively merge self with an OptionsHash or Hash
+      #
+      # @param [OptionsHash, Hash] hash The hash to merge from
+      #
+      # @example
+      #   base = { :key => { :subkey1 => 'subval', :subkey2 => 'subval' } }
+      #   hash = { :key => { :subkey1 => 'newval'} }
+      #
+      #   base.merge(hash)
+      #   #=> {:key =>
+      #         {:subkey1 => 'newval',
+      #          :subkey2 => 'subval' }
+      #
+      # @return [OptionsHash] The combined hash
+      def merge hash
+        rmerge(self, hash)
+      end
+
+      # Helper for formatting collections
+      # Computes the indentation level for elements of the collection
+      # Yields indentation to block to so the caller can create
+      # map of element strings
+      # Places delimiters in the correct location
+      # Joins everything with correct EOL
+      #
+      #
+      # !@visibility private
+      def as_coll( opening, closing, in_lvl, in_inc, &block )
+        delim_indent = in_inc * in_lvl
+        elem_indent  = in_inc * (in_lvl + 1)
+
+        open_brace  = opening
+        close_brace = delim_indent + closing
+
+        fmtd_coll = block.call( elem_indent )
+        str_coll = fmtd_coll.join( ',' + EOL )
+
+        return open_brace + EOL + str_coll + EOL + close_brace
+      end
+
+      # Pretty prints a collection
+      #
+      # @param [Enumerable] collection The collection to be printed
+      # @param [Integer]    in_lvl     The level of indentation
+      # @param [String]     in_inc     The increment to indent
+      #
+      # @example
+      #   base = {:key => { :subkey1 => 'subval', :subkey2 => ['subval'] }}
+      #   self.fmt_collection( base )
+      #   #=> '{
+      #            "key": {
+      #                "subkey": "subval",
+      #                "subkey2": [
+      #                    "subval"
+      #                ]
+      #            }
+      #        }'
+      #
+      # @return [String] The collection as a pretty JSON object
+      def fmt_collection( collection, in_lvl = 0, in_inc = DIV )
+        if collection.respond_to? :each_pair
+          string = fmt_assoc( collection, in_lvl, in_inc )
+        else
+          string = fmt_list( collection, in_lvl, in_inc )
+        end
+
+        return string
+      end
+
+      # Pretty prints an associative collection
+      #
+      # @param [#each_pair] coll    The collection to be printed
+      # @param [Integer]    in_lvl  The level of indentation
+      # @param [String]     in_inc  The increment to indent
+      #
+      # @example
+      #   base = { :key => 'value', :key2 => 'value' }
+      #   self.fmt_assoc( base )
+      #   #=> '{
+      #            "key": "value",
+      #            "key2": "value"
+      #        }'
+      #
+      # @return [String] The collection as a pretty JSON object
+      def fmt_assoc( coll, in_lvl = 0, in_inc = DIV )
+        if coll.empty?
+          return '{}'
+        else
+          as_coll '{', '}', in_lvl, in_inc do |elem_indent|
+            coll.map do |key, value|
+              assoc_line = elem_indent + '"' + key.to_s + '"' + ': '
+              assoc_line += fmt_value( value, in_lvl, in_inc )
+            end
+          end
+        end
+      end
+
+      # Pretty prints a list collection
+      #
+      # @param [#each]    coll    The collection to be printed
+      # @param [Integer]  in_lvl  The level of indentation
+      # @param [String]   in_inc  The increment to indent
+      #
+      # @example
+      #   base = [ 'first', 'second' ]
+      #   self.fmt_list( base )
+      #   #=> '[
+      #            "first",
+      #            "second"
+      #        ]'
+      #
+      # @return [String] The collection as a pretty JSON object
+      def fmt_list( coll, in_lvl = 0, in_inc = DIV )
+        if coll.empty?
+          return '[]'
+        else
+          as_coll '[', ']', in_lvl, in_inc do |indent|
+            coll.map do |el|
+              indent + fmt_value( el, in_lvl, in_inc )
+            end
+          end
+        end
+      end
+
+      # Chooses between collection and primitive formatting
+      #
+      # !@visibility private
+      def fmt_value( value, in_lvl = 0, in_inc = DIV )
+        if value.kind_of? Enumerable and not value.is_a? String
+          fmt_collection( value, in_lvl + 1, in_inc )
+        else
+          fmt_basic( value )
+        end
+      end
+
+      # Pretty prints primitive JSON values
+      #
+      # @param [Object] value The collection to be printed
+      #
+      # @example
+      #   self.fmt_value( 4 )
+      #   #=> '4'
+      #
+      # @example
+      #   self.fmt_value( true )
+      #   #=> 'true'
+      #
+      # @example
+      #   self.fmt_value( nil )
+      #   #=> 'null'
+      #
+      # @example
+      #   self.fmt_value( 'string' )
+      #   #=> '"string"'
+      #
+      # @return [String] The value as a valid JSON primitive
+      def fmt_basic( value )
+        case value
+        when Numeric, TrueClass, FalseClass then value.to_s
+        when NilClass then "null"
+        else "\"#{value}\""
+        end
+      end
+
+      # Pretty print the options as JSON
+      #
+      # @example
+      #   base = { :key => { :subkey1 => 'subval', :subkey2 => 'subval' } }
+      #   base.dump
+      #   #=>  '{
+      #             "key": {
+      #                 "subkey1": "subval",
+      #                 "subkey2": 2
+      #             }
+      #         }
+      #
+      # @return [String] The description of self
+      def dump
+        fmt_collection( self, 0, DIV )
+      end
+    end
+  end
+end

data/lib/beaker/options/parser.rb

@@ -0,0 +1,288 @@
+require 'yaml'
+
+module Beaker
+  module Options
+    #An Object that parses, merges and normalizes all supported Beaker options and arguments
+    class Parser
+      GITREPO      = 'git://github.com/puppetlabs'
+      #These options can have the form of arg1,arg2 or [arg] or just arg, 
+      #should default to []
+      LONG_OPTS    = [:helper, :load_path, :tests, :pre_suite, :post_suite, :install, :modules]
+      #These options expand out into an array of .rb files
+      RB_FILE_OPTS = [:tests, :pre_suite, :post_suite]
+
+      PARSE_ERROR = if RUBY_VERSION > '1.8.7'; then Psych::SyntaxError; else ArgumentError; end
+
+      #The OptionsHash of all parsed options
+      attr_accessor :options
+
+      # Raises an ArgumentError with associated message
+      # @param [String] msg The error message to be reported
+      # @raise [ArgumentError] Takes the supplied message and raises it as an ArgumentError
+      def parser_error msg = ""
+        raise ArgumentError, msg.to_s
+      end
+
+      # Returns the git repository used for git installations
+      # @return [String] The git repository
+      def repo
+        GITREPO
+      end
+
+      # Returns a description of Beaker's supported arguments
+      # @return [String] The usage String
+      def usage
+       @command_line_parser.usage
+      end
+
+      # Normalizes argument into an Array.  Argument can either be converted into an array of a single value,
+      # or can become an array of multiple values by splitting arg over ','.  If argument is already an
+      # array that array is returned untouched.
+      # @example
+      #   split_arg([1, 2, 3]) == [1, 2, 3] 
+      #   split_arg(1) == [1]
+      #   split_arg("1,2") == ["1", "2"]
+      #   split_arg(nil) == []
+      # @param [Array, String] arg Either an array or a string to be split into an array
+      # @return [Array] An array of the form arg, [arg], or arg.split(',')
+      def split_arg arg
+        arry = []
+        if arg.is_a?(Array)
+          arry += arg
+        elsif arg =~ /,/
+          arry += arg.split(',')
+        else
+          arry << arg
+        end
+        arry
+      end
+
+      # Generates a list of files based upon a given path or list of paths.
+      #
+      # Looks recursively for .rb files in paths.
+      #
+      # @param [Array] paths Array of file paths to search for .rb files
+      # @return [Array] An Array of fully qualified paths to .rb files
+      # @raise [ArgumentError] Raises if no .rb files are found in searched directory or if
+      #                         no .rb files are found overall
+      def file_list(paths)
+        files = []
+        if not paths.empty?
+          paths.each do |root|
+            if File.file? root then
+              files << root
+            else
+              discover_files = Dir.glob(
+                File.join(root, "**/*.rb")
+              ).select { |f| File.file?(f) }
+              if discover_files.empty?
+                parser_error "empty directory used as an option (#{root})!"
+              end
+              files += discover_files
+            end
+          end
+        end
+        if files.empty?
+          parser_error "no .rb files found in #{paths.to_s}"
+        end
+        files
+      end
+
+      #Converts array of paths into array of fully qualified git repo URLS with expanded keywords
+      #
+      #Supports the following keywords
+      #  PUPPET 
+      #  FACTER
+      #  HIERA 
+      #  HIERA-PUPPET
+      #@example
+      #  opts = ["PUPPET/3.1"]
+      #  parse_git_repos(opts) == ["#{GITREPO}/puppet.git#3.1"]
+      #@param [Array] git_opts An array of paths
+      #@return [Array] An array of fully qualified git repo URLs with expanded keywords
+      def parse_git_repos(git_opts)
+        git_opts.map! { |opt|
+          case opt
+            when /^PUPPET\//
+              opt = "#{GITREPO}/puppet.git##{opt.split('/', 2)[1]}"
+            when /^FACTER\//
+              opt = "#{GITREPO}/facter.git##{opt.split('/', 2)[1]}"
+            when /^HIERA\//
+              opt = "#{GITREPO}/hiera.git##{opt.split('/', 2)[1]}"
+            when /^HIERA-PUPPET\//
+              opt = "#{GITREPO}/hiera-puppet.git##{opt.split('/', 2)[1]}"
+          end
+          opt
+        }
+        git_opts
+      end
+
+      #Constructor for Parser
+      #
+      def initialize
+         @command_line_parser = Beaker::Options::CommandLineParser.new
+      end
+
+      # Parses ARGV or provided arguments array, file options, hosts options and combines with environment variables and
+      # preset defaults to generate a Hash representing the Beaker options for a given test run
+      #
+      # Order of priority is as follows:
+      #   1.  environment variables are given top priority
+      #   2.  host file options
+      #   3.  the 'CONFIG' section of the hosts file
+      #   4.  ARGV or provided arguments array
+      #   5.  options file values
+      #   6.  default or preset values are given the lowest priority
+      #
+      # @param [Array] args ARGV or a provided arguments array
+      # @raise [ArgumentError] Raises error on bad input
+      def parse_args(args = ARGV)
+        #NOTE on argument precedence:
+        #
+        # Will use env, then hosts/config file, then command line, then file options
+        # 
+        @options = Beaker::Options::Presets.presets
+        cmd_line_options = @command_line_parser.parse!(args)
+        file_options = Beaker::Options::OptionsFileParser.parse_options_file(cmd_line_options[:options_file])
+        # merge together command line and file_options
+        #   overwrite file options with command line options
+        cmd_line_and_file_options = file_options.merge(cmd_line_options)
+        # merge command line and file options with defaults
+        #   overwrite defaults with command line and file options 
+        @options = @options.merge(cmd_line_and_file_options)
+
+        if not @options[:help]
+          #read the hosts file that contains the node configuration and hypervisor info
+          hosts_options = Beaker::Options::HostsFileParser.parse_hosts_file(@options[:hosts_file])
+          # merge in host file vars
+          #   overwrite options (default, file options, command line, env) with host file options
+          @options = @options.merge(hosts_options)
+          # merge in env vars
+          #   overwrite options (default, file options, command line, hosts file) with env
+          env_vars = Beaker::Options::Presets.env_vars
+          @options = @options.merge(env_vars)
+
+          normalize_args
+        end
+
+        @options
+      end
+
+      # Determine is a given file exists and is a valid YAML file
+      # @param [String] f The YAML file path to examine
+      # @param [String] msg An options message to report in case of error
+      # @raise [ArgumentError] Raise if file does not exist or is not valid YAML
+      def check_yaml_file(f, msg = "")
+        if not File.file?(f)
+          parser_error "#{f} does not exist (#{msg})"
+        end
+        begin
+          YAML.load_file(f)
+        rescue PARSE_ERROR => e
+          parser_error "#{f} is not a valid YAML file (#{msg})\n\t#{e}"
+        end
+      end
+
+      #Validate all merged options values for correctness
+      #
+      #Currently checks:
+      #  - if a keyfile is provided then use it
+      #  - paths provided to --test, --pre-suite, --post-suite provided lists of .rb files for testing
+      #  - --type is one of 'pe' or 'git'
+      #  - --fail-mode is one of 'fast', 'stop' or nil
+      #  - if using blimpy hypervisor an EC2 YAML file exists
+      #  - if using the aix, solaris, or vcloud hypervisors a .fog file exists
+      #  - that one and only one master is defined per set of hosts
+      #  - that solaris/windows/aix hosts are agent only for PE tests OR
+      #  - that windows/aix host are agent only if type is not 'pe'
+      #
+      #@raise [ArgumentError] Raise if argument/options values are invalid
+      def normalize_args
+
+        #use the keyfile if present
+        if @options.has_key?(:keyfile)
+          @options[:ssh][:keys] = [@options[:keyfile]]
+        end
+
+        #split out arguments - these arguments can have the form of arg1,arg2 or [arg] or just arg
+        #will end up being normalized into an array
+        LONG_OPTS.each do |opt|
+          if @options.has_key?(opt)
+            @options[opt] = split_arg(@options[opt])
+            if RB_FILE_OPTS.include?(opt)
+              @options[opt] = file_list(@options[opt])
+            end
+            if opt == :install
+              @options[:install] = parse_git_repos(@options[:install])
+            end
+          else
+            @options[opt] = []
+          end
+        end 
+ 
+        #check for valid type
+        if @options[:type] !~ /(pe)|(git)/
+          parser_error "--type must be one of pe or git, not '#{@options[:type]}'"
+        end
+
+        #check for valid fail mode
+        if not ["fast", "stop", nil].include?(@options[:fail_mode])
+          parser_error "--fail-mode must be one of fast, stop" 
+        end
+
+        #check for config files necessary for different hypervisors
+        hypervisors = [] 
+        @options[:HOSTS].each_key do |name|  
+          hypervisors << @options[:HOSTS][name][:hypervisor].to_s
+        end
+        hypervisors.uniq!
+        hypervisors.each do |visor|
+          if ['blimpy'].include?(visor)
+            check_yaml_file(@options[:ec2_yaml], "required by #{visor}")
+          end
+          if ['aix', 'solaris', 'vcloud'].include?(visor)
+            check_yaml_file(@options[:dot_fog], "required by #{visor}")
+          end
+        end
+
+        #check that roles of hosts make sense
+        # - must be one and only one master
+        roles = []
+        @options[:HOSTS].each_key do |name|  
+          roles << @options[:HOSTS][name][:roles]
+        end
+        master = 0
+        roles.each do |role_array|
+          if role_array.include?('master')
+            master += 1
+          end
+        end
+        if master > 1 or master < 1
+          parser_error "One and only one host/node may have the role 'master', fix #{@options[:hosts_file]}"
+        end
+
+        #check that solaris/windows/el-4 boxes are only agents
+        @options[:HOSTS].each_key do |name|
+          host = @options[:HOSTS][name]
+          if (host[:platform] =~ /windows|el-4/) ||
+             (@options.is_pe? && host[:platform] =~ /solaris/)
+
+            test_host_roles(name, host)
+          end
+        end
+
+      end
+
+      private
+
+      # @api private
+      def test_host_roles(host_name, host_hash)
+        host_roles = host_hash[:roles]
+        if (host_roles - ['agent']).size != 0
+          parser_error "#{host_hash[:platform].to_s} box '#{host_name}' can only have role 'agent', has roles #{host_roles.to_s}"
+        end
+      end
+
+    end
+  end
+end