beaker-hostgenerator 0.5.0 → 0.6.0

data/lib/beaker-hostgenerator/generator.rb

@@ -1,43 +1,35 @@
-require 'beaker-hostgenerator/util'
 require 'beaker-hostgenerator/data'
-require 'beaker-hostgenerator/exceptions'
 require 'beaker-hostgenerator/roles'
+require 'beaker-hostgenerator/hypervisor'
+require 'beaker-hostgenerator/parser'
 
 require 'yaml'
 
 module BeakerHostGenerator
   class Generator
     include BeakerHostGenerator::Data
-    include BeakerHostGenerator::Exceptions
-    include BeakerHostGenerator::Utils
-
-    attr_reader :options
-
-    def initialize options
-      @options = options
-    end
-
-    def self.create( options )
-      hypervisor_type = options[:hypervisor]
-
-      hclass = case hypervisor_type
-               when /vmpooler/
-                 BeakerHostGenerator::Vmpooler
-               when /vagrant/
-                 BeakerHostGenerator::Vagrant
-               else
-                 raise "Invalid hypervisor #{type}"
-               end
-
-      return hclass.new(options)
-    end
-
-    def generate tokens
+    include BeakerHostGenerator::Parser
+    include BeakerHostGenerator::Roles
+
+    # Main host generation entry point, returns a YAML map as a string for the
+    # given host specification and optional configuration.
+    #
+    # @param layout [String] The raw hosts specification user input.
+    #                        For example `"centos6-64m-redhat7-64a"`.
+    # @param options [Hash] Global, optional configuration such as the default
+    #                       hypervisor or OS info version.
+    #
+    # @returns [String] A complete YAML map as a string defining the HOSTS and
+    #                   CONFIG sections as required by Beaker.
+    def generate(layout, options)
+      tokens = tokenize_layout(layout)
+      config = {}.deep_merge(BASE_CONFIG)
       nodeid = Hash.new(1)
       ostype = nil
+      bhg_version = options[:osinfo_version] || 0
 
       tokens.each do |token|
-        if is_ostype_token?(token, @options[:osinfo_version])
+        if is_ostype_token?(token, bhg_version)
           if nodeid[ostype] == 1 and ostype != nil
             raise "Error: no nodes generated for #{ostype}"
           end
@@ -45,85 +37,45 @@ module BeakerHostGenerator
           next
         end
 
-        node_info = __parse_node_info_token(token)
+        node_info = parse_node_info_token(token)
 
+        # Build node host name
+        platform = "#{ostype}-#{node_info['bits']}"
+        host_name = "#{platform}-#{nodeid[ostype]}"
+
+        node_info['platform'] = platform
         node_info['ostype'] = ostype
         node_info['nodeid'] = nodeid[ostype]
 
-        config = {
-          'pe_dir' => pe_dir(pe_version, pe_family),
-          'pe_ver' => pe_version,
-          'pe_upgrade_dir' => pe_dir(pe_upgrade_version, pe_upgrade_family),
-          'pe_upgrade_ver' => pe_upgrade_version,
-        }
+        host_config = base_host_config(options)
 
-        [:pe_dir, :pe_ver, :pe_upgrade_dir, :pe_upgrade_ver].each do |option|
-          if @options[option]
-            config[option.to_s] = @options[option]
-          end
-        end
+        # Delegate to the hypervisor
+        hypervisor = BeakerHostGenerator::Hypervisor.create(node_info, options)
+        host_config = hypervisor.generate_node(node_info, host_config, bhg_version)
+        config['CONFIG'].deep_merge!(hypervisor.global_config())
 
-        host_name, host_config = generate_node(
-          node_info, config, bhg_version=@options[:osinfo_version])
+        # Merge in any arbitrary key-value host settings. Treat the 'hostname'
+        # setting specially, and don't merge it in as an arbitrary setting.
+        arbitrary_settings = node_info['host_settings']
+        host_name = arbitrary_settings.delete('hostname') if
+          arbitrary_settings.has_key?('hostname')
+        host_config.merge!(arbitrary_settings)
 
         if PE_USE_WIN32 && ostype =~ /windows/ && node_info['bits'] == "64"
           host_config['ruby_arch'] = 'x86'
           host_config['install_32'] = true
         end
 
-        if not @options[:disable_default_role]
-          host_config['roles'] = ['agent']
-        else
-          host_config['roles'] = []
-        end
-
-        host_config['roles'].concat __generate_host_roles(node_info)
-        host_config['roles'].uniq!
+        generate_host_roles!(host_config, node_info, options)
 
-        if not @options[:disable_role_config]
-          host_config['roles'].each do |role|
-            host_config.deep_merge! __get_role_config(role)
-          end
-        end
-
-        @config['HOSTS'][host_name] = host_config
+        config['HOSTS'][host_name] = host_config
         nodeid[ostype] += 1
       end
 
-      return @config.to_yaml
-    end
-
-    def __get_role_config role
-      begin
-        r = BeakerHostGenerator::Roles.new
-        m = r.method(role)
-      rescue NameError
-        return {}
-      end
-
-      return m.call
-    end
-
-    def __parse_node_info_token token
-      node_info = NODE_REGEX.match(token)
-
-      if node_info
-        node_info = Hash[ node_info.names.zip( node_info.captures ) ]
-      else
-        raise InvalidNodeSpecError.new, "Invalid node_info token: #{token}"
-      end
-
-      if node_info['arbitrary_roles']
-        node_info['arbitrary_roles'] = node_info['arbitrary_roles'].split(',') || ''
-      else
-        # Default to empty list to avoid having to check for nil elsewhere
-        node_info['arbitrary_roles'] = []
-      end
-
-      return node_info
+      return config.to_yaml
     end
 
-    def __generate_host_roles node_info
+    def get_host_roles(node_info)
       roles = []
 
       node_info['roles'].each_char do |c|
@@ -137,15 +89,23 @@ module BeakerHostGenerator
       return roles
     end
 
-    def is_ostype_token?
-      raise "Method 'is_ostype_token?' not implemented!"
-    end
+    private
 
-    def generate_node
-      raise "Method 'generate_node' not implemented!"
-    end
+    def generate_host_roles!(host_config, node_info, options)
+      if not options[:disable_default_role]
+        host_config['roles'] = ['agent']
+      else
+        host_config['roles'] = []
+      end
+
+      host_config['roles'].concat get_host_roles(node_info)
+      host_config['roles'].uniq!
 
+      if not options[:disable_role_config]
+        host_config['roles'].each do |role|
+          host_config.deep_merge! get_role_config(role)
+        end
+      end
+    end
   end
 end
-
-require 'beaker-hostgenerator/generator/vmpooler'

data/lib/beaker-hostgenerator/hypervisor.rb

@@ -0,0 +1,97 @@
+module BeakerHostGenerator
+
+  # Defines an Interface for the implementation of a hypervisor, and provides
+  # a static module function `create(node_info, options)` for instantiating
+  # the appropriate hypervisor implementation.
+  #
+  # New hypervisor implementations must define the methods in the Interface
+  # class, and add a new element to the `supported_hypervisors` map.
+  #
+  # Any number of hypervisors are used by a single Generator during host
+  # generation in the `BeakerHostGenerator::Generator#generate` method.
+  # Whenever a host specifies a specific hypervisor implementation, the
+  # Generator will instantiate the appropriate hypervisor via
+  # `BeakerHostGenerator::Hypervisor.create`.
+  module Hypervisor
+
+    # Static factory method to instantiate the appropriate hypervisor for the
+    # given node. If no hypervisor is specified in the node info, then the
+    # hypervisor specified in the options will be created.
+    #
+    # @param node_info [Hash{String=>Object}] Node data parsed from the input
+    #                                         spec string.
+    #
+    # @option options [String] :hypervisor The string name of the hypervisor to
+    #                          create. An exception will be thrown if the
+    #                          hypervisor is unrecognized.
+    def self.create(node_info, options)
+      name = node_info['host_settings']['hypervisor'] || options[:hypervisor]
+      hypervisor = supported_hypervisors[name]
+      if hypervisor
+        hypervisor.new
+      else
+        raise "Invalid hypervisor: #{name}"
+      end
+    end
+
+    # Returns a map of all valid hypervisor implementations, where the keys are
+    # the string names and the values are the implementation classes.
+    #
+    # The string names are part of the beaker-hostgenerator API as they are
+    # used for specifying the default or per-host hypervisor in the layout
+    # specification input string.
+    #
+    # @returns [Hash{String=>Hypervisor::Interface}] A map of hypervisor names
+    #                                                and their implementations.
+    def self.supported_hypervisors()
+      {
+        'none' => BeakerHostGenerator::Hypervisor::None,
+        'vmpooler' => BeakerHostGenerator::Hypervisor::Vmpooler
+      }
+    end
+
+    class Interface
+      # Returns a map containing any general configuration required by this
+      # hypervisor. This map will be merged into the 'CONFIG' section of the
+      # final hosts configuration output.
+      #
+      # This will only be called if the hypervisor is used for a node, in which
+      # case the returned map will be merged in with global configuration from
+      # other hypervisors.
+      def global_config()
+        {}
+      end
+
+      # Returns a map of host configuration for a single node.
+      #
+      # This will be called for each individual node requested in the hosts
+      # specification input.
+      #
+      # Any configuration that is required by this hypervisor but is not
+      # specific to each node can be put in the `global_config` map.
+      #
+      # @param [Hash{String=>String}] node_info General info about the given
+      #                               node, such as the ostype, nodeid, and
+      #                               bits.
+      #
+      # @param [Hash{String=>Object}] base_config The node definition so far,
+      #                               which serves a starting point for this
+      #                               hypervisor to build upon. It is expected
+      #                               that this map will be merged into the map
+      #                               returned by this method.
+      #
+      # @param [Integer] bhg_version The version of OS info to use when building
+      #                              up the node definition.
+      def generate_node(node_info, base_config, bhg_version)
+        raise "Method 'generate_node' not implemented!"
+      end
+    end
+  end
+end
+
+# Require the hypervisor implementations so they can be referenced/instantiated
+# in the `create` factory method. We need to put these require statements at the
+# bottom of this file to avoid circular references between this file and the
+# hypervisor implementation files.
+require 'beaker-hostgenerator/hypervisor/vmpooler'
+require 'beaker-hostgenerator/hypervisor/none'

data/lib/beaker-hostgenerator/hypervisor/none.rb

@@ -0,0 +1,17 @@
+require 'beaker-hostgenerator/hypervisor'
+require 'beaker-hostgenerator/data'
+require 'deep_merge'
+
+module BeakerHostGenerator::Hypervisor
+  class None < BeakerHostGenerator::Hypervisor::Interface
+    include BeakerHostGenerator::Data
+
+    def generate_node(node_info, base_config, bhg_version)
+      platform = node_info['platform']
+      general_info = get_platform_info(bhg_version, platform, :general)
+      base_config.deep_merge! general_info
+      base_config['hypervisor'] = 'none'
+      return base_config
+    end
+  end
+end

data/lib/beaker-hostgenerator/hypervisor/vmpooler.rb

@@ -0,0 +1,32 @@
+require 'beaker-hostgenerator/data'
+require 'beaker-hostgenerator/hypervisor'
+require 'deep_merge'
+
+module BeakerHostGenerator
+  module Hypervisor
+    class Vmpooler < BeakerHostGenerator::Hypervisor::Interface
+      include BeakerHostGenerator::Data
+
+      def global_config()
+        {
+          'pooling_api' => 'http://vmpooler.delivery.puppetlabs.net/'
+        }
+      end
+
+      def generate_node(node_info, base_config, bhg_version)
+        # set hypervisor
+        base_config['hypervisor'] = 'vmpooler'
+
+        platform = node_info['platform']
+        platform_info = get_platform_info(bhg_version, platform, :vmpooler)
+        base_config.deep_merge! platform_info
+
+        # Some vmpooler/vsphere platforms have special requirements.
+        # We munge the node host config here if that is necessary.
+        fixup_node base_config
+
+        return base_config
+      end
+    end
+  end
+end

data/lib/beaker-hostgenerator/parser.rb

@@ -0,0 +1,199 @@
+require 'beaker-hostgenerator/data'
+require 'beaker-hostgenerator/exceptions'
+
+module BeakerHostGenerator
+  # Functions for parsing the raw user input host layout string and turning
+  # them into proper data structures suitable for processing by the Generator.
+  #
+  # The functions mainly perform data type conversions, like splitting the
+  # single input string into a list of strings, each of which will be processed
+  # further by other functions in this module.
+  #
+  # For example, given the raw user input string that defines the host layout,
+  # you would first split it into tokens via `tokenize_layout`, and then for
+  # each token you would call `is_ostype_token?` and/or `parse_node_info_token`.
+  module Parser
+
+    # Capture role and bit width information about the node.
+    #
+    # See Ruby Regexp class for information on the capture groups used below.
+    # http://ruby-doc.org/core-2.2.0/Regexp.html#class-Regexp-label-Character+Classes
+    #
+    # Examples node specs and their resulting roles
+    #
+    #  64compile_master,zuul,meow.a
+    #   * compile_master
+    #   * zuul
+    #   * meow
+    #   * agent
+    #
+    #  32herp.cdma
+    #   * herp
+    #   * dashboard
+    #   * database
+    #   * master
+    #   * agent
+    #
+    #  64dashboard,master,agent,database.
+    #   * dashboard
+    #   * master
+    #   * agent
+    #   * database
+    #
+    NODE_REGEX=/\A(?<bits>\d+)((?<arbitrary_roles>([[:lower:]_]*|\,)*)\.)?(?<roles>[uacldfm]*)(?<host_settings>\{[[:print:]]*\})?\Z/
+
+    module_function
+
+    # Breaks apart the host input string into chunks suitable for processing
+    # by the generator. Returns an array of substrings of the input spec string.
+    #
+    # The input string is expected to be properly formatted using the dash `-`
+    # character as a delimiter. Dashes may also be used within braces `{...}`,
+    # which are used to define arbitrary key-values on a node.
+    #
+    # @param spec [String] Well-formatted string specification of the hosts to
+    #                      generate. For example `"centos6-64m-debian8-32a"`.
+    # @returns [Array<String>] Input string split into substrings suitable for
+    #                          processing by the generator. For example
+    #                          `["centos6", "64m", "debian8", "32a"]`.
+    def tokenize_layout(layout_spec)
+      # Here we allow dashes in certain parts of the spec string
+      # i.e. "centos6-64m{hostname=foo-bar}-debian8-32"
+      # by first replacing all occurrences of - with | that exist within
+      # the braces {...}.
+      #
+      # So we'd end up with:
+      #   "centos6-64m{hostname=foo|bar}-debian8-32"
+      #
+      # Which we can then simply split on - into:
+      #   ["centos6", "64{hostname=foo|bar}", "debian8", "32"]
+      #
+      # And then finally turn the | back into - now that we've
+      # properly decomposed the spec string:
+      #   ["centos6", "64{hostname=foo-bar}", "debian8", "32"]
+      #
+      # NOTE we've specifically chosen to use the pipe character |
+      # due to its unlikely occurrence in the user input string.
+      spec = String.new(layout_spec) # Copy so we can replace characters inline
+      within_braces = false
+      spec.chars.each_with_index do |char, index|
+        case char
+        when '{'
+          within_braces = true
+        when '}'
+          within_braces = false
+        when '-'
+          spec[index] = '|' if within_braces
+        end
+      end
+      tokens = spec.split('-')
+      tokens.map { |t| t.gsub('|', '-') }
+    end
+
+    # Tests if a string token represents an OS platform (i.e. "centos6" or
+    # "debian8") and not another part of the host specification like the
+    # architecture bit (i.e. "32" or "64").
+    #
+    # This is used when parsing the host generator input string to determine
+    # if we're introducing a host for a new platform or if we're adding another
+    # host for a current platform.
+    #
+    # @param [String] token A piece of the host generator input that might refer
+    #                 to an OS platform. For example `"centos6"` or `"debian8"`.
+    #
+    # @param [Integer] bhg_version The version of OS info to use when testing
+    #                  for whether the token represent an OS platform.
+    def is_ostype_token?(token, bhg_version)
+      BeakerHostGenerator::Data.get_platforms(bhg_version).each do |platform|
+        ostype = platform.split('-')[0]
+        if ostype == token
+          return true
+        end
+      end
+      return false
+    end
+
+    # Converts a string token that represents a node (and not an OS type) into
+    # a proper hash map with keys representing the various regex capture groups
+    # in `NODE_REGEX` and values being the captured text.
+    #
+    # Throws an exception if the token is not in the expected formatted, as
+    # determined by `NODE_REGEX.match`.
+    #
+    # It is expected that the `Generator` will have initimate knowledge about
+    # the keys and values in the returned map, as it may be adjusted and given
+    # to the hypervisors or other abstractions for processing.
+    #
+    # @param token [String] The portion of the user input layout specifiction
+    #                       that describes the node (and not the OS platform).
+    #                       For example `"64myrole.mdca"`.
+    #
+    # @returns [Hash{String=>Object}] A map containing the regex capture groups
+    #                                 suitable for processing by the Generator
+    #                                 and hypervisors.
+    def parse_node_info_token(token)
+      node_info = NODE_REGEX.match(token)
+
+      if node_info
+        node_info = Hash[ node_info.names.zip( node_info.captures ) ]
+      else
+        raise BeakerHostGenerator::Exceptions::InvalidNodeSpecError.new,
+              "Invalid node_info token: #{token}"
+      end
+
+      if node_info['arbitrary_roles']
+        node_info['arbitrary_roles'] =
+          node_info['arbitrary_roles'].split(',') || ''
+      else
+        # Default to empty list to avoid having to check for nil elsewhere
+        node_info['arbitrary_roles'] = []
+      end
+
+      if node_info['host_settings']
+        node_info['host_settings'] =
+          settings_string_to_map(node_info['host_settings'])
+      else
+        node_info['host_settings'] = {}
+      end
+
+      return node_info
+    end
+
+    # Transforms the arbitrary host settings map from a string representation
+    # to a proper hash map data structure for merging into the host
+    # configuration.
+    #
+    # The string is expected to be of the form "{key1=value1,key2=value2,...}".
+    # Any whitespace found in the string will be stripped and ignored.
+    #
+    # Throws an exception of the string is malformed in any way.
+    #
+    # @param host_settings [String] Non-nil user input string that defines host
+    #                               specific settings.
+    #
+    # @returns [Hash{String=>String}] The host_settings string as a map.
+    def settings_string_to_map(host_settings)
+      # Strip it down to a list of pairs
+      settings_pairs =
+        host_settings.
+        delete('{}').
+        gsub(' ', '').
+        split(',').
+        map { |keyvalue| keyvalue.split('=') }
+
+      # Validate they're actually pairs, and that all keys are non-empty
+      settings_pairs.each do |pair|
+        if pair.length != 2
+          raise BeakerHostGenerator::Exceptions::InvalidNodeSpecError,
+                "Malformed host settings: #{host_settings}"
+        end
+        if pair.first.nil? || pair.first.empty?
+          raise BeakerHostGenerator::Exceptions::InvalidNodeSpecError,
+                "Malformed host settings: #{host_settings}"
+        end
+      end
+
+      Hash[settings_pairs]
+    end
+  end
+end