elevage 0.1.0

data/lib/elevage/environment.rb

@@ -0,0 +1,223 @@
+require 'yaml'
+require 'resolv'
+
+require_relative 'constants'
+require_relative 'platform'
+require_relative 'provisioner'
+
+# rubocop:disable ClassLength
+module Elevage
+  # Environment class
+  class Environment
+    attr_accessor :name
+    attr_accessor :vcenter
+    attr_accessor :components
+    attr_accessor :nodenameconvention
+
+    # rubocop:disable LineLength
+    def initialize(env)
+      # Confirm environment has been defined in the platform
+      platform = Elevage::Platform.new
+      fail(IOError, ERR[:env_not_defined]) unless platform.environments.include?(env)
+      # Confirm environment file exists
+      envfile = ENV_FOLDER + env.to_s + '.yml'
+      fail(IOError, ERR[:no_env_file]) unless env_file_exists?(envfile)
+      # Build environment hash from environment and platform defintion files
+      environment = build_env(env, YAML.load_file(envfile).fetch('environment'), platform)
+      # Populate class variables
+      @name = env
+      @vcenter = environment['vcenter']
+      @components = environment['components']
+      @nodenameconvention = platform.nodenameconvention
+    end
+    # rubocop:enable LineLength
+
+    # Public: Environment class method
+    # Returns multiline string = IP, fqdn, runlist
+    def list_nodes
+      nodes =  @vcenter['destfolder'].to_s + "\n"
+      @components.each do |component, _config|
+        (1..@components[component]['count']).each do |i|
+          nodes += @components[component]['addresses'][i - 1].ljust(18, ' ') +
+                   node_name(component, i) + @vcenter['domain'] + '   ' +
+                   @components[component]['runlist'].to_s + "\n"
+        end
+      end
+      nodes
+    end
+
+    # rubocop:disable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity
+    def healthy?
+      platform = Elevage::Platform.new
+      health = ''
+      health += MSG[:invalid_env_vcenter] if @vcenter.nil?
+      @components.each do |component, v|
+        health += MSG[:invalid_env_network] if v['network'].nil?
+        health += MSG[:invalid_env_count] unless (0..POOL_LIMIT).member?(v['count'])
+        health += MSG[:invalid_env_compute] if v['compute'].nil?
+        health += MSG[:invalid_env_ip] if v['count'] != v['addresses'].size
+        if v['addresses'].nil?
+          health += MSG[:invalid_env_ip]
+        else
+          v['addresses'].each { |ip| health += MSG[:invalid_env_ip] unless Resolv::IPv4::Regex.match(ip) }
+        end
+        health += MSG[:invalid_env_tier] unless platform.tiers.include?(v['tier'])
+        health += MSG[:invalid_env_image] if v['image'].nil?
+        health += MSG[:invalid_env_port] unless v['port'].is_a?(Integer) || v['port'].nil?
+        health += MSG[:invalid_env_runlist] if v['runlist'].nil? || v['runlist'].empty?
+        health += MSG[:invalid_env_componentrole] unless v['componentrole'].include?('#') if v['componentrole']
+        health += MSG[:env_component_mismatch] unless platform.components.include?(component)
+      end
+      health += MSG[:env_component_mismatch] unless platform.components.size == @components.size
+      if health.length > 0
+        puts health + "\n#{health.lines.count} environment offense(s) detected"
+        false
+      else
+        true
+      end
+    end
+    # rubocop:enable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity
+
+    # Public: basic class puts string output
+    def to_s
+      puts @name
+      puts @vcenter.to_yaml
+      puts @components.to_yaml
+      puts @nodenameconvention.to_yaml
+    end
+
+    # Public: method to request provisioning of all or a portion
+    # of the environment
+    # rubocop:disable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity
+    def provision(type: all, tier: nil, component: nil, instance: nil, options: nil)
+      # Create the ProvisionerRunQueue to batch up our tasks
+      runner = ProvisionerRunQueue.new
+
+      # Modify behavior for dry-run (no concurrency)
+      if !options['dry-run']
+        runner.max_concurrent = options[:concurrency]
+      else
+        puts "Dry run requested, forcing concurrency to '1'."
+        runner.max_concurrent = 1
+      end
+
+      @components.each do |component_name, component_data|
+        next unless type.eql?(:all) || component_data['tier'].match(/#{tier}/i) && component_name.match(/#{component}/i)
+
+        1.upto(component_data['addresses'].count) do |component_instance|
+          next unless instance == component_instance || instance.nil?
+
+          instance_name = node_name(component_name, component_instance)
+
+          # Create the Provisioner
+          provisioner = Elevage::Provisioner.new(instance_name, component_data, component_instance, self, options)
+
+          # Add it to the queue
+          runner.provisioners << provisioner
+
+        end
+      end
+
+      runner.to_s if options['dry-run']
+
+      # Process the queue
+      runner.run
+    end
+    # rubocop:enable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity
+
+    private
+
+    # Private: updates env hash with necessary info from Platform files.
+    # This is a blend of env and Platform info needed to construct
+    # Environment class object
+    #
+    # Params
+    #   env: string passed from commend line, simple environment name
+    #   env_yaml: hash from requested environment.yml
+    #   platform: Platform class object built from standard
+    #             platform definition files
+    #
+    # Returns Hash: updated env_yaml hash
+    # rubocop:disable MethodLength, LineLength
+    def build_env(env, env_yaml, platform)
+      # substitute vcenter resources from vcenter.yml for location defined in environment file
+      env_yaml['vcenter'] = platform.vcenter[env_yaml['vcenter']]
+      # merge component resources from environment file and platform definition
+      # platform info will be inserted where not found in env files, env overrides will be unchanged
+      #
+      # Note: this function does not do error checking for components that exist in env file but
+      # not in Platform definition. Such files will be retained but not have any Platform
+      # component info. The Build command will run error checking before building, but to support
+      #  the debugging value of the list command only hash.merge! is performed at this point.
+      platform.components.each do |component, _config|
+        env_yaml['components'][component].merge!(platform.components[component]) { |_key, v1, _v2| v1 } unless env_yaml['components'][component].nil?
+      end
+      # substitute network and components for specified values from platform definition files
+      env_yaml['components'].each do |component, _config|
+        env_yaml['components'][component]['network'] = platform.network[env_yaml['components'][component]['network']]
+        env_yaml['components'][component]['compute'] = platform.compute[env_yaml['components'][component]['compute']]
+        unless env_yaml['components'][component]['runlist'].nil?
+          env_yaml['components'][component]['runlist'] = run_list(env_yaml['components'][component]['runlist'], env_yaml['components'][component]['componentrole'], component)
+        end
+      end
+      unless env_yaml['vcenter'].nil?
+        # append env name to destination folder if appendenv == true
+        env_yaml['vcenter']['destfolder'] += (env_yaml['vcenter']['appendenv'] ? '/' + env.to_s : '')
+        # prepend app name to domain if appenddomain == true
+        env_yaml['vcenter']['appenddomain'] ? env_yaml['vcenter']['domain'] = '.' + platform.name + '.' + env_yaml['vcenter']['domain'] : ''
+      end
+      env_yaml
+    end
+    # rubocop:enable MethodLength, LineLength
+
+    # Private: construct a node hostname from parameters
+    #
+    # Params
+    #   component: Hash, environment components
+    #   instance: integer, passed from loop iterator
+    #
+    # Returns hostname as String
+    # rubocop:disable MethodLength
+    def node_name(component, instance)
+      name = ''
+      @nodenameconvention.each do |i|
+        case i
+        when 'environment'
+          name += @name
+        when 'component'
+          name += component
+        when 'instance'
+          name += instance.to_s.rjust(2, '0')
+        when 'geo'
+          name += @vcenter['geo'].to_s[0]
+        else
+          name += i
+        end
+      end
+      name
+    end
+    # rubocop:enable MethodLength
+
+    # Private: Constructs the node runlist from parameters
+    #
+    # Params
+    #   list: Array of strings from component runlist hash key value
+    #   componentrole:  String value from component, performs simple
+    #                   string substitution of for component string
+    #                   in component role string
+    #   component: String, component name
+    #
+    # Returns runlist as String
+    # rubocop:disable LineLength
+    def run_list(list, componentrole, component)
+      list.join(',') + (componentrole ? ',' + componentrole.gsub('#', component) : '')
+    end
+    # rubocop:enable LineLength
+
+    def env_file_exists?(env_file)
+      fail(IOError, ERR[:no_environment_file]) unless File.file?(env_file)
+      true
+    end
+  end
+end
+# rubocop:enable ClassLength

data/lib/elevage/generate.rb

@@ -0,0 +1,48 @@
+require 'thor/group'
+
+module Elevage
+  # Create new environment desired state files from platform template
+  class Generate < Thor::Group
+    include Thor::Actions
+    argument :env
+
+    def self.source_root
+      File.dirname(__FILE__)
+    end
+
+    # rubocop:disable MethodLength, CyclomaticComplexity, PerceivedComplexity
+    def create_environment
+      fail IOError, ERR[:env_exists] if File.file?(ENV_FOLDER + env + '.yml')
+      platform = Elevage::Platform.new
+      platformfile = File.open(YML_PLATFORM, 'r')
+      #
+      # The things from here forward I would rather have in the template file
+      # but that is even uglier, trying to get formatting correct
+      # will need to investigate some POWER erb skills to clean this up
+      @env_pools = ''
+      @env_components = ''
+      line = ''
+      line = platformfile.gets until line =~ /pools/
+      platform.pools.each do |k, _v|
+        line = platformfile.gets until line.include?(k)
+        @env_pools += line
+        next_line = platformfile.gets
+        @env_pools += "#{next_line}" if next_line.include?('<<')
+        @env_pools += "      network:\n\n"
+      end
+      line = platformfile.gets until line =~ /components/
+      platform.components.each do |k, v|
+        line = platformfile.gets until line.include?(k)
+        @env_components += line
+        next_line = platformfile.gets
+        @env_components += "#{next_line}" if next_line.include?('<<')
+        @env_components += "      addresses:\n"
+        (1..v['count']).each { @env_components += "        -\n" }
+        @env_components += "\n"
+      end
+      template(TEMPLATE_ENV, ENV_FOLDER + env + '.yml')
+      puts "#{env}.yml added in environments folder"
+    end
+    # rubocop:enable MethodLength, CyclomaticComplexity, PerceivedComplexity
+  end
+end

data/lib/elevage/health.rb

@@ -0,0 +1,27 @@
+require 'thor/group'
+
+module Elevage
+  # Evaluate health of platform definition files
+  class Health < Thor::Group
+    include Thor::Actions
+
+    def self.source_root
+      File.dirname(__FILE__)
+    end
+
+    # rubocop:disable LineLength
+    def check_platform
+      @platform = Elevage::Platform.new
+      puts @platform.healthy? ? MSG_HEALTHY : fail(IOError, ERR[:fail_health_check])
+    end
+    # rubocop:enable LineLength
+
+    # rubocop:disable LineLength
+    def check_environments
+      @platform.environments.each do |env|
+        puts Elevage::Environment.new(env).healthy? ? (env + MSG_ENV_HEALTHY) : fail(IOError, ERR[:fail_health_check])
+      end
+    end
+    # rubocop:enable LineLength
+  end
+end

data/lib/elevage/new.rb

@@ -0,0 +1,30 @@
+require 'thor/group'
+
+module Elevage
+  # Create new platform definition files and environments folder structure
+  class New < Thor::Group
+    include Thor::Actions
+
+    argument :platform
+
+    def self.source_root
+      File.dirname(__FILE__)
+    end
+
+    # Confirm command is not being run in folder with
+    # existing platform definition
+    def already_exists?
+      File.file?(YML_PLATFORM) && fail(IOError, ERR[:platform_exists])
+    end
+
+    def create_platform_file
+      template(TEMPLATE_PLATFORM, YML_PLATFORM)
+    end
+
+    def create_infrastructure_files
+      template(TEMPLATE_VCENTER, YML_VCENTER)
+      template(TEMPLATE_NETWORK, YML_NETWORK)
+      template(TEMPLATE_COMPUTE, YML_COMPUTE)
+    end
+  end
+end

data/lib/elevage/platform.rb

@@ -0,0 +1,105 @@
+# rubocop:disable LineLength
+require 'yaml'
+require 'resolv'
+require 'English'
+
+module Elevage
+  # Platform class
+  class Platform
+    attr_accessor :name, :description
+    attr_accessor :environments
+    attr_accessor :tiers
+    attr_accessor :nodenameconvention
+    attr_accessor :pools
+    attr_accessor :components
+    attr_accessor :vcenter
+    attr_accessor :network
+    attr_accessor :compute
+
+    # rubocop:disable MethodLength
+    def initialize
+      fail unless platform_files_exists?
+      platform = YAML.load_file(YML_PLATFORM).fetch('platform')
+      @name = platform['name']
+      @description = platform['description']
+      @environments = platform['environments']
+      @tiers = platform['tiers']
+      @nodenameconvention = platform['nodenameconvention']
+      @pools = platform['pools']
+      @components = platform['components']
+      @vcenter = YAML.load_file(YML_VCENTER).fetch('vcenter')
+      @network = YAML.load_file(YML_NETWORK).fetch('network')
+      @compute = YAML.load_file(YML_COMPUTE).fetch('compute')
+    end
+    # rubocop:enable MethodLength
+
+    # rubocop:disable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity, AmbiguousOperator
+    def healthy?
+      health = ''
+      # Array of string checked for empty values
+      health += MSG[:empty_environments] unless @environments.all?
+      health += MSG[:empty_tiers] unless @tiers.all?
+      health += MSG[:empty_nodenameconvention] unless @nodenameconvention.all?
+      # Loop through all pool definitions, check for valid settings
+      @pools.each do |_pool, v|
+        health += MSG[:pool_count_size] unless (0..POOL_LIMIT).member?(v['count'])
+        health += MSG[:invalid_tiers] unless @tiers.include?(v['tier'])
+        health += MSG[:no_image_ref] if v['image'].nil?
+        health += MSG[:invalid_compute] unless @compute.key?(v['compute'])
+        health += MSG[:invalid_port] if v['port'].nil?
+        health += MSG[:invalid_runlist] unless v['runlist'].all?
+        health += MSG[:invalid_componentrole] unless v['componentrole'].include?('#') if v['componentrole']
+      end
+      # Loop through all vcenter definitions, check for valid settings
+      @vcenter.each do |_vcenter, v|
+        health += MSG[:invalid_geo] if v['geo'].nil?
+        health += MSG[:invalid_timezone] unless (0..TIMEZONE_LIMIT).member?(v['timezone'].to_i)
+        health += MSG[:invalid_host] if v['host'].nil?
+        health += MSG[:invalid_datacenter] if v['datacenter'].nil?
+        health += MSG[:invalid_imagefolder] if v['imagefolder'].nil?
+        health += MSG[:invalid_destfolder] if v['destfolder'].nil?
+        health += MSG[:invalid_appendenv] unless v['appendenv'] == true || v['appendenv'] == false
+        health += MSG[:invalid_appenddomain] unless v['appenddomain'] == true || v['appenddomain'] == false
+        health += MSG[:empty_datastores] unless v['datastores'].all?
+        health += MSG[:invalid_domain] if v['domain'].nil?
+        v['dnsips'].each { |ip| health += MSG[:invalid_ip] unless Resolv::IPv4::Regex.match(ip) }
+      end
+      # Loop through all network definitions, check for valid settings
+      @network.each do |_network, v|
+        health += MSG[:empty_network] if v.values.any? &:nil?
+        health += MSG[:invalid_gateway] unless Resolv::IPv4::Regex.match(v['gateway'])
+      end
+      # Loop through all compute definitions, check for valid settings
+      @compute.each do |_compute, v|
+        health += MSG[:invalid_cpu] unless (0..CPU_LIMIT).member?(v['cpu'])
+        health += MSG[:invalid_ram] unless (0..RAM_LIMIT).member?(v['ram'])
+      end
+      if health.length > 0
+        puts health + "\n#{health.lines.count} platform offense(s) detected"
+        false
+      else
+        true
+      end
+    end
+    # rubocop:enable MethodLength, LineLength, CyclomaticComplexity, PerceivedComplexity, AmbiguousOperator
+
+    private
+
+    # Private: confirms existence of the standard platform definition files
+    # Returns true if all standard files present
+    def platform_files_exists?
+      fail(IOError, ERR[:no_platform_file]) unless File.file?(YML_PLATFORM)
+      fail(IOError, ERR[:no_vcenter_file]) unless File.file?(YML_VCENTER)
+      fail(IOError, ERR[:no_network_file]) unless File.file?(YML_NETWORK)
+      fail(IOError, ERR[:no_compute_file]) unless File.file?(YML_COMPUTE)
+      true
+    end
+
+    # Unimplemented - part of future Communication health check option
+    # def valid_vcenter_host?(address)
+    #   _result = `ping -q -c 3 #{address}`
+    #   $CHILD_STATUS.exitstatus == 0 ? true : false
+    #   true
+    # end
+  end
+end

data/lib/elevage/provisioner.rb

@@ -0,0 +1,169 @@
+require 'thread'
+require 'open4'
+require_relative 'constants'
+require_relative 'platform'
+require_relative 'provisionerrunqueue'
+
+module Elevage
+  # Provisioner class
+  class Provisioner
+    attr_accessor :name
+    attr_accessor :component
+    attr_accessor :instance
+    attr_accessor :environment
+    attr_accessor :vcenter
+
+    # Set us up to build the specified instance of component
+    def initialize(name, component, instance, environment, options)
+      @name = name
+      @component = component
+      @instance = instance
+      @environment = environment
+      @options = options
+      @vcenter = @environment.vcenter
+    end
+
+    def to_s
+      puts "Name: #{@name}"
+      puts "Instance: #{@instance}"
+      puts "Component: #{@component}"
+      puts @component.to_yaml
+      puts 'Environment:'
+      puts @environment.to_yaml
+    end
+
+    # Public: Build the node
+    # rubocop:disable MethodLength, LineLength, GlobalVars, CyclomaticComplexity
+    def build
+      knife_cmd = generate_knife_cmd
+
+      # Modify behavior for dry-run
+      # Echo command to stdout and logfile instead of executing command.
+      if @options['dry-run']
+        puts knife_cmd
+        knife_cmd = "echo #{knife_cmd}"
+      end
+
+      # Open the logfile for writing
+      logfile = File.new("#{@options[:logfiles]}/#{@name}.log", 'w')
+
+      stamp = @options['dry-run'] ? '' : "#{Time.now} [#{$$}]: "
+      puts "#{stamp}#{@name}: logging to #{logfile.path}"
+      logfile.puts "#{stamp}#{@name}: Provisioning."
+
+      # Execute the knife command, capturing stderr and stdout as they
+      # produce anything, and push it all into a Queue object, which we then
+      # write to the log file as things come available.
+      status = Open4.popen4(knife_cmd) do |_pid, _stdin, stdout, stderr|
+        sem = Mutex.new
+        # Set and forget the thread for stderr...
+        # err_thread = Thread.new do
+        Thread.new do
+          while (line = stderr.gets)
+            sem.synchronize { logfile.puts line }
+          end
+        end
+        out_thread = Thread.new do
+          while (line = stdout.gets)
+            sem.synchronize { logfile.puts line }
+          end
+        end
+        out_thread.join
+        # err_thread.exit
+      end
+
+      stamp = @options['dry-run'] ? '' : "#{Time.now} [#{$$}]: "
+      logfile.puts "#{stamp}#{@name}: exit status: #{status.exitstatus}"
+      logfile.close
+
+      # Inform our master whether we succeeded or failed. Any non-zero
+      # exit status is a failure, and the details will be in the logfile
+      status.exitstatus == 0 ? true : false
+    end
+    # rubocop:enable MethodLength, LineLength, GlobalVars, CyclomaticComplexity
+
+    private
+
+    # Private: Determine which datastore to use for this specific
+    # provisioning.
+    def select_datastore
+      if @options['dry-run']
+        @vcenter['datastores'][0]
+      else
+        @vcenter['datastores'][rand(@vcenter['datastores'].count)]
+      end
+    end
+
+    # Private: Build the knife command that will do the provisioning.
+    # rubocop:disable MethodLength, LineLength
+    def generate_knife_cmd
+      knife_cmd = 'knife vsphere vm clone --vsinsecure --start'
+
+      # Authentication and host
+      knife_cmd << " --vsuser #{@options[:vsuser]}"
+      knife_cmd << " --vspass #{@options[:vspass]}"
+      knife_cmd << " --vshost #{@vcenter['host']}"
+
+      # VM Template (what we're cloning)
+      knife_cmd << " --folder '#{@vcenter['imagefolder']}'"
+      knife_cmd << " --template '#{@component['image']}'"
+
+      # vSphere destination information (where the clone will end up)
+      knife_cmd << " --vsdc '#{@vcenter['datacenter']}'"
+      knife_cmd << " --dest-folder '#{@vcenter['destfolder']}'"
+      knife_cmd << " --resource-pool '#{@vcenter['resourcepool']}'"
+      knife_cmd << " --datastore '#{select_datastore}'"
+
+      # VM Hardware
+      knife_cmd << " --ccpu #{@component['compute']['cpu']}"
+      knife_cmd << " --cram #{@component['compute']['ram']}"
+
+      # VM Networking
+      knife_cmd << " --cvlan '#{@component['network']['vlanid']}'"
+      knife_cmd << " --cips #{@component['addresses'][@instance - 1]}/#{@component['network']['netmask']}"
+      knife_cmd << " --cdnsips #{@vcenter['dnsips'].join(',')}"
+      knife_cmd << " --cgw #{@component['network']['gateway']}"
+      knife_cmd << " --chostname #{@name}"
+      knife_cmd << " --ctz #{@vcenter['timezone']}"
+
+      # AD Domain and DNS Suffix
+      domain = @vcenter['domain']
+      domain = domain[1, domain.length] if domain.start_with?('.')
+      knife_cmd << " --cdomain #{domain}"
+      knife_cmd << " --cdnssuffix #{domain}"
+
+      # Knife Bootstrap options
+      knife_cmd << ' --bootstrap'
+      knife_cmd << " --template-file '#{@options['template-file']}'"
+
+      # knife fqdn specifies how knife will connect to the target (in this case by IP)
+      knife_cmd << " --fqdn #{@component['addresses'][@instance - 1]}"
+      knife_cmd << " --ssh-user #{@options['ssh-user']}"
+      knife_cmd << " --identity-file '#{@options['ssh-key']}'"
+
+      # What the node should be identified as in Chef
+      nodename = String.new(@name)
+      nodename << '.' << @environment.name if @vcenter['appendenv']
+      nodename << @vcenter['domain'] if @vcenter['appenddomain']
+      knife_cmd << " --node-name '#{nodename}'"
+
+      # Assign the run_list
+      knife_cmd << " --run-list '#{@component['runlist']}'"
+
+      # Assign the Chef environment
+      knife_cmd << " --environment '#{@environment.name}'"
+
+      # What version of chef-client are we bootstrapping (not sure this is necessary)
+      knife_cmd << " --bootstrap-version #{@options['bootstrap-version']}"
+
+      # Finally, the name of the VM as seen by vSphere.
+      # Whereas nodename will optionally append the domain name, VM names should *always* have the domain name
+      # appended. The only optional bit is including the chef environment in the name.
+      vmname = String.new(@name)
+      vmname << '.' << @environment.name if @vcenter['appendenv']
+      vmname << @vcenter['domain']
+      knife_cmd << " #{vmname}"
+    end
+    # rubocop:enable MethodLength, LineLength
+  end
+end