cloudbox-server 0.0.1

data/lib/common/services_description.rb

@@ -0,0 +1,192 @@
+# This file is part of cloudbox-service project
+# @author tmarmin (INGENSI)
+
+module Common
+
+  module ServicesDescription
+    private
+    # This variable gives informations about each service available in cloudbox.
+    # :MASTER : master components,
+    # :SLAVE : slave components,
+    # :CONFIG : nil if service doesn't require configuration, name of the config type else,
+    # :SLAVEONMASTER : slave components to install on master node by default.
+    # :DEPS : dependencies with other services.
+    @services_description = {
+        "HDFS" => {
+            :MASTER => %w{NAMENODE}, # Maybe add SECONDARY NAMENODE
+            :SLAVE => %w{DATANODE},
+            :CONFIG => "hdfs-site",
+            :SLAVEONMASTER => [],
+            :DEPS => []},
+        "MAPREDUCE" => {
+            :MASTER => %w{JOBTRACKER},
+            :SLAVE => %w{TASKTRACKER},
+            :CONFIG => "mapred-site",
+            :SLAVEONMASTER => [],
+            :DEPS => %w{HDFS}},
+        "HIVE" => {
+            :MASTER => %w{HIVE_SERVER},
+            :SLAVE => [],
+            :CONFIG => "hive-site",
+            :SLAVEONMASTER => [],
+            :DEPS => %w{MAPREDUCE}},
+        "HBASE" => {
+            :MASTER => %w{HBASE_MASTER},
+            :SLAVE => %w{HBASE_REGIONSERVER},
+            :CONFIG => "hbase-site",
+            :SLAVEONMASTER => [],
+            :DEPS => %w{HDFS ZOOKEEPER}},
+        "ZOOKEEPER" => {
+            :MASTER => %w{ZOOKEEPER_SERVER},
+            :SLAVE => [],
+            :CONFIG => "zookeeper-site",
+            :SLAVEONMASTER => [],
+            :DEPS => []},
+        "OOZIE" => {
+            :MASTER => %w{OOZIE_SERVER},
+            :SLAVE => [],
+            :CONFIG => "oozie-site",
+            :SLAVEONMASTER => [],
+            :DEPS => []},
+        "NAGIOS" => {
+            :MASTER => %w{NAGIOS_SERVER},
+            :SLAVE => [],
+            :CONFIG => nil,
+            :SLAVEONMASTER => [],
+            :DEPS => []},
+        "GANGLIA" => {
+            :MASTER => %w{GANGLIA_SERVER},
+            :SLAVE => %w{GANGLIA_MONITOR},
+            :CONFIG => nil,
+            :SLAVEONMASTER => %w{GANGLIA_MONITOR},
+            :DEPS => []}
+    }
+
+    public
+    def get_services_list
+      return @services_description.keys
+    end
+
+    public
+    def get_components_list
+      components = []
+      @services_description.each_value do |value|
+        components.concat(value[:MASTER])
+        components.concat(value[:SLAVE])
+      end
+
+      return components.uniq
+    end
+
+    public
+    def get_components(service_name)
+      components = []
+      unless @services_description[service_name].nil?
+        components = @services_description[service_name][:MASTER] + @services_description[service_name][:SLAVE]
+      end
+
+      return components
+    end
+
+    public
+    def get_config_name_service(service_name)
+      config_name = nil
+      unless @services_description[service_name].nil?
+        config_name = @services_description[service_name][:CONFIG]
+      end
+
+      return config_name
+    end
+
+    public
+    def get_master_nodes(service_name)
+      components = []
+      unless @services_description[service_name].nil?
+        components = @services_description[service_name][:MASTER]
+      end
+
+      return components
+    end
+
+    public
+    def get_slave_nodes(service_name)
+      components = []
+      unless @services_description[service_name].nil?
+        components = @services_description[service_name][:SLAVE]
+      end
+
+      return components
+    end
+
+    public
+    def get_slave_nodes_on_master(service_name)
+      components = []
+      unless @services_description[service_name].nil?
+        components = @services_description[service_name][:SLAVEONMASTER]
+      end
+
+      return components
+    end
+
+    public
+    def get_deps(service_name)
+      deps = []
+      unless @services_description[service_name].nil?
+        deps = @services_description[service_name][:DEPS]
+        deps.each do |deps_name|
+          deps.concat(get_deps(deps_name))
+        end
+      end
+
+      return deps.uniq
+    end
+
+    public
+    def get_depends_on(service_name)
+      depends = []
+
+      @services_description.each do |service, description|
+        [service, description]
+        if service != service_name && description[:DEPS].include?(service_name)
+          depends.push(service)
+          depends.concat(get_depends_on(service))
+        end
+      end
+
+      return depends.uniq
+    end
+
+    public
+    def get_master_components_deps(service_name)
+      deps = []
+      service_deps = get_deps(service_name)
+
+      service_deps.each do |service_dep|
+        deps.concat(get_master_nodes(service_dep))
+      end
+
+      return deps.uniq
+    end
+
+    def get_master_components_depends_on(service_name)
+      deps = []
+      service_deps = get_depends_on(service_name)
+
+      service_deps.each do |service_dep|
+        deps.concat(get_master_nodes(service_dep))
+      end
+
+      return deps.uniq
+    end
+
+    public
+    def get_service_name_by_comp_name(component_name)
+      @services_description.each do |service_name, values|
+        return service_name if (values[:MASTER].include?(component_name) || values[:SLAVE].include?(component_name))
+      end
+
+      return nil
+    end
+
+  end
+end

data/lib/exec.rb

@@ -0,0 +1,35 @@
+# This file is part of cloudbox-service project
+# @author tmarmin (INGENSI)
+
+$: << File.dirname(__FILE__)
+
+require 'exec/executable_command'
+require 'exec/command_option'
+
+require 'exec/node_ls'
+require 'exec/cluster_create'
+require 'exec/cluster_ls'
+require 'exec/cluster_delete'
+require 'exec/cluster_allocate'
+require 'exec/cluster_desallocate'
+require 'exec/service_conf_create'
+require 'exec/cluster_info'
+
+require 'exec/node_destroy'
+require 'exec/node_detach'
+require 'exec/node_info'
+require 'exec/node_install'
+require 'exec/node_ls'
+require 'exec/service_add'
+require 'exec/service_conf_ls'
+require 'exec/service_conf_apply'
+require 'exec/service_component_add'
+require 'exec/service_component_delete'
+require 'exec/service_component_detach'
+require 'exec/service_follow_request'
+require 'exec/service_info'
+require 'exec/service_install'
+require 'exec/service_ls'
+require 'exec/service_start'
+require 'exec/service_stop'
+require 'exec/service_conf_get'

data/lib/exec/check_parameter.rb

@@ -0,0 +1,208 @@
+# This file is part of cloudbox-server project
+# @author tmarmin (INGENSI)
+
+require 'rubygems'
+require 'ipaddress'
+
+require 'lib/common/services_description'
+
+module Exec
+  #Module who gather all methods whoch check synthax of parameter
+  module CheckParameter
+
+    # Check cluster name format.
+    # It has to be a string of alphanumric down case char.
+    def check_cluster_name(str)
+      return match_regex(str, /^[a-z0-9]+$/, "The vcluser name #{str} is not valid. It should be an alphanumeric (down case) string.")
+    end
+
+    # Check type name format of the configuration about a ambari cluster.
+    # It has to be a string of alphanumric down case char.
+    def check_type_conf(str)
+      return match_regex(str, /^[a-z0-9]+$/, "The type name #{str} of the configuration is not valid. It should be an alphanumeric (down case) string.")
+    end
+
+    # Check tag name format of the configuration about a ambari cluster.
+    # It has to be a string of alphanumric down case char.
+    #def check_tag_conf(str)
+    #  return match_regex(str, /^[a-z0-9]+$/, "The tag name #{str} of the configuration is not valid. It should be an alphanumeric (down case) string.")
+    #end
+
+    # Check crowbar node name format.
+    # It has to be a string of any characters.
+    def check_crowbar_node_name(str)
+      return match_regex(str, /^[A-Z0-9a-z\-]+$/, "The Node name #{str} is not valid.")
+    end
+
+    # Check raid name format.
+    # It has to be a string of alphanumric down case char.
+    def check_param_for_bios(str)
+      msg = ""
+      valid = true
+      if str.strip() != "Hadoop" && str.strip() != "HadoopInfra" && str.strip() != "Storage" && str.strip() != "Virtualization"
+        valid = false
+        msg = "The Bios name must be only \"Hadoop\", \"HadoopInfra\", \"Storage\" or \"Virtualization\""
+        return valid, msg
+      end
+      return valid, msg
+    end
+
+    # Check raid name format.
+    # It has to be a string of alphanumric down case char.
+    def check_param_for_raid(str)
+      msg = ""
+      valid = true
+      if str.strip() != "JBODOnly" && str.strip() != "Raid10"
+        valid = false
+        msg = "The Raid name must be only \"JBODOnly\" or \"Raid10\""
+        return valid, msg
+      end
+      return valid, msg
+    end
+
+    # Check vlan id format.
+    # It has to be a positive integer.
+    def check_vlan_id(str)
+      if str.to_i >= 10 && str.to_i <= 4096
+        return match_regex(str.to_s, /^[1-9][0-9]+$/, "ID Vlan '#{str}' is not a correct number.")
+      else
+        return false, "ID Vlan '#{str}' is not a correct number."
+      end
+    end
+
+    # Check ram size format.
+    # It has to be an integer followed by 'g' charatcter.
+    def check_ram_size(str)
+      return match_regex(str, /^[1-9]+[0-9]*g$/, "RAM value #{str} is not valid. It should be a positive integer followed by.")
+    end
+
+    # Check cpu weight format.
+    # It has to be a positive integer.
+    def check_cpu_weight(str)
+      return match_regex(str, /^[1-9]+[0-9]*$/, "CPU value '#{str}' is not valid. It should be a positive integer.")
+    end
+
+    # Check network address.
+    # Exemple of network format : 10.0.0.0/8
+    # Network adress must have 1 octet in order to have a CIDR equals to 8
+    # Network adress must have 2 octet in order to have a CIDR equals to 16
+    # Network adress must have 3 octet in order to have a CIDR equals to 24
+    def check_network_address(str)
+      msg = ""
+      valid = true
+      if str == nil
+        valid = false
+        msg = "Network adress and mask is missing"
+        return valid, msg
+      end
+      network = str.split("/").first
+      cidr = str.split("/").last
+
+      if network == nil
+        valid = false
+        msg = "Network adress is missing"
+        return valid, msg
+      end
+      if cidr == nil
+        valid = false
+        msg = "cidr mask is missing "
+        return valid, msg
+      end
+
+      if cidr != "24" && cidr != "16" && cidr != "8"
+        valid = false
+        msg = "cidr is only valid if equal 8 , 16 , 24 "
+        return valid, msg
+      end
+      unless IPAddress.valid?(network)
+        valid = false
+        msg = "Network adress is not valid "
+        return valid, msg
+      end
+      octet1 = network.split(".").first
+      octet2 = network.split(".").at(1)
+      octet3 = network.split(".").at(2)
+      octet4 = network.split(".").last
+      if cidr == "8"
+        if octet1 == "0" || octet2 != "0" || octet3 != "0" || octet4 !="0"
+          valid = false
+          msg = "Network adress is not valid "
+          return valid, msg
+        end
+      elsif cidr == "16"
+        if octet1 == "0" || octet3 != "0" || octet4 !="0"
+          valid = false
+          msg = "Network adress is not valid "
+          return valid, msg
+        end
+      elsif cidr == "24"
+        if octet1 == "0" || octet4 != "0"
+          valid = false
+          msg = "Network adress is not valid "
+          return valid, msg
+        end
+      end
+
+      return valid, msg
+    end
+
+    # Check hadoop service name.
+    # It has to be a known hadoop service.
+    def check_hadoop_service_name(str)
+      hadoop_services = Common::ServicesDescription.get_services_list
+      valid = true
+      msg = ""
+
+      unless hadoop_services.include?(str)
+        valid = false
+        msg = "Service name parameter has to be known Hadoop service."
+      end
+
+      return valid, msg
+    end
+
+    # Check hadoop component name.
+    # It has to be a known component name.
+    def check_hadoop_component_name(str)
+      hadoop_components = Common::ServicesDescription.get_components_list
+      valid = true
+      msg = ""
+
+      unless hadoop_components.include?(str)
+        valid = false
+        msg = "Component name parameter has to be known Hadoop component."
+      end
+
+      return valid, msg
+    end
+
+    # Check ambari host name
+    # @todo returns allways true for the moment.
+    def check_ambari_host_name(str)
+      return true, ""
+    end
+
+    # Check request id.
+    # It has to be an integer.
+    def check_request_id(str)
+      return match_regex(str, /^[0-9]*$/, "Request id has to be a positive integer.")
+    end
+
+    private
+    # Check if a string matches a regex
+    # @param str [String] The string to check.
+    # @param regexp [Rexexp] The regular expression.
+    # @param msg_if_nok [String] The message to return if str doesn't match with regex
+    # @return [Boolean, String] true,"" if msg matched with regex, false, msg_if_nok if doesn't.
+    def match_regex(str, regexp, msg_if_nok)
+      msg = ""
+      valid = true
+      regex=Regexp.new(regexp)
+      unless str.match(regex)
+        valid = false
+        msg = msg_if_nok
+      end
+      return valid, msg
+    end
+  end
+end

data/lib/exec/cluster_allocate.rb

@@ -0,0 +1,163 @@
+# This file is part of cloudbox-server project
+# Author: mbr (INGENSI)
+# Created: 22/08/13 11:08
+
+# You are a good developer if you document this class
+
+require 'exec/executable_command'
+require 'exec/check_parameter'
+require 'common/cloudbox_exceptions'
+require 'command/crowbar_node_command'
+require 'command/crowbar_node_list'
+require 'command/crowbar_cluster_allocate'
+require 'command/ambari_add_host'
+require 'command/ambari_host_list'
+
+include Exec::CheckParameter
+
+module Exec
+  # Allows user  to allocate node on a cluster
+  class ClusterAllocate < ExecutableCommand
+
+    public
+    # Default constructor of the class.
+    # @note Overrides default constructor by passing CustomCommandOption to super().
+    # @author mbretaud
+    def initialize(argv, stdin, stdout, stderr, command_name)
+      super(argv, stdin, stdout, stderr, command_name, CustomCommandOption)
+    end
+
+    private
+    # Parse and check the parameters of the function.
+    # @author tnoguer
+    def set_options
+      @options.add_option("a", "all", "All nodes which are ready.", false)
+      @options.add_option("C", "cluster_name", "The name of the vcluster .", true, true, method(:check_cluster_name))
+      @options.add_option("n", "list_nodes", "The list of nodes.", false, true, method(:check_crowbar_node_name))
+    end
+
+    def exec
+      @logger.info("Exec::ClusterAllocate   Executing ClusterAllocate")
+      @logger.info("Exec::ClusterAllocate   Allocate node on the vcluster '#{@values['cluster']}'...")
+      Color::print_log("NONE", "Allocate node on the vcluster '#{@values['cluster']}'...", @stdout)
+
+      output = ""
+
+      if @values['all']
+        ###### Allocate all nodes ######
+        @logger.info("executing CrowbarNodeList")
+        cmd = Command::CrowbarNodeList.new(@logger, "Ready")
+        list_nodes = cmd.exec
+
+        list_nodes_filter = Array.new
+        list_nodes.each { |node|
+          list_nodes_filter << node.split(" ").at(0).strip.to_s
+        }
+
+        cmd = Command::CrowbarClusterAllocate.new(@logger, @values['cluster_name'], true, list_nodes_filter)
+        output += cmd.exec()
+
+        Color::echo_ok(@stdout)
+        output.each{|out|
+          Color::print_log("NONE", "#{out}", @stdout)
+          Color::echo_ok(@stdout)
+        }
+      elsif @values['list_nodes'] != nil
+        ###### Allocate a list of nodes ######
+        cmd = Command::CrowbarNodeList.new(@logger, "Ready")
+        list_nodes_ready = cmd.exec()
+
+        list_nodes = Hash.new
+        @values['list_nodes'].split(" ").each{|node|
+          list_nodes[node] = node
+        }
+
+        list_nodes_filter = Array.new
+        list_nodes.each {|key, value|
+          node_exists = false
+          list_nodes_ready.each{|node|
+            if node == key
+              node_exists = true
+            end
+          }
+
+          if !node_exists
+            raise NodeNotExistsError.new("The node '#{key}' is not exists.")
+          end
+
+          list_nodes_filter << key.strip.to_s
+        }
+
+        cmd = Command::CrowbarClusterAllocate.new(@logger, @values['cluster_name'], false, list_nodes_filter)
+        output += cmd.exec()
+
+        Color::echo_ok(@stdout)
+        output.each{|out|
+          Color::print_log("NONE", "#{out}", @stdout)
+          Color::echo_ok(@stdout)
+        }
+      end
+
+      @logger.info("Exec::ClusterAllocate   Allocate node on the ambari cluster '#{@values['cluster']}'...")
+      Color::print_log("NONE", "Allocate node on the vcluster '#{@values['cluster']}'...", @stdout)
+
+      output = ""
+
+      ###### Allocate nodes on the ambari cluster ######
+      cmd = Command::AmbariHostList.new(@values['cluster_name'])
+      list_hosts = cmd.exec()
+
+      list_nodes_filter.each do |node|
+        host_exists = false
+        attempts = 0
+        node = "#{@values['cluster_name']}-#{node.gsub(/\s+/, "").strip}"
+
+        list_hosts.each{|host|
+          if host.strip == node.strip
+            host_exists = true
+          end
+        }
+
+        if !host_exists
+          begin
+            cmd = Command::AmbariAddHost.new(@values['cluster_name'], node)
+            cmd.exec()
+          rescue => e
+            attempts += 1
+            sleep 5
+            retry unless attempts > 10
+            raise ClusterAllocateError.new("Allocate the host '#{node}' to the ambari cluster '#{@values['cluster_name']}'.")
+          end
+          output += "Add the host #{node} into the cluster #{@values['cluster_name']}.\n"
+        else
+          output += "The host #{node} into the cluster #{@values['cluster_name']} is already exists...\n"
+        end
+
+        Color::echo_ok(@stdout)
+        output.each{|out|
+          Color::print_log("NONE", "#{out}", @stdout)
+          Color::echo_ok(@stdout)
+        }
+      end
+    end
+
+    private
+    # Inherited CommandOption used only by ServiceLs.
+    class CustomCommandOption < CommandOption
+
+      # This method overrides the default banner generating method
+      def get_banner
+        return "Usage: #@command_name -C CLUSTER_NAME -a|-n NODE_NAME"
+      end
+
+      protected
+      # This method overrides the default behaviour.
+      # @raise [Common::MissingParameter] Never raise exception for the moment :)
+      def check_mandatory()
+        raise Common::MissingParameter.new("-C", get_help()) if !@values["help"] && @values["cluster_name"].nil?
+        raise Common::MissingParameter.new("-n or -a", get_help()) if !@values["help"] && @values["list_nodes"].nil? && !@values["all"] && !@values["cluster_name"].nil?
+        raise Common::InconsistentParameter.new("can not be used together.", get_help(), "-a", "-n") if !@values["help"] && @values["all"] && !@values["list_nodes"].nil?
+      end
+    end
+  end
+end