dopi 0.17.0

data/lib/dopi/command_set.rb

@@ -0,0 +1,66 @@
+#
+#
+# This class represents the set commands for a
+# specific node and step.
+#
+# It will also manage the run group restrictions
+#
+
+module Dopi
+  class CommandSet
+    include Dopi::State
+    attr_reader :plan, :step, :node
+
+    def initialize(step_parser, step, node)
+      @step_parser = step_parser
+      @step = step
+      @plan = step.plan
+      @node = node
+
+      commands.each{|command| state_add_child(command)}
+    end
+
+    def name
+      @node.name
+    end
+
+    def commands
+      @commands ||= @step_parser.commands.map do |command|
+        Dopi::Command.create_plugin_instance(command, @step, node)
+      end
+    end
+
+    def valid?
+      begin
+        commands.all?{|command| command.meta_valid?}
+      rescue PluginLoaderError => e
+        Dopi.log.error("Step '#{name}': Can't load plugin : #{e.message}")
+        false
+      end
+    end
+
+    def run(noop)
+      commands.each do |command|
+        break if state_failed? or signals[:stop]
+        command.meta_run(noop)
+        break unless command.state_done?
+      end
+    end
+
+    def load_state(state_hash)
+      return if state_hash.empty?
+      commands.each_with_index do |command, i|
+        command.load_state(state_hash[i])
+      end
+    end
+
+    def state_hash
+      commands.map do |command|
+        command.state_hash
+      end
+    end
+
+  end
+end
+
+

data/lib/dopi/connector/local.rb

@@ -0,0 +1,77 @@
+#
+# This connector simply executes commands on the local
+# machine. It does all the signal handling and makes
+# sure processes are stopped or killed.
+#
+require 'pty'
+require 'open3'
+
+module Dopi
+  module Connector
+    module Local
+
+      # The command method executes the command of the step.
+      # Returns an array with stdio, sterror and exit code.
+      def local_command(env, command_string)
+        master, slave = PTY.open
+        stdout_r, stdout_w = IO.pipe
+        stderr_r, stderr_w = IO.pipe
+        cmd_stdout = ''
+        cmd_stderr = ''
+        options = {
+          :pgroup          => true,
+          :unsetenv_others => true,
+          :in              => slave,
+          :out             => stdout_w,
+          :err             => stderr_w,
+        }
+        log(:debug, "Executing #{command_string} for command #{name}")
+        log(:debug, "Environment: #{env.to_s}")
+
+        pid = Process.spawn(merged_env(env), command_string, options)
+        slave.close
+        stdout_w.close
+        stderr_w.close
+
+        signal_handler = Proc.new do |signal|
+          case signal
+          when :abort then Process.kill(:TERM, pid)
+          when :kill  then Process.kill(:KILL, pid)
+          end
+        end
+
+        on_signal(signal_handler)
+        stdout_thread = Thread.new do
+          until ( line = stdout_r.gets ).nil? do
+            cmd_stdout << line
+            log(:debug, line.gsub("\n", '').gsub("\r", ''))
+          end
+        end
+
+        stderr_thread = Thread.new do
+          until ( line = stderr_r.gets ).nil? do
+            cmd_stderr << line
+            log(:error, line.gsub("\n", '').gsub("\r", ''))
+          end
+        end
+
+        _, status = Process.wait2(pid)
+        stdout_thread.join
+        stderr_thread.join
+        delete_on_signal(signal_handler)
+        [ cmd_stdout, cmd_stderr, status.exitstatus ]
+      end
+
+    private
+
+      def merged_env(env)
+        {
+          'HOME' => ENV['HOME'],
+          'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
+        }.merge(env)
+      end
+
+    end
+  end
+end
+

data/lib/dopi/connector/ssh.rb

@@ -0,0 +1,170 @@
+#
+# This connector will execute commands over ssh
+#
+#
+module Dopi
+  module Connector
+    module Ssh
+      include Dopi::Connector::Local
+      include Dopi::CommandParser::Credentials
+
+    public
+
+      def validate_ssh
+        log_validation_method(:port_valid?, CommandParsingError)
+        log_validation_method(:quiet_valid?, CommandParsingError)
+        log_validation_method(:check_host_key_valid?, CommandParsingError)
+        log_validation_method(:base64_valid?, CommandParsingError)
+        log_validation_method(:ssh_options_valid?, CommandParsingError)
+        validate_credentials
+      end
+
+      def ssh_command(env, command_string)
+        credential = working_ssh_credential
+        ssh_command_string = create_ssh_command_string(credential, env, command_string)
+        local_env = create_local_env(credential)
+        local_env.merge!(env) unless base64 # keep old behaviour for escaping
+        local_command(local_env, ssh_command_string)
+      end
+
+      def port
+        @port ||= port_valid? ? hash[:port].to_s : '22'
+      end
+
+      def quiet
+        @quiet || quiet_valid? ? hash[:quiet] : true
+      end
+
+      def check_host_key
+        @check_host_key || check_host_key_valid? ? hash[:check_host_key] : false
+      end
+
+      def base64
+        @base64 || base64_valid? ? hash[:base64] : true
+      end
+
+      def ssh_options
+        #TBD
+        []
+      end
+
+      def supported_credential_types
+        [:username_password, :ssh_key]
+      end
+
+    private
+
+      # this method checks for every credential if a login is possible
+      # and will return the first one where it is possible.
+      # If none of the credentials work it will raise a CommandConnectionError.
+      def working_ssh_credential
+        credentials.find do |credential|
+          ssh_command_string = create_ssh_command_string(credential, {}, 'exit')
+          local_env = create_local_env(credential)
+          local_command(local_env, ssh_command_string)[2] == 0
+        end or raise CommandConnectionError,
+          "Can't establish connection with node #{@node.name} with any of the given" +
+          "credentials #{credentials.map{|c| c.name}.join(', ')}"
+      end
+
+      def create_local_env(credential)
+        if credential.type == :username_password
+          { 'SSHPASS' => credential.password }
+        else
+          {}
+        end
+      end
+
+      def create_ssh_command_string(credential, env, command_string)
+        address = @node.address(port)
+        opts = options(credential)
+        prefix = credential.type == :username_password ? sshpass_cmd : ''
+        cmd = "#{ssh_env_string(env)} #{command_string}"
+        real_cmd = if base64
+          log(:debug, "Unencoded command: '#{cmd}'")
+          ssh_encode_command(cmd)
+        else
+          ssh_escape_command(cmd)
+        end
+        "#{prefix}ssh #{opts} #{credential.username}@#{address} \"#{real_cmd}\""
+      end
+
+      def options(credential)
+        opts = []
+        opts << "-p #{port}"
+        opts << '-q' if quiet
+        opts << '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' unless check_host_key
+        if credential.type == :ssh_key
+          opts << '-o ChallengeResponseAuthentication=no'
+          opts << '-o PasswordAuthentication=no'
+          opts << "-i #{credential.private_key}"
+        end
+        # Force allocation of tty, needed to propagate signals to remotely
+        # spawned processes
+        opts << '-tt'
+        opts += ssh_options
+        opts += ssh_options_defaults if respond_to?(:ssh_options_defaults)
+        opts.join(' ')
+      end
+
+      def sshpass_bin
+        @sshpass_bin ||= ENV['PATH'].split(':').map{|p| File.join(p, 'sshpass')}.find do |f|
+          File.exists?(f) && File.executable?(f)
+        end
+      end
+
+      def sshpass_cmd
+        @sshpass_cmd ||= sshpass_bin ? sshpass_bin + ' -e ' : ""
+      end
+
+      def ssh_env_string(env)
+        env.map{|variable,value| "export #{variable}=#{value};" }.join(' ')
+      end
+
+      def ssh_escape_command(cmd)
+        cmd.gsub("\\", "\\\\\\\\").gsub('"', '\\"')
+      end
+
+      def ssh_encode_command(cmd)
+        "echo -n #{Base64.strict_encode64(cmd)} | base64 -d | bash"
+      end
+
+      def port_valid?
+        return false unless hash.kind_of?(Hash)
+        return false if hash[:port].nil? # is optional
+        hash[:port].kind_of?(Fixnum) or
+          raise CommandParsingError, "Plugin #{name}: The value for port must be a number"
+        hash[:port].between?(0, 65536) or
+          raise CommandParsingError, "Plugin #{name}: The value for port must bigger than 0 and below 65536"
+        true
+      end
+
+      def quiet_valid?
+        return false unless hash.kind_of?(Hash)
+        return false if hash[:quiet].nil? # is optional
+        hash[:quiet].kind_of?(TrueClass) or hash[:quiet].kind_of?(FalseClass) or
+          raise CommandParsingError, "Plugin #{name}: The value for 'quiet' must be boolean"
+      end
+
+      def check_host_key_valid?
+        return false unless hash.kind_of?(Hash)
+        return false if hash[:check_host_key].nil? # is optional
+        hash[:check_host_key].kind_of?(TrueClass) or hash[:check_host_key].kind_of?(FalseClass) or
+          raise CommandParsingError, "Plugin #{name}: The value for 'check_host_key' must be boolean"
+      end
+
+      def base64_valid?
+        return false unless hash.kind_of?(Hash)
+        return false if hash[:base64].nil? # is optional
+        hash[:base64].kind_of?(TrueClass) or hash[:base64].kind_of?(FalseClass) or
+          raise CommandParsingError, "Plugin #{name}: The value for 'base64' must be boolean"
+      end
+
+      def ssh_options_valid?
+        #TBD
+        false
+      end
+
+    end
+  end
+end

data/lib/dopi/connector/winrm.rb

@@ -0,0 +1,167 @@
+#
+# DOPi Plugin: WinRM Command
+#
+require 'winrm'
+require 'gssapi'
+
+module Dopi
+  module Connector
+    module Winrm
+      include Dopi::CommandParser::Credentials
+
+      def validate_winrm
+        log_validation_method(:port_valid?)
+        log_validation_method(:ssl_valid?)
+        log_validation_method(:ca_trust_path_valid?)
+        log_validation_method(:disable_sspi_valid?)
+        log_validation_method(:basic_auth_only_valid?)
+        validate_credentials
+      end
+
+      def winrm_command(command_string)
+        cmd_stdout = ""
+        cmd_stderr = ""
+        log(:debug, "Executing '#{command_string}' for command #{name}")
+        result = winrm.run_cmd(command_string) do |stdout, stderr|
+          unless stdout.nil? or stdout.empty?
+            cmd_stdout << stdout
+            log(:debug, stdout)
+          end
+          unless stderr.nil? or stderr.empty?
+            cmd_stderr << stderr
+            log(:error, stderr)
+          end
+        end
+        winrm.close
+        [cmd_stdout, cmd_stderr, result[:exitcode]]
+      end
+
+      def winrm_powershell_command(command_string)
+        log(:debug, "Unencoded Powershell command '#{command_string}'")
+        script = WinRM::PowershellScript.new(command_string)
+        winrm_command("powershell -encodedCommand #{script.encoded()}")
+      end
+
+      def winrm
+        working_command_executor = nil
+        credentials.each do |credential|
+          begin
+            winrm_web_service = WinRM::WinRMWebService.new(
+              endpoint,
+              auth_method(credential),
+              :realm           => credential.realm,
+              :service         => credential.service,
+              :keytab          => credential.keytab,
+              :user            => credential.username,
+              :pass            => credential.password,
+              :disable_sspi    => disable_sspi,
+              :basic_auth_only => basic_auth_only,
+              :ca_trust_path   => ca_trust_path
+            )
+            winrm_web_service.set_timeout(operation_timeout)
+            command_executor = WinRM::CommandExecutor.new(winrm_web_service)
+            command_executor.open
+            command_executor.run_cmd('exit') # test connection
+          rescue WinRM::WinRMAuthorizationError, GSSAPI::GssApiError => e
+            log(:warn, "Unable to login with credential #{credential.name} : #{e.message}")
+          rescue SocketError => e
+            raise CommandConnectionError,
+              "A problem occurred while trying to connect to node #{@node.name} : #{e.message}"
+          else working_command_executor = command_executor
+          end
+        end
+        working_command_executor or
+          raise CommandExecutionError,
+            "Unable to login with any of the given credentials: #{credentials.map{|c| c.name}.join(', ')}"
+      end
+
+      def endpoint
+        "http://#{@node.address(port)}:#{port}/wsman"
+      end
+
+      def port
+        port_valid? ? hash[:port] : 5985
+      end
+
+      def ssl
+        ssl_valid? ? hash[:ssl] : true
+      end
+
+      def ca_trust_path
+        ca_trust_path_valid? ? hash[:ca_trust_path] : nil
+      end
+
+      def disable_sspi
+        disable_sspi_valid? ? hash[:disable_sspi] : nil
+      end
+
+      def basic_auth_only
+        basic_auth_only_valid? ? hash[:basic_auth_only] : nil
+      end
+
+      def operation_timeout
+        operation_timeout_valid? ? hash[:operation_timeout] : ( plugin_timeout - 5 )
+      end
+
+      def supported_credential_types
+        [:username_password, :kerberos]
+      end
+
+    private
+
+      def auth_method(credential)
+        case credential.type
+        when :kerberos then :kerberos
+        when :username_password then ssl ? :ssl : :plaintext
+        end
+      end
+
+      def port_valid?
+        return false if hash.nil?
+        return false if hash[:port].nil?
+        hash[:port].kind_of?(Fixnum) and (hash[:port] > 0) and (hash[:port] < 65536) or
+          raise CommandParsingError, "The value for 'port' has to be a number in the range of 1-65535"
+      end
+
+      def ssl_valid?
+        return false if hash.nil?
+        return false if hash[:ssl].nil?
+        hash[:ssl].kind_of?(TrueClass) or hash[:ssl].kind_of?(FalseClass) or
+          raise CommandParsingError, "The value for 'ssl_valid' has to be true or false"
+      end
+
+      def ca_trust_path_valid?
+        return false if hash.nil?
+        return false if hash[:ca_trust_path].nil?
+        hash[:ca_trust_path].kind_of?(String) or
+          raise CommandParsingError, "The value for ca_trust_path has to be a string"
+        File.directory?(hash[:ca_trust_path]) or
+          raise CommandParsingError, "The directory in 'ca_trust_path' does not exist"
+      end
+
+      def disable_sspi_valid?
+        return false if hash.nil?
+        return false if hash[:disable_sspi].nil?
+        hash[:disable_sspi].kind_of?(TrueClass) or hash[:disable_sspi].kind_of?(FalseClass) or
+          raise CommandParsingError, "The value for 'disable_sspi' has to be true or false"
+      end
+
+      def basic_auth_only_valid?
+        return false if hash.nil?
+        return false if hash[:basic_auth_only].nil?
+        hash[:basic_auth_only].kind_of?(TrueClass) or hash[:basic_auth_only].kind_of?(FalseClass) or
+          raise CommandParsingError, "The value for 'basic_auth_only' has to be true or false"
+      end
+
+      def operation_timeout_valid?
+        return false if hash.nil?
+        return false if hash[:operation_timeout].nil?
+        hash[:operation_timeout].kind_of?(Fixnum) or
+          raise CommandParsingError, 'The value for operation_timeout has to be a number'
+        hash[:operation_timeout] >= 0 or
+          raise CommandParsingError, 'The value for operation_timeout has to be positive number'
+      end
+
+    end
+  end
+end

data/lib/dopi/error.rb

@@ -0,0 +1,43 @@
+#
+# Various error classes for DOPi
+#
+# Error hierarchy:
+#
+#   PluginLoaderError
+#   StateTransitionError
+#   NoRoleFoundError
+#   CommandParsingError
+#   CommandExecutionError
+#     CommandExecutionError
+#       ConnectionError
+#         NodeConnectionError
+#         CommandConnectionError
+#
+module Dopi
+  class PluginLoaderError < StandardError
+  end
+
+  class StateTransitionError < StandardError
+  end
+
+  class NoRoleFoundError < StandardError
+  end
+
+  class CommandParsingError < StandardError
+  end
+
+  class CommandExecutionError < StandardError
+  end
+
+  class GracefulExit < StandardError
+  end
+
+  class ConnectionError < CommandExecutionError
+  end
+
+  class NodeConnectionError < ConnectionError
+  end
+
+  class CommandConnectionError < ConnectionError
+  end
+end

data/lib/dopi/log.rb

@@ -0,0 +1,18 @@
+#
+# the logger stuff
+#
+require 'logger'
+require 'dop_common/log'
+
+module Dopi
+
+  def self.log
+    @log ||= DopCommon.log
+  end
+
+  def self.logger=(logger)
+    @log = logger
+    DopCommon.logger = logger
+  end
+
+end

data/lib/dopi/node.rb

@@ -0,0 +1,70 @@
+#
+# This class loades a deployment plan
+#
+require 'hiera'
+require 'yaml'
+require 'socket'
+require 'timeout'
+require 'forwardable'
+
+module Dopi
+  class Node
+    extend Forwardable
+
+    attr_accessor :node_info
+
+    def initialize(node_parser, plan)
+      @node_parser = node_parser
+      @plan = plan
+      @addresses = {}
+      @node_info = {}
+    end
+
+    def_delegators :@node_parser,
+      :name,
+      :fqdn,
+      :has_name?,
+      :config,
+      :has_config?,
+      :config_includes?,
+      :fact,
+      :has_fact?,
+      :role,
+      :has_role?
+
+    def addresses
+      [ fqdn, plan_ip_addresses, node_info_ip_addresses ].flatten.uniq
+    end
+
+    def address(port)
+      @addresses[port] ||= addresses.find {|addr| connection_possible?(addr,port)} or
+        raise NodeConnectionError, "Unable to establish a connection for node #{name} on port #{port} over #{addresses.join(', ')}"
+    end
+
+    def reset_address(port = nil)
+      port.nil? ? @addresses = {} : @addresses.delete(port)
+    end
+
+  private
+
+    def plan_ip_addresses
+      @node_parser.interfaces.map{|i| [:dhcp, :none].include?(i.ip) ? nil : i.ip}.compact
+    end
+
+    def node_info_ip_addresses
+      node_info[:ip_addresses] || []
+    end
+
+    def connection_possible?(address, port)
+      Timeout::timeout(DopCommon.config.connection_check_timeout.to_i) do
+        TCPSocket.new(address, port).close
+      end
+      Dopi.log.debug("Connection test with #{address}:#{port} for node #{name} ok")
+      true
+    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Timeout::Error, SocketError
+      Dopi.log.debug("Connection test with #{address}:#{port} for node #{name} failed")
+      false
+    end
+
+  end
+end