smart_proxy_remote_execution_ssh 0.0.1

data/bundler.d/Gemfile.local.rb

@@ -0,0 +1 @@
+gem 'pry'

data/bundler.d/remote_execution_ssh.rb

@@ -0,0 +1 @@
+gem 'smart_proxy_remote_execution_ssh'

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -0,0 +1,44 @@
+require 'smart_proxy_dynflow'
+
+require 'smart_proxy_remote_execution_ssh/version'
+require 'smart_proxy_remote_execution_ssh/plugin'
+
+require 'smart_proxy_remote_execution_ssh/dispatcher'
+require 'smart_proxy_remote_execution_ssh/command_action'
+
+require 'smart_proxy_remote_execution_ssh/api'
+
+module Proxy::RemoteExecution
+  module Ssh
+    class << self
+      attr_reader :dispatcher
+
+      def initialize
+        unless private_key_file
+          raise "settings for `ssh_identity_key` not set"
+        end
+
+        unless File.exist?(private_key_file)
+          raise "Ssh public key file #{private_key_file} doesn't exist.\n"\
+            "You can generate one with `ssh-keygen -t rsa -b 4096 -f #{private_key_file} -N ''`"
+        end
+
+        unless File.exist?(public_key_file)
+          raise "Ssh public key file #{public_key_file} doesn't exist"
+        end
+
+        @dispatcher = Proxy::RemoteExecution::Ssh::Dispatcher.spawn('proxy-ssh-dispatcher',
+                                                                    :clock  => Proxy::Dynflow.instance.world.clock,
+                                                                    :logger => Proxy::Dynflow.instance.world.logger)
+      end
+
+      def private_key_file
+        File.expand_path(Ssh::Plugin.settings.ssh_identity_key_file)
+      end
+
+      def public_key_file
+        File.expand_path("#{private_key_file}.pub")
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -0,0 +1,9 @@
+module Proxy::RemoteExecution
+  module Ssh
+    class Api < ::Sinatra::Base
+      get "/pubkey" do
+        File.read(Ssh.public_key_file)
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/command_action.rb

@@ -0,0 +1,75 @@
+module Proxy::RemoteExecution::Ssh
+  class CommandAction < ::Dynflow::Action
+    include Dynflow::Action::Cancellable
+    include ::Proxy::Dynflow::Callback::PlanHelper
+
+    def plan(input)
+      if callback = input['callback']
+        input[:task_id] = callback['task_id']
+      else
+        input[:task_id] ||= SecureRandom.uuid
+      end
+      plan_with_callback(input)
+    end
+
+    def run(event = nil)
+      case event
+      when nil
+        init_run
+      when Dispatcher::CommandUpdate
+        update = event
+        output[:result].concat(update.buffer_to_hash)
+        if update.exit_status
+          finish_run(update)
+        else
+          suspend
+        end
+      when Dynflow::Action::Cancellable::Cancel
+        kill_run
+      when Dynflow::Action::Skip
+        # do nothing
+      else
+        raise "Unexpected event #{event.inspect}"
+      end
+    end
+
+    def finalize
+      # To mark the task as a whole as failed
+      error! "Script execution failed" if failed_run?
+    end
+
+    def rescue_strategy_for_self
+      Dynflow::Action::Rescue::Skip
+    end
+
+    def command
+      @command ||= Dispatcher::Command.new(:id               => input[:task_id],
+                                           :host             => input[:hostname],
+                                           :ssh_user         => 'root',
+                                           :effective_user   => input[:effective_user],
+                                           :script           => input[:script],
+                                           :host_public_key  => input[:host_public_key],
+                                           :verify_host      => input[:verify_host],
+                                           :suspended_action => suspended_action)
+    end
+
+    def init_run
+      output[:result] = []
+      Proxy::RemoteExecution::Ssh.dispatcher.tell([:initialize_command, command])
+      suspend
+    end
+
+    def kill_run
+      Proxy::RemoteExecution::Ssh.dispatcher.tell([:kill, command])
+      suspend
+    end
+
+    def finish_run(update)
+      output[:exit_status] = update.exit_status
+    end
+
+    def failed_run?
+      output[:exit_status] != 0
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/connector.rb

@@ -0,0 +1,182 @@
+require 'net/ssh'
+require 'net/scp'
+
+module Proxy::RemoteExecution::Ssh
+  # Service that handles running external commands for Actions::Command
+  # Dynflow action. It runs just one (actor) thread for all the commands
+  # running in the system and updates the Dynflow actions periodically.
+  class Connector
+    class Data
+      attr_reader :data, :timestamp
+
+      def initialize(data, timestamp = Time.now)
+        @data = data
+        @timestamp = timestamp
+      end
+
+      def data_type
+        raise NotImplemented
+      end
+    end
+
+    class StdoutData < Data
+      def data_type
+        :stdout
+      end
+    end
+
+    class StderrData < Data
+      def data_type
+        :stderr
+      end
+    end
+
+    class DebugData < Data
+      def data_type
+        :debug
+      end
+    end
+
+    class StatusData < Data
+      def data_type
+        :status
+      end
+    end
+
+    MAX_PROCESS_RETRIES = 3
+
+    def initialize(host, user, options = {})
+      @host = host
+      @user = user
+      @logger = options[:logger] || Logger.new($stderr)
+      @client_private_key_file = options[:client_private_key_file]
+      @known_hosts_file = options[:known_hosts_file]
+    end
+
+    # Initiates run of the remote command and yields the data when
+    # available. The yielding doesn't happen automatically, but as
+    # part of calling the `refresh` method.
+    def async_run(command)
+      started = false
+      session.open_channel do |channel|
+        channel.on_data { |ch, data| yield StdoutData.new(data) }
+
+        channel.on_extended_data { |ch, type, data| yield StderrData.new(data) }
+
+        # standard exit of the command
+        channel.on_request("exit-status") { |ch, data| yield StatusData.new(data.read_long) }
+
+        # on signal: sedning the signal value (such as 'TERM')
+        channel.on_request("exit-signal") do |ch, data|
+          yield(StatusData.new(data.read_string))
+          ch.close
+          # wait for the channel to finish so that we know at the end
+          # that the session is inactive
+          ch.wait
+        end
+
+        channel.exec(command) do |ch, success|
+          started = true
+          unless success
+            yield DebugData.new("FAILED: couldn't execute command (ssh.channel.exec)")
+            yield StatusData.new("INIT_ERROR")
+          end
+        end
+      end
+      session.process(0) until started
+      return true
+    end
+
+    def run(command)
+      output = ""
+      exit_status = nil
+      channel = session.open_channel do |ch|
+        ch.on_data { |data| output.concat(data) }
+
+        ch.on_extended_data { |_, _, data| output.concat(data) }
+
+        ch.on_request("exit-status") { |_, data| exit_status = data.read_long }
+
+        # on signal: sedning the signal value (such as 'TERM')
+        ch.on_request("exit-signal") do |_, data|
+          exit_status = data.read_string
+          ch.close
+          ch.wait
+        end
+
+        ch.exec command do |_, success|
+          raise "could not execute command" unless success
+        end
+      end
+      channel.wait
+      return exit_status, output
+    end
+
+    # calls the callback registered in the `async_run` when some data
+    # for the session are available
+    def refresh
+      return if @session.nil?
+      tries = 0
+      begin
+        session.process(0)
+      rescue => e
+        @logger.error("Error while processing ssh channel: #{e.class} #{e.message}\n #{e.backtrace.join("\n")}")
+        tries += 1
+        if tries <= MAX_PROCESS_RETRIES
+          retry
+        else
+          raise e
+        end
+      end
+    end
+
+    def upload_file(local_path, remote_path)
+      ensure_remote_directory(File.dirname(remote_path))
+      scp = Net::SCP.new(session)
+      upload_channel = scp.upload(local_path, remote_path)
+      upload_channel.wait
+    ensure
+      if upload_channel
+        upload_channel.close
+        upload_channel.wait
+      end
+    end
+
+    def ensure_remote_directory(path)
+      exit_code, output = run("mkdir -p #{path}")
+      if exit_code != 0
+        raise "Unable to create directory on remote system #{path}: exit code: #{exit_code}\n #{output}"
+      end
+    end
+
+    def inactive?
+      @session.nil? || @session.channels.empty?
+    end
+
+    def close
+      @logger.debug("closing session to #{@user}@#{@host}")
+      @session.close unless @session.nil? || @session.closed?
+    end
+
+    private
+
+    def session
+      @session ||= begin
+                     @logger.debug("opening session to #{@user}@#{@host}")
+                     Net::SSH.start(@host, @user, ssh_options)
+                   end
+    end
+
+    def ssh_options
+      ssh_options = {}
+      ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
+      ssh_options[:user_known_hosts_file] = @known_hosts_file if @known_hosts_file
+      ssh_options[:keys_only] = true
+      # if the host public key is contained in the known_hosts_file,
+      # verify it, otherwise, if missing, import it and continue
+      ssh_options[:paranoid] = true
+      ssh_options[:auth_methods] = ["publickey"]
+      return ssh_options
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/dispatcher.rb

@@ -0,0 +1,235 @@
+require 'smart_proxy_remote_execution_ssh/connector'
+
+module Proxy::RemoteExecution::Ssh
+  # Service that handles running external commands for Actions::Command
+  # Dynflow action. It runs just one (actor) thread for all the commands
+  # running in the system and updates the Dynflow actions periodically.
+  class Dispatcher < ::Dynflow::Actor
+    # command comming from action
+    class Command
+      attr_reader :id, :host, :ssh_user, :effective_user, :script, :host_public_key, :suspended_action
+
+      def initialize(data)
+        validate!(data)
+
+        @id               = data[:id]
+        @host             = data[:host]
+        @ssh_user         = data[:ssh_user]
+        @effective_user   = data[:effective_user]
+        @script           = data[:script]
+        @host_public_key  = data[:host_public_key]
+        @suspended_action = data[:suspended_action]
+      end
+
+      def validate!(data)
+        required_fields = [:id, :host, :ssh_user, :script, :suspended_action]
+        missing_fields = required_fields.find_all { |f| !data[f] }
+        raise ArgumentError, "Missing fields: #{missing_fields}" unless missing_fields.empty?
+      end
+    end
+
+    # update sent back to the suspended action
+    class CommandUpdate
+      attr_reader :buffer, :exit_status
+
+      def initialize(buffer, exit_status)
+        @buffer      = buffer
+        @exit_status = exit_status
+      end
+
+      def buffer_to_hash
+        buffer.map do |buffer_data|
+          { :output_type => buffer_data.data_type,
+            :output      => buffer_data.data,
+            :timestamp   => buffer_data.timestamp.to_f }
+        end
+      end
+    end
+
+    def initialize(options = {})
+      @clock                   = options[:clock] || Dynflow::Clock.spawn('proxy-dispatcher-clock')
+      @logger                  = options[:logger] || Logger.new($stderr)
+      @connector_class         = options[:connector_class] || Connector
+      @local_working_dir       = options[:local_working_dir] || '/tmp/foreman-proxy-ssh/server'
+      @remote_working_dir      = options[:remote_working_dir] || '/tmp/foreman-proxy-ssh/client'
+      @refresh_interval        = options[:refresh_interval] || 1
+      @client_private_key_file = Proxy::RemoteExecution::Ssh.private_key_file
+
+      @connectors        = {}
+      @command_buffer    = Hash.new { |h, k| h[k] = [] }
+      @refresh_planned = false
+    end
+
+    def initialize_command(command)
+      @logger.debug("initalizing command [#{command}]")
+      connector = self.connector_for_command(command)
+      remote_script = cp_script_to_remote(connector, command)
+      if command.effective_user && command.effective_user != command.ssh_user
+        su_prefix = "su - #{command.effective_user} -c "
+      end
+      output_path = File.join(File.dirname(remote_script), 'output')
+
+      connector.async_run("#{su_prefix}#{remote_script} | /usr/bin/tee #{output_path}") do |data|
+        command_buffer(command) << data
+      end
+    rescue => e
+      @logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
+      command_buffer(command).concat([Connector::DebugData.new("Exception: #{e.class} #{e.message}"),
+                                      Connector::StatusData.new('INIT_ERROR')])
+    ensure
+      plan_next_refresh
+    end
+
+    def refresh
+      finished_commands = []
+      refresh_connectors
+
+      @command_buffer.each do |command, buffer|
+        unless buffer.empty?
+          status = refresh_command_buffer(command, buffer)
+          if status
+            finished_commands << command
+          end
+        end
+      end
+
+      finished_commands.each { |command| finish_command(command) }
+      close_inactive_connectors
+    ensure
+      @refresh_planned = false
+      plan_next_refresh
+    end
+
+    def refresh_command_buffer(command, buffer)
+      status = nil
+      @logger.debug("command #{command} got new output: #{buffer.inspect}")
+      buffer.delete_if do |data|
+        if data.is_a? Connector::StatusData
+          status = data.data
+          true
+        end
+      end
+      command.suspended_action << CommandUpdate.new(buffer, status)
+      clear_command(command)
+      if status
+        @logger.debug("command [#{command}] finished with status #{status}")
+        return status
+      end
+    end
+
+    def kill(command)
+      @logger.debug("killing command [#{command}]")
+      connector_for_command(command).run("pkill -f #{remote_command_file(command, 'script')}")
+    end
+
+    protected
+
+    def connector_for_command(command, only_if_exists = false)
+      if connector = @connectors[[command.host, command.ssh_user]]
+        return connector
+      end
+      return nil if only_if_exists
+      @connectors[[command.host, command.ssh_user]] = open_connector(command)
+    end
+
+    def local_command_dir(command)
+      File.join(@local_working_dir, command.id)
+    end
+
+    def local_command_file(command, filename)
+      File.join(local_command_dir(command), filename)
+    end
+
+    def remote_command_dir(command)
+      File.join(@remote_working_dir, command.id)
+    end
+
+    def remote_command_file(command, filename)
+      File.join(remote_command_dir(command), filename)
+    end
+
+    def ensure_local_directory(path)
+      if File.exist?(path)
+        raise "#{path} expected to be a directory" unless File.directory?(path)
+      else
+        FileUtils.mkdir_p(path)
+      end
+      return path
+    end
+
+    def cp_script_to_remote(connector, command)
+      local_script_file = write_command_file_locally(command, 'script', command.script)
+      File.chmod(0777, local_script_file)
+      remote_script_file = remote_command_file(command, 'script')
+      connector.upload_file(local_script_file, remote_script_file)
+      return remote_script_file
+    end
+
+    def write_command_file_locally(command, filename, content)
+      path = local_command_file(command, filename)
+      ensure_local_directory(File.dirname(path))
+      File.write(path, content)
+      return path
+    end
+
+    def open_connector(command)
+      options = { :logger => @logger }
+      options[:known_hosts_file] = prepare_known_hosts(command)
+      options[:client_private_key_file] = @client_private_key_file
+      @connector_class.new(command.host, command.ssh_user, options)
+    end
+
+    def prepare_known_hosts(command)
+      path = local_command_file(command, 'known_hosts')
+      if command.host_public_key
+        write_command_file_locally(command, 'known_hosts', "#{command.host} #{command.host_public_key}")
+      end
+      return path
+    end
+
+    def close_inactive_connectors
+      @connectors.delete_if do |_, connector|
+        if connector.inactive?
+          connector.close
+          true
+        end
+      end
+    end
+
+    def refresh_connectors
+      @logger.debug("refreshing #{@connectors.size} connectors")
+
+      @connectors.values.each do |connector|
+        begin
+          connector.refresh
+        rescue => e
+          @command_buffer.each do |command, buffer|
+            if connector_for_command(command, false)
+              buffer << Connector::DebugData.new("Exception: #{e.class} #{e.message}")
+            end
+          end
+        end
+      end
+    end
+
+    def command_buffer(command)
+      @command_buffer[command]
+    end
+
+    def clear_command(command)
+      @command_buffer[command] = []
+    end
+
+    def finish_command(command)
+      @command_buffer.delete(command)
+    end
+
+    def plan_next_refresh
+      if @connectors.any? && !@refresh_planned
+        @logger.debug("planning to refresh")
+        @clock.ping(reference, Time.now + @refresh_interval, :refresh)
+        @refresh_planned = true
+      end
+    end
+  end
+end