foreman_remote_execution 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b2960d7df5f74d0bb9455ea4f024b1d31b9a68f
-  data.tar.gz: 53d254b4f949852d53403d6ac15f6093211a3c79
+  metadata.gz: 8ca71ec8c1f3b330d327966298dea09c81a21378
+  data.tar.gz: a86edc45eaebca0fa3acfeecf38115fa6596d199
 SHA512:
-  metadata.gz: 720aaba4a13126df8deea7d0566aa154da1b5bde179fecd51cd072f634f37f0c189f2d7942c665dba93b5078cb3872477735ac628d09ca035c394306acbe109d
-  data.tar.gz: 4d57da51e9a0a8c651c778b5b0be52cc9c2159db07b532be6b2e4e1246f8d20b1b27f22217fdc028ea4dd180092963bcfec5da3e7b21f32a94629cf3e0599b5d
+  metadata.gz: db8e9e87cf16680859499d1f910f318b3ffba7e47bd180e30b5e43a25bebe04ef08a44b4225b519c73b312c4033d3b93dfababb80999ea04c667a0dbdd527d6f
+  data.tar.gz: 0d4b6974e4172fff1feee3a861cad6a14fb10788d1fb1a1dc8486328e94029681f3fbc9070394f706a434f6047dc2840f947f09422853d9faa551404698bd3ed

data/app/lib/actions/remote_execution/run_host_job.rb

@@ -1,15 +1,16 @@
 module Actions
   module RemoteExecution
     class RunHostJob < Actions::EntryAction
+      include ::Actions::Helpers::WithContinuousOutput
+      include ::Actions::Helpers::WithDelegatedAction
 
       middleware.do_not_use Dynflow::Middleware::Common::Transaction
-      include Actions::RemoteExecution::Helpers::LiveOutput
 
       def resource_locks
         :link
       end
 
-      def plan(job_invocation, host, template_invocation, proxy, options = {})
+      def plan(job_invocation, host, template_invocation, proxy_selector = ::RemoteExecutionProxySelector.new, options = {})
         action_subject(host, :job_category => job_invocation.job_category, :description => job_invocation.description)
 
         template_invocation.host_id = host.id
@@ -24,18 +25,21 @@ module Actions
 
         raise _('Could not use any template used in the job invocation') if template_invocation.blank?
 
-        if proxy.blank?
-          offline_proxies = options.fetch(:offline_proxies, [])
+        provider = template_invocation.template.provider_type.to_s
+        proxy = proxy_selector.determine_proxy(host, provider)
+        if proxy == :not_available
+          offline_proxies = proxy_selector.offline
           settings = { :count => offline_proxies.count, :proxy_names => offline_proxies.map(&:name).join(', ') }
           raise n_('The only applicable proxy %{proxy_names} is down',
                    'All %{count} applicable proxies are down. Tried %{proxy_names}',
-                   offline_proxies.count) % settings unless offline_proxies.empty?
-
-          settings = { :global_proxy   => 'remote_execution_global_proxy',
-                       :fallback_proxy => 'remote_execution_fallback_proxy' }
-
-          raise _('Could not use any proxy. Consider configuring %{global_proxy} ' +
-                  'or %{fallback_proxy} in settings') % settings
+                   offline_proxies.count) % settings
+        elsif proxy == :not_defined && !Setting['remote_execution_without_proxy']
+          settings = { :global_proxy => 'remote_execution_global_proxy',
+                       :fallback_proxy => 'remote_execution_fallback_proxy',
+                       :no_proxy => 'remote_execution_no_proxy' }
+
+          raise _('Could not use any proxy. Consider configuring %{global_proxy}, ' +
+                  '%{fallback_proxy} or %{no_proxy} in settings') % settings
         end
 
         renderer = InputTemplateRenderer.new(template_invocation.template, host, template_invocation)
@@ -43,28 +47,24 @@ module Actions
         raise _('Failed rendering template: %s') % renderer.error_message unless script
 
         provider = template_invocation.template.provider
-        plan_action(RunProxyCommand, proxy, hostname, script, provider.proxy_command_options(template_invocation, host))
+        action_options = provider.proxy_command_options(template_invocation, host).merge(:hostname => hostname, :script => script)
+        plan_delegated_action(proxy, ForemanRemoteExecutionCore::Actions::RunScript, action_options)
         plan_self
       end
 
       def finalize(*args)
-        host = Host.find(input[:host][:id])
-        host.refresh_statuses
-      rescue => e
-        ::Foreman::Logging.exception "Could not update execution status for #{input[:host][:name]}", e
+        check_exit_status
       end
 
-      def humanized_output
-        live_output.map { |line| line['output'].chomp }.join("\n")
+      def check_exit_status
+        if delegated_output[:exit_status].to_s != '0'
+          error! _('Playbook execution failed')
+        end
       end
 
       def live_output
-        proxy_command_action = planned_actions(RunProxyCommand).first
-        if proxy_command_action
-          proxy_command_action.live_output
-        else
-          execution_plan.errors.map { |e| exception_to_output(_('Failed to initialize command'), e) }
-        end
+        continuous_output.sort!
+        continuous_output.raw_outputs
       end
 
       def humanized_input
@@ -76,6 +76,43 @@ module Actions
         N_('Remote action:')
       end
 
+      def finalize
+        if exit_status.to_s != '0'
+          error! _('Playbook execution failed')
+        end
+      end
+
+      def rescue_strategy
+        ::Dynflow::Action::Rescue::Fail
+      end
+
+      def humanized_output
+        continuous_output.humanize
+      end
+
+      def continuous_output_providers
+        super << self
+      end
+
+      def fill_continuous_output(continuous_output)
+        fill_planning_errors_to_continuous_output(continuous_output) unless exit_status
+        delegated_output.fetch('result', []).each do |raw_output|
+          continuous_output.add_raw_output(raw_output)
+        end
+        final_timestamp = (continuous_output.last_timestamp || task.ended_at).to_f + 1
+        if exit_status
+          continuous_output.add_output(_('Exit status: %s') % exit_status, 'stdout', final_timestamp)
+        elsif run_step && run_step.error
+          continuous_output.add_ouput(_('Job finished with error') + ": #{run_step.error.exception_class} - #{run_step.error.message}", 'debug', final_timestamp)
+        end
+      rescue => e
+        continuous_output.add_exception(_('Error loading data from proxy'), e)
+      end
+
+      def exit_status
+        delegated_output[:exit_status]
+      end
+
       def find_ip_or_hostname(host)
         %w(execution primary provision).each do |flag|
           interface = host.send(flag + '_interface')
@@ -91,6 +128,19 @@ module Actions
 
       private
 
+      def delegated_output
+        if input[:delegated_action_id]
+          super
+        elsif phase?(Present)
+          # for compatibility with old actions
+          if old_action = all_planned_actions.first
+            old_action.output.fetch('proxy_output', {})
+          else
+            {}
+          end
+        end
+      end
+
       def verify_permissions(host, template_invocation)
         raise _('User can not execute job on host %s') % host.name unless User.current.can?(:view_hosts, host)
         raise _('User can not execute this job template') unless User.current.can?(:view_job_templates, template_invocation.template)

data/app/lib/actions/remote_execution/run_hosts_job.rb

@@ -23,16 +23,14 @@ module Actions
 
       def create_sub_plans
         job_invocation = JobInvocation.find(input[:job_invocation_id])
-        load_balancer = ProxyLoadBalancer.new
+        proxy_selector = RemoteExecutionProxySelector.new
 
         # composer creates just "pattern" for template_invocations because target is evaluated
         # during actual run (here) so we build template invocations from these patterns
         job_invocation.targeting.hosts.map do |host|
           template_invocation = job_invocation.pattern_template_invocation_for_host(host).deep_clone
           template_invocation.host_id = host.id
-          proxy = determine_proxy(template_invocation, host, load_balancer)
-          trigger(RunHostJob, job_invocation, host, template_invocation, proxy,
-                  :offline_proxies => load_balancer.offline)
+          trigger(RunHostJob, job_invocation, host, template_invocation, proxy_selector)
         end
       end
 
@@ -56,22 +54,6 @@ module Actions
       def humanized_name
         '%s:' % _(super)
       end
-
-      private
-
-      def determine_proxy(template_invocation, host, load_balancer)
-        provider = template_invocation.template.provider_type.to_s
-        host_proxies = host.remote_execution_proxies(provider)
-        strategies = [:subnet, :fallback, :global]
-        proxy = nil
-
-        strategies.each do |strategy|
-          proxy = load_balancer.next(host_proxies[strategy]) if host_proxies[strategy].present?
-          break if proxy
-        end
-
-        proxy
-      end
     end
   end
 end

data/app/models/host_status/execution_status.rb

@@ -10,7 +10,7 @@ class HostStatus::ExecutionStatus < HostStatus::Status
   # mapping to string representation
   STATUS_NAMES = { OK => 'succeeded', ERROR => 'failed', QUEUED => 'queued', RUNNING => 'running' }
 
-  def relevant?
+  def relevant?(*args)
     execution_tasks.present?
   end
 

data/app/models/setting/remote_execution.rb

@@ -14,6 +14,10 @@ class Setting::RemoteExecution < Setting
                  "If locations or organizations are enabled, the search will be limited to the host's " +
                  'organization or location.'),
                  true),
+        self.set('remote_execution_without_proxy',
+                 N_('When enabled, the remote execution will try to run the commands directly, when no
+                     proxy with remote execution feature is configured for the host.'),
+                 false),
         self.set('remote_execution_ssh_user',
                  N_('Default user to use for SSH.  You may override per host by setting a parameter called remote_execution_ssh_user.'),
                  'root'),

data/app/services/remote_execution_proxy_selector.rb

@@ -0,0 +1,5 @@
+class RemoteExecutionProxySelector < ::ForemanTasks::ProxySelector
+  def available_proxies(host, provider)
+    host.remote_execution_proxies(provider)
+  end
+end

data/foreman_remote_execution.gemspec

@@ -20,7 +20,8 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'deface'
   s.add_dependency 'dynflow', '~> 0.8.10'
-  s.add_dependency 'foreman-tasks', '0.8.0'
+  s.add_dependency 'foreman_remote_execution_core'
+  s.add_dependency 'foreman-tasks', '~> 0.8.1'
 
   s.add_development_dependency 'rubocop'
   s.add_development_dependency 'rdoc'

data/foreman_remote_execution_core.gemspec

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+require File.expand_path('../lib/foreman_remote_execution_core/version', __FILE__)
+require 'date'
+
+Gem::Specification.new do |s|
+  s.name        = 'foreman_remote_execution_core'
+  s.version     = ForemanRemoteExecutionCore::VERSION
+  s.authors     = ['Ivan Nečas']
+  s.email       = ['inecas@redhat.com']
+  s.homepage    = 'https://github.com/theforeman/foreman_remote_execution'
+  s.summary     = 'Foreman remote execution - core bits'
+  s.description = <<DESC
+  Ssh remote execution provider code sharable between Foreman and Foreman-Proxy
+DESC
+  s.licenses    = ['GPL-3']
+
+  s.files = Dir['lib/foreman_remote_execution_core/**/*'] +
+      ['lib/foreman_remote_execution_core.rb']
+
+  s.add_runtime_dependency('foreman-tasks-core', '~> 0.1.0')
+  s.add_runtime_dependency('net-ssh')
+  s.add_runtime_dependency('net-scp')
+end

data/lib/foreman_remote_execution/engine.rb

@@ -1,3 +1,5 @@
+require 'foreman_remote_execution_core'
+
 module ForemanRemoteExecution
   class Engine < ::Rails::Engine
     engine_name 'foreman_remote_execution'

data/lib/foreman_remote_execution/version.rb

@@ -1,3 +1,3 @@
 module ForemanRemoteExecution
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

data/lib/foreman_remote_execution_core.rb

@@ -0,0 +1,18 @@
+require 'foreman_tasks_core'
+
+module ForemanRemoteExecutionCore
+  extend ForemanTasksCore::SettingsLoader
+  register_settings([:remote_execution_ssh, :smart_proxy_remote_execution_ssh_core],
+                    :ssh_identity_key_file => '~/.ssh/id_rsa_foreman_proxy',
+                    :ssh_user              => 'root',
+                    :remote_working_dir    => '/var/tmp',
+                    :local_working_dir     => '/var/tmp')
+
+  if ForemanTasksCore.dynflow_present?
+    require 'foreman_tasks_core/runner'
+    require 'foreman_remote_execution_core/script_runner'
+    require 'foreman_remote_execution_core/actions'
+  end
+
+  require 'foreman_remote_execution_core/version'
+end

data/lib/foreman_remote_execution_core/actions.rb

@@ -0,0 +1,11 @@
+require 'foreman_tasks_core/shareable_action'
+
+module ForemanRemoteExecutionCore
+  module Actions
+    class RunScript < ForemanTasksCore::Runner::Action
+      def initiate_runner
+        ScriptRunner.new(input)
+      end
+    end
+  end
+end

data/lib/foreman_remote_execution_core/script_runner.rb

@@ -0,0 +1,266 @@
+require 'net/ssh'
+require 'net/scp'
+
+module ForemanRemoteExecutionCore
+  class ScriptRunner < ForemanTasksCore::Runner::Base
+    EXPECTED_POWER_ACTION_MESSAGES = ["restart host", "shutdown host"]
+
+    def initialize(options)
+      super()
+      @host = options.fetch(:hostname)
+      @script = options.fetch(:script)
+      @ssh_user = options.fetch(:ssh_user, 'root')
+      @ssh_port = options.fetch(:ssh_port, 22)
+      @effective_user = options.fetch(:effective_user, nil)
+      @effective_user_method = options.fetch(:effective_user_method, 'sudo')
+      @host_public_key = options.fetch(:host_public_key, nil)
+      @verify_host = options.fetch(:verify_host, nil)
+
+      @client_private_key_file = settings.fetch(:ssh_identity_key_file)
+      @local_working_dir = options.fetch(:local_working_dir, settings.fetch(:local_working_dir))
+      @remote_working_dir = options.fetch(:remote_working_dir, settings.fetch(:remote_working_dir))
+    end
+
+    def start
+      remote_script = cp_script_to_remote
+      output_path = File.join(File.dirname(remote_script), 'output')
+
+      # pipe the output to tee while capturing the exit code
+      script = <<-SCRIPT
+          exec 4>&1
+          exit_code=`((#{su_prefix}#{remote_script}; echo $?>&3 ) | /usr/bin/tee #{output_path} ) 3>&1 >&4`
+          exec 4>&-
+          exit $exit_code
+      SCRIPT
+      logger.debug("executing script:\n#{script.lines.map { |line| "  | #{line}" }.join}")
+      run_async(script)
+    rescue => e
+      logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
+      publish_exception("Error initializing command", e)
+    end
+
+    def refresh
+      return if @session.nil?
+      with_retries do
+        with_disconnect_handling do
+          @session.process(0)
+        end
+      end
+    ensure
+      check_expecting_disconnect
+    end
+
+    def kill
+      if @session
+        run_sync("pkill -f #{remote_command_file('script')}")
+      else
+        logger.debug("connection closed")
+      end
+    rescue => e
+      publish_exception("Unexpected error", e, false)
+    end
+
+    def with_retries
+      tries = 0
+      begin
+        yield
+      rescue => e
+        logger.error("Unexpected error: #{e.class} #{e.message}\n #{e.backtrace.join("\n")}")
+        tries += 1
+        if tries <= MAX_PROCESS_RETRIES
+          logger.error('Retrying')
+          retry
+        else
+          publish_exception("Unexpected error", e)
+        end
+      end
+    end
+
+    def with_disconnect_handling
+      yield
+    rescue Net::SSH::Disconnect => e
+      @session.shutdown!
+      check_expecting_disconnect
+      if @expecting_disconnect
+        publish_exit_status(0)
+      else
+        publish_exception("Unexpected disconnect", e)
+      end
+    end
+
+    def close
+      @session.close if @session && !@session.closed?
+    end
+
+    private
+
+    def session
+      @session ||= begin
+                     @logger.debug("opening session to #{@ssh_user}@#{@host}")
+                     Net::SSH.start(@host, @ssh_user, ssh_options)
+                   end
+    end
+
+    def ssh_options
+      ssh_options = {}
+      ssh_options[:port] = @ssh_port if @ssh_port
+      ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
+      ssh_options[:user_known_hosts_file] = @known_hosts_file if @known_hosts_file
+      ssh_options[:keys_only] = true
+      # if the host public key is contained in the known_hosts_file,
+      # verify it, otherwise, if missing, import it and continue
+      ssh_options[:paranoid] = true
+      ssh_options[:auth_methods] = ["publickey"]
+      ssh_options[:user_known_hosts_file] = prepare_known_hosts if @host_public_key
+      return ssh_options
+    end
+
+    def settings
+      ForemanRemoteExecutionCore.settings
+    end
+
+    # Initiates run of the remote command and yields the data when
+    # available. The yielding doesn't happen automatically, but as
+    # part of calling the `refresh` method.
+    def run_async(command)
+      raise "Async command already in progress" if @started
+      @started = false
+      session.open_channel do |channel|
+        channel.request_pty
+        channel.on_data { |ch, data| publish_data(data, 'stdout') }
+        channel.on_extended_data { |ch, type, data| publish_data(data, 'stderr') }
+        # standard exit of the command
+        channel.on_request("exit-status") { |ch, data| publish_exit_status(data.read_long) }
+        # on signal: sending the signal value (such as 'TERM')
+        channel.on_request("exit-signal") do |ch, data|
+          publish_exit_status(data.read_string)
+          ch.close
+          # wait for the channel to finish so that we know at the end
+          # that the session is inactive
+          ch.wait
+        end
+        channel.exec(command) do |ch, success|
+          @started = true
+          raise("Error initializing command") unless success
+        end
+      end
+      session.process(0) until @started
+      return true
+    end
+
+    def run_sync(command)
+      output = ""
+      exit_status = nil
+      channel = session.open_channel do |ch|
+        ch.on_data { |data| output.concat(data) }
+        ch.on_extended_data { |_, _, data| output.concat(data) }
+        ch.on_request("exit-status") { |_, data| exit_status = data.read_long }
+        # on signal: sending the signal value (such as 'TERM')
+        ch.on_request("exit-signal") do |_, data|
+          exit_status = data.read_string
+          ch.close
+          ch.wait
+        end
+        ch.exec command do |_, success|
+          raise "could not execute command" unless success
+        end
+      end
+      channel.wait
+      return exit_status, output
+    end
+
+    def su_prefix
+      return if @effective_user.nil? || @effective_user == @ssh_user
+      case @effective_user_method
+      when 'sudo'
+        "sudo -n -u #{@effective_user} "
+      when 'su'
+        "su - #{@effective_user} -c "
+      else
+        raise "effective_user_method ''#{@effective_user_method}'' not supported"
+      end
+    end
+
+    def prepare_known_hosts
+      path = local_command_file('known_hosts')
+      if @host_public_key
+        write_command_file_locally('known_hosts', "#{@host} #{@host_public_key}")
+      end
+      return path
+    end
+
+    def local_command_dir
+      File.join(@local_working_dir, 'foreman-proxy', "foreman-ssh-cmd-#{@id}")
+    end
+
+    def local_command_file(filename)
+      File.join(local_command_dir, filename)
+    end
+
+    def remote_command_dir
+      File.join(@remote_working_dir, "foreman-ssh-cmd-#{id}")
+    end
+
+    def remote_command_file(filename)
+      File.join(remote_command_dir, filename)
+    end
+
+    def ensure_local_directory(path)
+      if File.exist?(path)
+        raise "#{path} expected to be a directory" unless File.directory?(path)
+      else
+        FileUtils.mkdir_p(path)
+      end
+      return path
+    end
+
+    def cp_script_to_remote
+      local_script_file = write_command_file_locally('script', sanitize_script(@script))
+      File.chmod(0555, local_script_file)
+      remote_script_file = remote_command_file('script')
+      upload_file(local_script_file, remote_script_file)
+      return remote_script_file
+    end
+
+    def upload_file(local_path, remote_path)
+      ensure_remote_directory(File.dirname(remote_path))
+      scp = Net::SCP.new(session)
+      upload_channel = scp.upload(local_path, remote_path)
+      upload_channel.wait
+    ensure
+      if upload_channel
+        upload_channel.close
+        upload_channel.wait
+      end
+    end
+
+    def ensure_remote_directory(path)
+      exit_code, output = run_sync("mkdir -p #{path}")
+      if exit_code != 0
+        raise "Unable to create directory on remote system #{path}: exit code: #{exit_code}\n #{output}"
+      end
+    end
+
+    def sanitize_script(script)
+      script.tr("\r", '')
+    end
+
+    def write_command_file_locally(filename, content)
+      path = local_command_file(filename)
+      ensure_local_directory(File.dirname(path))
+      File.write(path, content)
+      return path
+    end
+
+    # when a remote server disconnects, it's hard to tell if it was on purpose (when calling reboot)
+    # or it's an error. When it's expected, we expect the script to produce 'restart host' as
+    # its last command output
+    def check_expecting_disconnect
+      last_output = @continuous_output.raw_outputs.find { |d| d['output_type'] == 'stdout' }
+      return unless last_output
+      if EXPECTED_POWER_ACTION_MESSAGES.any? { |message| last_output['output'] =~ /^#{message}/ }
+        @expecting_disconnect = true
+      end
+    end
+  end
+end

data/lib/foreman_remote_execution_core/version.rb

@@ -0,0 +1,3 @@
+module ForemanRemoteExecutionCore
+  VERSION = '1.0.0'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman_remote_execution
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Foreman Remote Execution team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-07 00:00:00.000000000 Z
+date: 2016-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: deface
@@ -38,20 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.10
+- !ruby/object:Gem::Dependency
+  name: foreman_remote_execution_core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: foreman-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -160,10 +174,8 @@ files:
 - app/helpers/concerns/foreman_remote_execution/job_templates_extensions.rb
 - app/helpers/remote_execution_helper.rb
 - app/lib/actions/middleware/bind_job_invocation.rb
-- app/lib/actions/remote_execution/helpers/live_output.rb
 - app/lib/actions/remote_execution/run_host_job.rb
 - app/lib/actions/remote_execution/run_hosts_job.rb
-- app/lib/actions/remote_execution/run_proxy_command.rb
 - app/lib/proxy_api/remote_execution_ssh.rb
 - app/mailers/.gitkeep
 - app/models/concerns/foreman_remote_execution/bookmark_extensions.rb
@@ -198,7 +210,7 @@ files:
 - app/models/template_invocation_input_value.rb
 - app/overrides/execution_interface.rb
 - app/overrides/subnet_proxies.rb
-- app/services/proxy_load_balancer.rb
+- app/services/remote_execution_proxy_selector.rb
 - app/views/api/v2/foreign_input_sets/base.json.rabl
 - app/views/api/v2/foreign_input_sets/create.json.rabl
 - app/views/api/v2/foreign_input_sets/index.json.rabl
@@ -339,9 +351,14 @@ files:
 - doc/source/static/js/jquery.tocify.min.js
 - doc/source/static/js/scroll.js
 - foreman_remote_execution.gemspec
+- foreman_remote_execution_core.gemspec
 - lib/foreman_remote_execution.rb
 - lib/foreman_remote_execution/engine.rb
 - lib/foreman_remote_execution/version.rb
+- lib/foreman_remote_execution_core.rb
+- lib/foreman_remote_execution_core/actions.rb
+- lib/foreman_remote_execution_core/script_runner.rb
+- lib/foreman_remote_execution_core/version.rb
 - lib/tasks/foreman_remote_execution_tasks.rake
 - locale/Makefile
 - locale/action_names.rb
@@ -378,7 +395,6 @@ files:
 - test/test_plugin_helper.rb
 - test/unit/actions/run_host_job_test.rb
 - test/unit/actions/run_hosts_job_test.rb
-- test/unit/actions/run_proxy_command_test.rb
 - test/unit/concerns/exportable_test.rb
 - test/unit/concerns/host_extensions_test.rb
 - test/unit/concerns/nic_extensions_test.rb
@@ -388,7 +404,6 @@ files:
 - test/unit/job_invocation_test.rb
 - test/unit/job_template_effective_user_test.rb
 - test/unit/job_template_test.rb
-- test/unit/proxy_load_balancer_test.rb
 - test/unit/remote_execution_feature_test.rb
 - test/unit/remote_execution_provider_test.rb
 - test/unit/targeting_test.rb
@@ -428,7 +443,6 @@ test_files:
 - test/test_plugin_helper.rb
 - test/unit/actions/run_host_job_test.rb
 - test/unit/actions/run_hosts_job_test.rb
-- test/unit/actions/run_proxy_command_test.rb
 - test/unit/concerns/exportable_test.rb
 - test/unit/concerns/host_extensions_test.rb
 - test/unit/concerns/nic_extensions_test.rb
@@ -438,7 +452,6 @@ test_files:
 - test/unit/job_invocation_test.rb
 - test/unit/job_template_effective_user_test.rb
 - test/unit/job_template_test.rb
-- test/unit/proxy_load_balancer_test.rb
 - test/unit/remote_execution_feature_test.rb
 - test/unit/remote_execution_provider_test.rb
 - test/unit/targeting_test.rb

data/app/lib/actions/remote_execution/helpers/live_output.rb

@@ -1,17 +0,0 @@
-module Actions
-  module RemoteExecution
-    module Helpers
-      module LiveOutput
-        def exception_to_output(context, exception, timestamp = Time.now.getlocal)
-          format_output(context + ": #{exception.class} - #{exception.message}", 'debug', timestamp)
-        end
-
-        def format_output(message, type = 'debug', timestamp = Time.now.getlocal)
-          { 'output_type' => type,
-            'output' => message,
-            'timestamp' => timestamp.to_f }
-        end
-      end
-    end
-  end
-end

data/app/lib/actions/remote_execution/run_proxy_command.rb

@@ -1,96 +0,0 @@
-module Actions
-  module RemoteExecution
-    class RunProxyCommand < Actions::ProxyAction
-
-      include ::Dynflow::Action::Cancellable
-      include Actions::RemoteExecution::Helpers::LiveOutput
-
-      def plan(proxy, hostname, script, options = {})
-        options = { :effective_user => nil }.merge(options)
-        super(proxy, options.merge(:hostname => hostname, :script => script))
-      end
-
-      def proxy_action_name
-        'Proxy::RemoteExecution::Ssh::CommandAction'
-      end
-
-      def on_data(data)
-        if data[:result] == 'initialization_error'
-          handle_connection_exception(data[:metadata][:exception_class]
-                                       .constantize
-                                       .new(data[:metadata][:exception_message]))
-        else
-          super(data)
-          error! _('Script execution failed') if failed_run?
-        end
-      end
-
-
-      def rescue_strategy
-        ::Dynflow::Action::Rescue::Skip
-      end
-
-      def failed_run?
-        output[:result] == 'initialization_error' ||
-          (exit_status && proxy_output[:exit_status] != 0)
-      end
-
-      def exit_status
-        proxy_output && proxy_output[:exit_status]
-      end
-
-      def live_output
-        records = connection_messages
-        if !task.pending?
-          records.concat(finalized_output)
-        else
-          records.concat(current_proxy_output)
-        end
-        records.sort_by { |record| record['timestamp'].to_f }
-      end
-
-      private
-
-      def connection_messages
-        metadata.fetch(:failed_proxy_tasks, []).map do |failure_data|
-          format_output(_('Initialization error: %s') % "#{failure_data[:exception_class]} - #{failure_data[:exception_message]}", 'debug', failure_data[:timestamp])
-        end
-      end
-
-      def current_proxy_output
-        return [] unless output[:proxy_task_id]
-        proxy_data = proxy.status_of_task(output[:proxy_task_id])['actions'].detect { |action| action['class'] == proxy_action_name }
-        proxy_data.fetch('output', {}).fetch('result', [])
-      rescue => e
-        ::Foreman::Logging.exception("Failed to load data for task #{task.id} from proxy #{input[:proxy_url]}", e)
-        [exception_to_output(_('Error loading data from proxy'), e)]
-      end
-
-      def finalized_output
-        records = []
-
-        if proxy_result.present?
-          records.concat(proxy_result)
-        else
-          records << format_output(_('No output'))
-        end
-
-        if exit_status
-          records << format_output(_('Exit status: %s') % exit_status, 'stdout', final_timestamp(records))
-        elsif run_step && run_step.error
-          records << format_output(_('Job finished with error') + ": #{run_step.error.exception_class} - #{run_step.error.message}", 'debug', final_timestamp(records))
-        end
-        return records
-      end
-
-      def final_timestamp(records)
-        return task.ended_at if records.blank?
-        records.last.fetch('timestamp', task.ended_at).to_f + 1
-      end
-
-      def proxy_result
-        self.output.fetch(:proxy_output, {}).fetch(:result, []) || []
-      end
-    end
-  end
-end

data/app/services/proxy_load_balancer.rb

@@ -1,33 +0,0 @@
-class ProxyLoadBalancer
-
-  attr_reader :offline
-
-  def initialize
-    @tasks   = {}
-    @offline = []
-  end
-
-  # Get the least loaded proxy from the given list of proxies
-  def next(proxies)
-    exclude = @tasks.keys + @offline
-    @tasks.merge!(get_counts(proxies - exclude))
-    next_proxy = @tasks.select { |proxy, _| proxies.include?(proxy) }.min_by { |_, job_count| job_count }.try(:first)
-    @tasks[next_proxy] += 1 if next_proxy.present?
-    next_proxy
-  end
-
-  private
-
-  def get_counts(proxies)
-    proxies.inject({}) do |result, proxy|
-      begin
-        proxy_api = ProxyAPI::ForemanDynflow::DynflowProxy.new(:url => proxy.url)
-        result[proxy] = proxy_api.tasks_count('running')
-      rescue => e
-        @offline << proxy
-        Foreman::Logging.exception "Could not fetch task counts from #{proxy}, skipped.", e
-      end
-      result
-    end
-  end
-end

data/test/unit/actions/run_proxy_command_test.rb

@@ -1,134 +0,0 @@
-require 'test_plugin_helper'
-
-module ForemanRemoteExecution
-  class RunProxyCommandTest < ActiveSupport::TestCase
-    include Dynflow::Testing
-
-    let(:host) { FactoryGirl.build(:host, :with_execution) }
-    let(:proxy) { host.remote_execution_proxies('SSH')[:subnet].first }
-    let(:hostname) { 'myhost.example.com' }
-    let(:script) { 'ping -c 5 redhat.com' }
-    let(:connection_options) { { 'retry_interval' => 15, 'retry_count' => 4, 'timeout' => 60 } }
-    let(:action) do
-      create_and_plan_action(Actions::RemoteExecution::RunProxyCommand, proxy, host.name, script)
-    end
-    let(:timestamp) { 1_443_194_805.9192207 }
-
-    it 'plans for running the command action on server' do
-      assert_run_phase action, { :hostname       => host.name,
-                                 :script         => script,
-                                 :proxy_url      => proxy.url,
-                                 :effective_user => nil,
-                                 :connection_options => connection_options }
-    end
-
-    it 'sends to command to ssh provider' do
-      action.proxy_action_name.must_equal 'Proxy::RemoteExecution::Ssh::CommandAction'
-    end
-
-    it "doesn't block on failure" do
-      action.rescue_strategy.must_equal ::Dynflow::Action::Rescue::Skip
-    end
-
-    describe '#live_output' do
-      let(:task) { ForemanTasks::Task.new }
-
-      let(:action) do
-        planned_action = create_and_plan_action(Actions::RemoteExecution::RunProxyCommand, proxy, hostname, script)
-        create_action_presentation(Actions::RemoteExecution::RunProxyCommand).tap do |action|
-          action.stubs(:task).returns(task)
-          action.stubs(:proxy).returns(ProxyAPI::ForemanDynflow::DynflowProxy.new(:url => proxy.url))
-          action.instance_variable_set('@input', planned_action.input)
-          action.output.merge!(:proxy_task_id => '123')
-        end
-      end
-
-      let(:live_output) do
-        action.live_output
-      end
-
-      describe 'when the task is finished' do
-        before do
-          task.state = 'stopped'
-          task.ended_at = Time.at(timestamp + 1).utc
-        end
-
-        describe 'the task finished sucessfully' do
-          before do
-            action.output.merge!(:proxy_output => { :result => [{ 'output_type' => 'stdout', 'output' => 'Hello', 'timestamp' => timestamp}],
-                                                    :exit_status => 0 })
-          end
-
-          it "doesn't fetch data from proxy anymore" do
-            action.proxy.expects(:status_of_task).never
-            live_output.size.must_equal 2
-            live_output[0]['output_type'].must_equal 'stdout'
-            live_output[0]['output'].must_equal 'Hello'
-            live_output[0]['timestamp'].must_be_kind_of Float
-            live_output[1]['output_type'].must_equal 'stdout'
-            live_output[1]['output'].must_equal 'Exit status: 0'
-            live_output[1]['timestamp'].must_be_kind_of Float
-          end
-        end
-
-        describe 'there was not output data from proxy' do
-          before do
-            action.output.merge!(:proxy_output => {})
-          end
-
-          it "doesn't fetch data from proxy anymore" do
-            action.proxy.expects(:status_of_task).never
-            live_output.size.must_equal 1
-            live_output[0]['output_type'].must_equal 'debug'
-            live_output[0]['output'].must_equal 'No output'
-            live_output[0]['timestamp'].must_be_kind_of Float
-          end
-        end
-      end
-
-      describe 'when something went wrong while fetching the data' do
-        before do
-          action.proxy.expects(:status_of_task).raises('Something went wrong')
-        end
-
-        it 'reports the failure as part of the live output' do
-          live_output.size.must_equal 1
-          live_output.first['output_type'].must_equal 'debug'
-          live_output.first['output'].must_equal 'Error loading data from proxy: RuntimeError - Something went wrong'
-          live_output.first['timestamp'].must_be_kind_of Float
-        end
-      end
-
-      describe 'when there was some connection error while running the command' do
-        before do
-          action.output.merge!(:proxy_task_id => nil,
-                               :metadata => { :failed_proxy_tasks => [action.send(:format_exception, RuntimeError.new('Connection error'))]})
-        end
-
-        it 'reports the failure as part of the live output' do
-          live_output.size.must_equal 1
-          live_output.first['output_type'].must_equal 'debug'
-          live_output.first['output'].must_equal 'Initialization error: RuntimeError - Connection error'
-          live_output.first['timestamp'].must_be_kind_of Float
-        end
-      end
-
-      describe 'when proxy returns valid data' do
-        before do
-          action.proxy.expects(:status_of_task).returns('actions' =>
-                                                        [{ 'class' => 'Proxy::RemoteExecution::Ssh::CommandAction',
-                                                           'output' => { 'result' => [ { 'output_type' => 'stdout',
-                                                                                         'output' => 'Hello',
-                                                                                         'timestamp' => timestamp }]}}])
-        end
-
-        it 'reports the failure as part of the live output' do
-          live_output.size.must_equal 1
-          live_output.first['output_type'].must_equal 'stdout'
-          live_output.first['output'].must_equal 'Hello'
-          live_output.first['timestamp'].must_be_kind_of Float
-        end
-      end
-    end
-  end
-end

data/test/unit/proxy_load_balancer_test.rb

@@ -1,27 +0,0 @@
-require 'test_plugin_helper'
-
-describe ProxyLoadBalancer do
-  let(:load_balancer) { ProxyLoadBalancer.new }
-
-  before do
-    ProxyAPI::ForemanDynflow::DynflowProxy.any_instance.stubs(:tasks_count).returns(0)
-  end
-
-  it 'load balances' do
-    count = 3
-    ProxyAPI::ForemanDynflow::DynflowProxy.any_instance.expects(:tasks_count).raises.then.times(count - 1).returns(0)
-    proxies = FactoryGirl.create_list(:smart_proxy, count, :ssh)
-
-    available = proxies.reduce([]) do |found, _|
-      found << load_balancer.next(proxies)
-    end
-
-    available.count.must_equal count
-    available.uniq.count.must_equal count - 1
-    load_balancer.offline.count.must_equal 1
-  end
-
-  it 'returns nil for if no proxy is available' do
-    load_balancer.next([]).must_be_nil
-  end
-end