foreman_remote_execution_core 1.1.4 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e57511a4f21ec8c2648077cad42df197263a300a5f4cdbc76572d5548e32e4b2
-  data.tar.gz: ef5e95f9e50e8f94058404e9208069089e9990b2a32af1329eb54026726becbe
+  metadata.gz: e1576329ece4e20e9506b989b77557686d030c329c71f70386c7415e93a73e24
+  data.tar.gz: 4021d34ddbaa469bb1f4152c0a3c404c976a97910422a6fd60868661992ab774
 SHA512:
-  metadata.gz: 8ed29e3e1eb432b340d335d4e5419293d412337fb88ed76fa6fdff2e69e2971ab503ea741a9a0b9d67d0e37feb9da44906e1c42efec46667a631e4ec790813dc
-  data.tar.gz: b9e438caaeb891d5954ccbf78893620a123b785ba4de7f1ea8001fc016cea6b335ee35520c356b2f0ac9258bb2f5729d491916e355a4978c2fd77477c52e45fd
+  metadata.gz: b0ce710cc252baca86acb51cfe294b76f8de319b5516bc8f303252fff52a8ef1e074c34ea00f7bd179b2811dc85daf292035a2736cb421046aa46e2a37137d73
+  data.tar.gz: 65092102cb80436b6de9ce165905e8d87c387bf70256b244224d780cd2335f43b0323a06afed5c72e78fa8297566dcbe01f0624dee89b25645d8e2bf292d91a4

data/lib/foreman_remote_execution_core.rb

@@ -3,16 +3,16 @@ require 'foreman_tasks_core'
 module ForemanRemoteExecutionCore
   extend ForemanTasksCore::SettingsLoader
   register_settings([:remote_execution_ssh, :smart_proxy_remote_execution_ssh_core],
-                    :ssh_identity_key_file   => '~/.ssh/id_rsa_foreman_proxy',
-                    :ssh_user                => 'root',
-                    :remote_working_dir      => '/var/tmp',
-                    :local_working_dir       => '/var/tmp',
-                    :kerberos_auth           => false,
-                    :async_ssh               => false,
-                    # When set to nil, makes REX use the runner's default interval
-                    :runner_refresh_interval => nil,
-                    :ssh_log_level           => :fatal,
-                    :cleanup_working_dirs    => true)
+    :ssh_identity_key_file   => '~/.ssh/id_rsa_foreman_proxy',
+    :ssh_user                => 'root',
+    :remote_working_dir      => '/var/tmp',
+    :local_working_dir       => '/var/tmp',
+    :kerberos_auth           => false,
+    :async_ssh               => false,
+    # When set to nil, makes REX use the runner's default interval
+    :runner_refresh_interval => nil,
+    :ssh_log_level           => :fatal,
+    :cleanup_working_dirs    => true)
 
   SSH_LOG_LEVELS = %w(debug info warn error fatal).freeze
 
@@ -32,7 +32,9 @@ module ForemanRemoteExecutionCore
       raise "Wrong value '#{@settings[:ssh_log_level]}' for ssh_log_level, must be one of #{SSH_LOG_LEVELS.join(', ')}"
     end
 
-    current = if defined?(SmartProxyDynflowCore)
+    current = if defined?(::Proxy::SETTINGS)
+                ::Proxy::SETTINGS.log_level.to_s.downcase
+              elsif defined?(SmartProxyDynflowCore)
                 SmartProxyDynflowCore::SETTINGS.log_level.to_s.downcase
               else
                 Rails.configuration.log_level.to_s
@@ -70,6 +72,10 @@ module ForemanRemoteExecutionCore
     end
     require 'foreman_remote_execution_core/dispatcher'
     require 'foreman_remote_execution_core/actions'
+
+    if defined?(::SmartProxyDynflowCore)
+      SmartProxyDynflowCore::TaskLauncherRegistry.register('ssh', ForemanTasksCore::TaskLauncher::Batch)
+    end
   end
 
   require 'foreman_remote_execution_core/version'

data/lib/foreman_remote_execution_core/actions.rb

@@ -6,9 +6,10 @@ module ForemanRemoteExecutionCore
       def initiate_runner
         additional_options = {
           :step_id => run_step_id,
-          :uuid => execution_plan_id
+          :uuid => execution_plan_id,
         }
-        ForemanRemoteExecutionCore.runner_class.build(input.merge(additional_options))
+        ForemanRemoteExecutionCore.runner_class.build(input.merge(additional_options),
+          suspended_action: suspended_action)
       end
 
       def runner_dispatcher

data/lib/foreman_remote_execution_core/async_scripts/control.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# Control script for the remote execution jobs.
+#
+# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
+# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
+#
+# What the script provides is also a manual mode, where the author of the rex script can take
+# full control of the job lifecycle. This allows keeping the marked as running even when
+# the initial script finishes.
+#
+# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
+# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+if ! command -v curl >/dev/null; then
+    echo 'curl is required' >&2
+    exit 1
+fi
+
+# send the callback data to proxy
+update() {
+    "$BASE_DIR/retrieve.sh" push_update
+}
+
+# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
+# in seconds on how long to wait when reading.
+wait_for_pipe() {
+    pipe_path=$1
+    if [ -n "$2" ]; then
+        timeout="-t $2"
+    fi
+    if read $timeout <>"$pipe_path"; then
+        rm "$pipe_path"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# function run in background, when receiving update data via STDIN.
+periodic_update() {
+    interval=1
+    # reading some data from periodic_update_control signals we're done
+    while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
+        update
+    done
+    # one more update before we finish
+    update
+    # signal the main process that we are finished
+    echo > "$BASE_DIR/periodic_update_finished"
+}
+
+# signal the periodic_update process that the main process is finishing
+periodic_update_finish() {
+    if [ -e "$BASE_DIR/periodic_update_control" ]; then
+       echo > "$BASE_DIR/periodic_update_control"
+    fi
+}
+
+ACTION=${1:-finish}
+
+case "$ACTION" in
+    init-script-finish)
+        if ! [ -e "$BASE_DIR/manual_mode" ]; then
+            # make the exit code of initialization script the exit code of the whole job
+            cp init_exit_code exit_code
+            update
+        fi
+        ;;
+    finish)
+        # take exit code passed via the command line, with fallback
+        # to the exit code of the initialization script
+        exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
+        echo $exit_code > "$BASE_DIR/exit_code"
+        update
+        if [ -e "$BASE_DIR/manual_mode" ]; then
+            rm "$BASE_DIR/manual_mode"
+        fi
+        ;;
+    update)
+        # read data from input when redirected though a pipe
+        if ! [ -t 0 ]; then
+            # couple of named pipes to coordinate the main process with the periodic_update
+            mkfifo "$BASE_DIR/periodic_update_control"
+            mkfifo "$BASE_DIR/periodic_update_finished"
+            trap "periodic_update_finish" EXIT
+            # run periodic update as separate process to keep sending updates in output to server
+            periodic_update &
+            # redirect the input into output
+            tee -a "$BASE_DIR/output"
+            periodic_update_finish
+            # ensure the periodic update finished before we return
+            wait_for_pipe "$BASE_DIR/periodic_update_finished"
+        else
+            update
+        fi
+        ;;
+    # mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
+    # commands to send output to the remote execution job or mark it as finished.
+    manual-mode)
+        touch "$BASE_DIR/manual_mode"
+        ;;
+    *)
+        echo "Unknown action $ACTION"
+        exit 1
+        ;;
+esac

data/lib/foreman_remote_execution_core/async_scripts/retrieve.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+if ! pgrep --help 2>/dev/null >/dev/null; then
+    echo DONE 1
+    echo "pgrep is required" >&2
+    exit 1
+fi
+
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+# load the data required for generating the callback
+. "$BASE_DIR/env.sh"
+URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
+AUTH="$TASK_ID:$OTP"
+CURL="curl --silent --show-error --fail --max-time 10"
+
+MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
+MY_PID=$$
+echo $MY_PID >"$MY_LOCK_FILE"
+LOCK_FILE="$BASE_DIR/retrieve_lock"
+TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
+
+RUN_TIMEOUT=30 # for how long can the script hold the lock
+WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
+START_TIME=$(date +%s)
+
+fail() {
+    echo RUNNING
+    echo "$1"
+    exit 1
+}
+
+acquire_lock() {
+    # try to acquire lock by creating the file (ln should be atomic an fail in case
+    # another process succeeded first). We also check the content of the lock file,
+    # in case our process won when competing over the lock while invalidating
+    # the lock on timeout.
+    ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
+    return $?
+}
+
+# acquiring the lock before proceeding, to ensure only one instance of the script is running
+while ! acquire_lock; do
+    # we failed to create retrieve_lock - assuming there is already another retrieve script running
+    current_pid=$(head -n1 "$LOCK_FILE")
+    if [ -z "$current_pid" ]; then
+        continue
+    fi
+    # check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
+    # if it is, so that we don't have a stalled processes here
+    lock_lines_count=$(wc -l < "$LOCK_FILE")
+    current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
+    current_time=$(date +%s)
+
+    if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
+        # We were waiting for the lock for too long - just give up
+        fail "Wait time exceeded $WAIT_TIMEOUT"
+    elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
+        # The previous lock it hold for too long - re-acquiring procedure
+        if [ "$lock_lines_count" -gt 1 ]; then
+           # there were multiple processes waiting for lock without resolution
+           # longer than the $RUN_TIMEOUT - we reset the lock file and let processes
+           # to compete
+           echo "RETRY" > "$LOCK_FILE"
+        fi
+        if [ "$current_pid" != "RETRY" ]; then
+            # try to kill the currently stalled process
+            kill -9 "$current_pid" 2>/dev/null
+        fi
+        # try to add our process as one candidate
+        echo $MY_PID >> "$LOCK_FILE"
+        if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
+            # our process won the competition for the new lock: it is the first pid
+            # after the original one in the lock file - take ownership of the lock
+            # next iteration only this process will get through
+            echo $MY_PID >"$LOCK_FILE"
+        fi
+    else
+        # still waiting for the original owner to finish
+        sleep 1
+    fi
+done
+
+release_lock() {
+    rm "$MY_LOCK_FILE"
+    rm "$LOCK_FILE"
+}
+# ensure the release the lock at exit
+trap "release_lock" EXIT
+
+# make sure we clear previous tmp output file
+if [ -e "$TMP_OUTPUT_FILE" ]; then
+    rm "$TMP_OUTPUT_FILE"
+fi
+
+pid=$(cat "$BASE_DIR/pid")
+[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
+position=$(cat "$BASE_DIR/position")
+
+prepare_output() {
+    if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
+        echo RUNNING
+    else
+        echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
+    fi
+    [ -f "$BASE_DIR/output" ] || exit 0
+    tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
+    cat "$TMP_OUTPUT_FILE"
+}
+
+# prepare the callback payload
+payload() {
+    if [ -n "$1" ]; then
+        exit_code="$1"
+    else
+        exit_code=null
+    fi
+
+    if [ -e "$BASE_DIR/manual_mode" ]; then
+        manual_mode=true
+        output=$(prepare_output | base64 -w0)
+    else
+        manual_mode=false
+    fi
+
+    echo "{ \"exit_code\": $exit_code,"\
+         "  \"step_id\": \"$STEP_ID\","\
+         "  \"manual_mode\": $manual_mode,"\
+         "  \"output\": \"$output\" }"
+}
+
+if [ "$1" = "push_update" ]; then
+    if [ -e "$BASE_DIR/exit_code" ]; then
+        exit_code="$(cat "$BASE_DIR/exit_code")"
+        action="done"
+    else
+        exit_code=""
+        action="update"
+    fi
+    $CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
+    success=$?
+else
+    prepare_output
+    success=$?
+fi
+
+if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
+    # in case the retrieval was successful, move the position of the cursor to be read next time
+    bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
+    expr "${position}" + "${bytes}" > "$BASE_DIR/position"
+fi

data/lib/foreman_remote_execution_core/fake_script_runner.rb

@@ -24,8 +24,8 @@ module ForemanRemoteExecutionCore
         @data.freeze
       end
 
-      def build(options)
-        new(options)
+      def build(options, suspended_action:)
+        new(options, suspended_action: suspended_action)
       end
     end
 

data/lib/foreman_remote_execution_core/log_filter.rb

@@ -4,10 +4,11 @@ module ForemanRemoteExecutionCore
       @base_logger = base_logger
     end
 
-    def add(severity, *args)
+    def add(severity, *args, &block)
       severity ||= ::Logger::UNKNOWN
       return true if @base_logger.nil? || severity < @level
-      @base_logger.add(severity, *args)
+
+      @base_logger.add(severity, *args, &block)
     end
   end
 end

data/lib/foreman_remote_execution_core/polling_script_runner.rb

@@ -1,10 +1,27 @@
+require 'base64'
+
 module ForemanRemoteExecutionCore
   class PollingScriptRunner < ScriptRunner
 
     DEFAULT_REFRESH_INTERVAL = 60
 
-    def initialize(options, user_method)
-      super(options, user_method)
+    def self.load_script(name)
+      script_dir = File.expand_path('../async_scripts', __FILE__)
+      File.read(File.join(script_dir, name))
+    end
+
+    # The script that controls the flow of the job, able to initiate update or
+    # finish on the task, or take over the control over script lifecycle
+    CONTROL_SCRIPT = load_script('control.sh')
+
+    # The script always outputs at least one line
+    # First line of the output either has to begin with
+    # "RUNNING" or "DONE $EXITCODE"
+    # The following lines are treated as regular output
+    RETRIEVE_SCRIPT = load_script('retrieve.sh')
+
+    def initialize(options, user_method, suspended_action: nil)
+      super(options, user_method, suspended_action: suspended_action)
       @callback_host = options[:callback_host]
       @task_id = options[:uuid]
       @step_id = options[:step_id]
@@ -13,19 +30,19 @@ module ForemanRemoteExecutionCore
 
     def prepare_start
       super
-      basedir         = File.dirname @remote_script
-      @pid_path       = File.join(basedir, 'pid')
-      @retrieval_script ||= File.join(basedir, 'retrieve')
-      prepare_retrieval unless @prepared
+      @base_dir = File.dirname @remote_script
+      upload_control_scripts
     end
 
-    def control_script
+    def initialization_script
       close_stdin = '</dev/null'
       close_fds = close_stdin + ' >/dev/null 2>/dev/null'
-      # pipe the output to tee while capturing the exit code in a file, don't wait for it to finish, output PID of the main command
-      <<-SCRIPT.gsub(/^\s+\| /, '')
-      | sh -c '(#{@user_method.cli_command_prefix}#{@remote_script} #{close_stdin}; echo $?>#{@exit_code_path}) | /usr/bin/tee #{@output_path} >/dev/null; #{callback_scriptlet}' #{close_fds} &
-      | echo $! > '#{@pid_path}'
+      main_script = "(#{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
+      control_script_finish = "#{@control_script_path} init-script-finish"
+      <<-SCRIPT.gsub(/^ +\| /, '')
+      | export CONTROL_SCRIPT="#{@control_script_path}"
+      | sh -c '#{main_script}; #{control_script_finish}' #{close_fds} &
+      | echo $! > '#{@base_dir}/pid'
       SCRIPT
     end
 
@@ -35,9 +52,13 @@ module ForemanRemoteExecutionCore
 
     def refresh
       err = output = nil
-      with_retries do
+      begin
         _, output, err = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
+      rescue => e
+        @logger.info("Error while connecting to the remote host on refresh: #{e.message}")
       end
+      return if output.nil? || output.empty?
+
       lines = output.lines
       result = lines.shift.match(/^DONE (\d+)?/)
       publish_data(lines.join, 'stdout') unless lines.empty?
@@ -45,8 +66,21 @@ module ForemanRemoteExecutionCore
       if result
         exitcode = result[1] || 0
         publish_exit_status(exitcode.to_i)
-        run_sync("rm -rf \"#{remote_command_dir}\"") if @cleanup_working_dirs
+        cleanup
       end
+    ensure
+      destroy_session
+    end
+
+    def external_event(event)
+      data = event.data
+      if data['manual_mode']
+        load_event_updates(data)
+      else
+        # getting the update from automatic mode - reaching to the host to get the latest update
+        return run_refresh
+      end
+    ensure
       destroy_session
     end
 
@@ -55,66 +89,42 @@ module ForemanRemoteExecutionCore
       ForemanTasksCore::OtpManager.drop_otp(@task_id, @otp) if @otp
     end
 
-    def prepare_retrieval
-      # The script always outputs at least one line
-      # First line of the output either has to begin with
-      # "RUNNING" or "DONE $EXITCODE"
-      # The following lines are treated as regular output
-      base = File.dirname(@output_path)
-      posfile = File.join(base, 'position')
-      tmpfile = File.join(base, 'tmp')
-      script = <<-SCRIPT.gsub(/^ +\| /, '')
-      | #!/bin/sh
-      | pid=$(cat "#{@pid_path}")
-      | if ! pgrep --help 2>/dev/null >/dev/null; then
-      |   echo DONE 1
-      |   echo "pgrep is required" >&2
-      |   exit 1
-      | fi
-      | if pgrep -P "$pid" >/dev/null 2>/dev/null; then
-      |   echo RUNNING
-      | else
-      |   echo "DONE $(cat "#{@exit_code_path}" 2>/dev/null)"
-      | fi
-      | [ -f "#{@output_path}" ] || exit 0
-      | [ -f "#{posfile}" ] || echo 1 > "#{posfile}"
-      | position=$(cat "#{posfile}")
-      | tail --bytes "+${position}" "#{@output_path}" > "#{tmpfile}"
-      | bytes=$(cat "#{tmpfile}" | wc --bytes)
-      | expr "${position}" + "${bytes}" > "#{posfile}"
-      | cat "#{tmpfile}"
-      SCRIPT
-      @logger.debug("copying script:\n#{script.lines.map { |line| "    | #{line}" }.join}")
-      cp_script_to_remote(script, 'retrieve')
-      @prepared = true
-    end
+    def upload_control_scripts
+      return if @control_scripts_uploaded
 
-    def callback_scriptlet(callback_script_path = nil)
-      if @otp
-        callback_script_path = cp_script_to_remote(callback_script, 'callback') if callback_script_path.nil?
-        "#{@user_method.cli_command_prefix}#{callback_script_path}"
-      else
-        ':' # Shell synonym for "do nothing"
-      end
+      cp_script_to_remote(env_script, 'env.sh')
+      @control_script_path = cp_script_to_remote(CONTROL_SCRIPT, 'control.sh')
+      @retrieval_script = cp_script_to_remote(RETRIEVE_SCRIPT, 'retrieve.sh')
+      @control_scripts_uploaded = true
     end
 
-    def callback_script
+    # Script setting the dynamic values to env variables: it's sourced from other control scripts
+    def env_script
       <<-SCRIPT.gsub(/^ +\| /, '')
-      | #!/bin/sh
-      | exit_code=$(cat "#{@exit_code_path}")
-      | url="#{@callback_host}/dynflow/tasks/#{@task_id}/done"
-      | json="{ \\\"step_id\\\": #{@step_id} }"
-      | if which curl >/dev/null; then
-      |   curl -X POST -d "$json" -u "#{@task_id}:#{@otp}" "$url"
-      | else
-      |   echo 'curl is required' >&2
-      |   exit 1
-      | fi
+      | CALLBACK_HOST="#{@callback_host}"
+      | TASK_ID="#{@task_id}"
+      | STEP_ID="#{@step_id}"
+      | OTP="#{@otp}"
       SCRIPT
     end
 
     private
 
+    # Generates updates based on the callback data from the manual mode
+    def load_event_updates(event_data)
+      continuous_output = ForemanTasksCore::ContinuousOutput.new
+      if event_data.key?('output')
+        lines = Base64.decode64(event_data['output']).sub(/\A(RUNNING|DONE).*\n/, '')
+        continuous_output.add_output(lines, 'stdout')
+      end
+      cleanup if event_data['exit_code']
+      new_update(continuous_output, event_data['exit_code'])
+    end
+
+    def cleanup
+      run_sync("rm -rf \"#{remote_command_dir}\"") if @cleanup_working_dirs
+    end
+
     def destroy_session
       if @session
         @logger.debug("Closing session with #{@ssh_user}@#{@host}")

data/lib/foreman_remote_execution_core/script_runner.rb

@@ -1,11 +1,11 @@
 require 'net/ssh'
 require 'fileutils'
 
-# rubocop:disable Lint/HandleExceptions
+# rubocop:disable Lint/SuppressedException
 begin
   require 'net/ssh/krb'
 rescue LoadError; end
-# rubocop:enable Lint/HandleExceptions:
+# rubocop:enable Lint/SuppressedException:
 
 module ForemanRemoteExecutionCore
   class SudoUserMethod
@@ -32,7 +32,7 @@ module ForemanRemoteExecutionCore
     end
 
     def filter_password?(received_data)
-      !@effective_user_password.empty? && @password_sent && received_data.match(@effective_user_password)
+      !@effective_user_password.empty? && @password_sent && received_data.match(Regexp.escape(@effective_user_password))
     end
 
     def sent_all_data?
@@ -49,7 +49,7 @@ module ForemanRemoteExecutionCore
   end
 
   class DzdoUserMethod < SudoUserMethod
-    LOGIN_PROMPT = /password/i
+    LOGIN_PROMPT = /password/i.freeze
 
     def login_prompt
       LOGIN_PROMPT
@@ -68,7 +68,8 @@ module ForemanRemoteExecutionCore
       @ssh_user = ssh_user
     end
 
-    def on_data(_, _); end
+    def on_data(_, _)
+    end
 
     def filter_password?(received_data)
       false
@@ -82,11 +83,13 @@ module ForemanRemoteExecutionCore
       "su - #{effective_user} -c "
     end
 
-    def reset; end
+    def reset
+    end
   end
 
   class NoopUserMethod
-    def on_data(_, _); end
+    def on_data(_, _)
+    end
 
     def filter_password?(received_data)
       false
@@ -96,9 +99,11 @@ module ForemanRemoteExecutionCore
       true
     end
 
-    def cli_command_prefix; end
+    def cli_command_prefix
+    end
 
-    def reset; end
+    def reset
+    end
   end
 
   class ScriptRunner < ForemanTasksCore::Runner::Base
@@ -108,8 +113,8 @@ module ForemanRemoteExecutionCore
     DEFAULT_REFRESH_INTERVAL = 1
     MAX_PROCESS_RETRIES = 4
 
-    def initialize(options, user_method)
-      super()
+    def initialize(options, user_method, suspended_action: nil)
+      super suspended_action: suspended_action
       @host = options.fetch(:hostname)
       @script = options.fetch(:script)
       @ssh_user = options.fetch(:ssh_user, 'root')
@@ -127,7 +132,7 @@ module ForemanRemoteExecutionCore
       @user_method = user_method
     end
 
-    def self.build(options)
+    def self.build(options, suspended_action:)
       effective_user = options.fetch(:effective_user, nil)
       ssh_user = options.fetch(:ssh_user, 'root')
       effective_user_method = options.fetch(:effective_user_method, 'sudo')
@@ -136,23 +141,23 @@ module ForemanRemoteExecutionCore
                       NoopUserMethod.new
                     elsif effective_user_method == 'sudo'
                       SudoUserMethod.new(effective_user, ssh_user,
-                                         options.fetch(:secrets, {}).fetch(:sudo_password, nil))
+                        options.fetch(:secrets, {}).fetch(:sudo_password, nil))
                     elsif effective_user_method == 'dzdo'
                       DzdoUserMethod.new(effective_user, ssh_user,
-                                         options.fetch(:secrets, {}).fetch(:sudo_password, nil))
+                        options.fetch(:secrets, {}).fetch(:sudo_password, nil))
                     elsif effective_user_method == 'su'
                       SuUserMethod.new(effective_user, ssh_user)
                     else
                       raise "effective_user_method '#{effective_user_method}' not supported"
                     end
 
-      new(options, user_method)
+      new(options, user_method, suspended_action: suspended_action)
     end
 
     def start
       prepare_start
-      script = control_script
-      logger.debug("executing script:\n#{script.lines.map { |line| "  | #{line}" }.join}")
+      script = initialization_script
+      logger.debug("executing script:\n#{indent_multiline(script)}")
       trigger(script)
     rescue => e
       logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
@@ -169,7 +174,8 @@ module ForemanRemoteExecutionCore
       @exit_code_path = File.join(File.dirname(@remote_script), 'exit_code')
     end
 
-    def control_script
+    # the script that initiates the execution
+    def initialization_script
       # pipe the output to tee while capturing the exit code in a file
       <<-SCRIPT.gsub(/^\s+\| /, '')
       | sh <<WRAPPER
@@ -181,6 +187,7 @@ module ForemanRemoteExecutionCore
 
     def refresh
       return if @session.nil?
+
       with_retries do
         with_disconnect_handling do
           @session.process(0)
@@ -252,6 +259,10 @@ module ForemanRemoteExecutionCore
 
     private
 
+    def indent_multiline(string)
+      string.lines.map { |line| "  | #{line}" }.join
+    end
+
     def should_cleanup?
       @session && !@session.closed? && @cleanup_working_dirs
     end
@@ -269,7 +280,6 @@ module ForemanRemoteExecutionCore
       ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
       ssh_options[:password] = @ssh_password if @ssh_password
       ssh_options[:passphrase] = @key_passphrase if @key_passphrase
-      ssh_options[:user_known_hosts_file] = @known_hosts_file if @known_hosts_file
       ssh_options[:keys_only] = true
       # if the host public key is contained in the known_hosts_file,
       # verify it, otherwise, if missing, import it and continue
@@ -291,6 +301,7 @@ module ForemanRemoteExecutionCore
     # part of calling the `refresh` method.
     def run_async(command)
       raise 'Async command already in progress' if @started
+
       @started = false
       @user_method.reset
 
@@ -346,6 +357,7 @@ module ForemanRemoteExecutionCore
         end
         ch.exec command do |_, success|
           raise 'could not execute command' unless success
+
           started = true
         end
       end
@@ -390,7 +402,9 @@ module ForemanRemoteExecutionCore
     end
 
     def cp_script_to_remote(script = @script, name = 'script')
-      upload_data(sanitize_script(script), remote_command_file(name), 555)
+      path = remote_command_file(name)
+      @logger.debug("copying script to #{path}:\n#{indent_multiline(script)}")
+      upload_data(sanitize_script(script), path, 555)
     end
 
     def upload_data(data, path, permissions = 555)
@@ -407,6 +421,7 @@ module ForemanRemoteExecutionCore
       if status != 0
         raise "Unable to upload file to #{path} on remote system: exit code: #{status}"
       end
+
       path
     end
 
@@ -440,6 +455,7 @@ module ForemanRemoteExecutionCore
     def check_expecting_disconnect
       last_output = @continuous_output.raw_outputs.find { |d| d['output_type'] == 'stdout' }
       return unless last_output
+
       if EXPECTED_POWER_ACTION_MESSAGES.any? { |message| last_output['output'] =~ /^#{message}/ }
         @expecting_disconnect = true
       end

data/lib/foreman_remote_execution_core/version.rb

@@ -1,3 +1,3 @@
 module ForemanRemoteExecutionCore
-  VERSION = '1.1.4'.freeze
+  VERSION = '1.3.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman_remote_execution_core
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.3.1
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-09 00:00:00.000000000 Z
+date: 2020-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: foreman-tasks-core
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -48,6 +48,8 @@ files:
 - LICENSE
 - lib/foreman_remote_execution_core.rb
 - lib/foreman_remote_execution_core/actions.rb
+- lib/foreman_remote_execution_core/async_scripts/control.sh
+- lib/foreman_remote_execution_core/async_scripts/retrieve.sh
 - lib/foreman_remote_execution_core/dispatcher.rb
 - lib/foreman_remote_execution_core/fake_script_runner.rb
 - lib/foreman_remote_execution_core/log_filter.rb
@@ -73,8 +75,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Foreman remote execution - core bits