smart_proxy_remote_execution_ssh 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9532552fe00ed7bf26c3755d710ad2ea87b4ac9255a12851a2a2f20cd6e3afbf
-  data.tar.gz: 5b2d309cc8bdec3ed2c20d2b0a944c6bc1863a3fd728e7febc5e020dc3f88239
+  metadata.gz: 750910e916f0d4ad411cf868636075e597573dd8cca720e414156bcee55331dc
+  data.tar.gz: 4003e71f358abc47847fb9a2e4cf3c211189bc915b6b4196dd02d6d21633d2b8
 SHA512:
-  metadata.gz: a2359e2706bb4d465fbb98285e48099d0ff3c6f58ab7aae0e4972d101ec2b5663bea62e965be074c784dea19cda7de551d1f27f26aba6c4502d4d96c9151bd7d
-  data.tar.gz: f7bce64d724947712ee9c2d394802b07f64df2c581d92d36cb3ac4732b9a8609b2c1935e4189a39a0712692c2946111e426eb2ea5452f379906eb3ff379142ee
+  metadata.gz: efa2a87ce6a6125f7701979305a0c5fcc1fb3ad2703d5aef140505943789b45648311e46f892791a919a88d757f019485ff45209efc22423cc6543a298224a03
+  data.tar.gz: 4411e680ca841903d47b295090c4a194f92dfe91cc58796272d42b9c370ad6a3e6a8bc34973f0d7a85d93fe604788993428d944277b05426194893ecff0a9d51

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -1,3 +1,4 @@
+require 'foreman_tasks_core'
 require 'smart_proxy_remote_execution_ssh/version'
 require 'smart_proxy_dynflow'
 require 'smart_proxy_remote_execution_ssh/webrick_ext'
@@ -19,6 +20,8 @@ module Proxy::RemoteExecution
         unless File.exist?(public_key_file)
           raise "Ssh public key file #{public_key_file} doesn't exist"
         end
+
+        validate_ssh_log_level!
       end
 
       def private_key_file
@@ -28,6 +31,30 @@ module Proxy::RemoteExecution
       def public_key_file
         File.expand_path("#{private_key_file}.pub")
       end
+
+      def validate_ssh_log_level!
+        wanted_level = Plugin.settings.ssh_log_level.to_s
+        levels = Plugin::SSH_LOG_LEVELS
+        unless levels.include? wanted_level
+          raise "Wrong value '#{Plugin.settings.ssh_log_level}' for ssh_log_level, must be one of #{levels.join(', ')}"
+        end
+
+        current = ::Proxy::SETTINGS.log_level.to_s.downcase
+
+        # regular log levels correspond to upcased ssh logger levels
+        ssh, regular = [wanted_level, current].map do |wanted|
+          levels.each_with_index.find { |value, _index| value == wanted }.last
+        end
+
+        if ssh < regular
+          raise 'ssh_log_level cannot be more verbose than regular log level'
+        end
+
+        Plugin.settings.ssh_log_level = Plugin.settings.ssh_log_level.to_sym
+      end
     end
+
+    require 'smart_proxy_dynflow_core/task_launcher_registry'
+    SmartProxyDynflowCore::TaskLauncherRegistry.register('ssh', ForemanTasksCore::TaskLauncher::Batch)
   end
 end

data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb

@@ -0,0 +1,20 @@
+require 'foreman_tasks_core/shareable_action'
+
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    class RunScript < ForemanTasksCore::Runner::Action
+      def initiate_runner
+        additional_options = {
+          :step_id => run_step_id,
+          :uuid => execution_plan_id,
+        }
+        Proxy::RemoteExecution::Ssh::Plugin.runner_class.build(input.merge(additional_options),
+                                                               suspended_action: suspended_action)
+      end
+
+      def runner_dispatcher
+        Dispatcher.instance
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh

@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# Control script for the remote execution jobs.
+#
+# The initial script calls `$CONTROL_SCRIPT init-script-finish` once the original script exits.
+# In automatic mode, the exit code is sent back to the proxy on `init-script-finish`.
+#
+# What the script provides is also a manual mode, where the author of the rex script can take
+# full control of the job lifecycle. This allows keeping the marked as running even when
+# the initial script finishes.
+#
+# The manual mode is turned on by calling `$CONTROL_SCRIPT manual-control`. After calling this,
+# one can call `echo message | $CONTROL_SCRIPT update` to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code) to send output to the remote execution jobs
+# and `$CONTROL_SCRIPT finish 0` once finished (with 0 as exit code)
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+if ! command -v curl >/dev/null; then
+    echo 'curl is required' >&2
+    exit 1
+fi
+
+# send the callback data to proxy
+update() {
+    "$BASE_DIR/retrieve.sh" push_update
+}
+
+# wait for named pipe $1 to retrieve data. If $2 is provided, it serves as timeout
+# in seconds on how long to wait when reading.
+wait_for_pipe() {
+    pipe_path=$1
+    if [ -n "$2" ]; then
+        timeout="-t $2"
+    fi
+    if read $timeout <>"$pipe_path"; then
+        rm "$pipe_path"
+        return 0
+    else
+        return 1
+    fi
+}
+
+# function run in background, when receiving update data via STDIN.
+periodic_update() {
+    interval=1
+    # reading some data from periodic_update_control signals we're done
+    while ! wait_for_pipe "$BASE_DIR/periodic_update_control" "$interval"; do
+        update
+    done
+    # one more update before we finish
+    update
+    # signal the main process that we are finished
+    echo > "$BASE_DIR/periodic_update_finished"
+}
+
+# signal the periodic_update process that the main process is finishing
+periodic_update_finish() {
+    if [ -e "$BASE_DIR/periodic_update_control" ]; then
+       echo > "$BASE_DIR/periodic_update_control"
+    fi
+}
+
+ACTION=${1:-finish}
+
+case "$ACTION" in
+    init-script-finish)
+        if ! [ -e "$BASE_DIR/manual_mode" ]; then
+            # make the exit code of initialization script the exit code of the whole job
+            cp init_exit_code exit_code
+            update
+        fi
+        ;;
+    finish)
+        # take exit code passed via the command line, with fallback
+        # to the exit code of the initialization script
+        exit_code=${2:-$(cat "$BASE_DIR/init_exit_code")}
+        echo $exit_code > "$BASE_DIR/exit_code"
+        update
+        if [ -e "$BASE_DIR/manual_mode" ]; then
+            rm "$BASE_DIR/manual_mode"
+        fi
+        ;;
+    update)
+        # read data from input when redirected though a pipe
+        if ! [ -t 0 ]; then
+            # couple of named pipes to coordinate the main process with the periodic_update
+            mkfifo "$BASE_DIR/periodic_update_control"
+            mkfifo "$BASE_DIR/periodic_update_finished"
+            trap "periodic_update_finish" EXIT
+            # run periodic update as separate process to keep sending updates in output to server
+            periodic_update &
+            # redirect the input into output
+            tee -a "$BASE_DIR/output"
+            periodic_update_finish
+            # ensure the periodic update finished before we return
+            wait_for_pipe "$BASE_DIR/periodic_update_finished"
+        else
+            update
+        fi
+        ;;
+    # mark the script to be in manual mode: this means the script author needs to use `update` and `finish`
+    # commands to send output to the remote execution job or mark it as finished.
+    manual-mode)
+        touch "$BASE_DIR/manual_mode"
+        ;;
+    *)
+        echo "Unknown action $ACTION"
+        exit 1
+        ;;
+esac

data/lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+if ! pgrep --help 2>/dev/null >/dev/null; then
+    echo DONE 1
+    echo "pgrep is required" >&2
+    exit 1
+fi
+
+BASE_DIR="$(dirname "$(readlink -f "$0")")"
+
+# load the data required for generating the callback
+. "$BASE_DIR/env.sh"
+URL_PREFIX="$CALLBACK_HOST/dynflow/tasks/$TASK_ID"
+AUTH="$TASK_ID:$OTP"
+CURL="curl --silent --show-error --fail --max-time 10"
+
+MY_LOCK_FILE="$BASE_DIR/retrieve_lock.$$"
+MY_PID=$$
+echo $MY_PID >"$MY_LOCK_FILE"
+LOCK_FILE="$BASE_DIR/retrieve_lock"
+TMP_OUTPUT_FILE="$BASE_DIR/tmp_output"
+
+RUN_TIMEOUT=30 # for how long can the script hold the lock
+WAIT_TIMEOUT=60 # for how long the script is trying to acquire the lock
+START_TIME=$(date +%s)
+
+fail() {
+    echo RUNNING
+    echo "$1"
+    exit 1
+}
+
+acquire_lock() {
+    # try to acquire lock by creating the file (ln should be atomic an fail in case
+    # another process succeeded first). We also check the content of the lock file,
+    # in case our process won when competing over the lock while invalidating
+    # the lock on timeout.
+    ln "$MY_LOCK_FILE" "$LOCK_FILE" 2>/dev/null || [ "$(head -n1 "$LOCK_FILE")" = "$MY_PID" ]
+    return $?
+}
+
+# acquiring the lock before proceeding, to ensure only one instance of the script is running
+while ! acquire_lock; do
+    # we failed to create retrieve_lock - assuming there is already another retrieve script running
+    current_pid=$(head -n1 "$LOCK_FILE")
+    if [ -z "$current_pid" ]; then
+        continue
+    fi
+    # check whether the lock is not too old (compared to $RUN_TIMEOUT) and try to kill
+    # if it is, so that we don't have a stalled processes here
+    lock_lines_count=$(wc -l < "$LOCK_FILE")
+    current_lock_time=$(stat --format "%Y" "$LOCK_FILE")
+    current_time=$(date +%s)
+
+    if [ "$(( current_time - START_TIME ))" -gt "$WAIT_TIMEOUT" ]; then
+        # We were waiting for the lock for too long - just give up
+        fail "Wait time exceeded $WAIT_TIMEOUT"
+    elif [ "$(( current_time - current_lock_time ))" -gt "$RUN_TIMEOUT" ]; then
+        # The previous lock it hold for too long - re-acquiring procedure
+        if [ "$lock_lines_count" -gt 1 ]; then
+           # there were multiple processes waiting for lock without resolution
+           # longer than the $RUN_TIMEOUT - we reset the lock file and let processes
+           # to compete
+           echo "RETRY" > "$LOCK_FILE"
+        fi
+        if [ "$current_pid" != "RETRY" ]; then
+            # try to kill the currently stalled process
+            kill -9 "$current_pid" 2>/dev/null
+        fi
+        # try to add our process as one candidate
+        echo $MY_PID >> "$LOCK_FILE"
+        if [ "$( head -n2 "$LOCK_FILE" | tail -n1 )" = "$MY_PID" ]; then
+            # our process won the competition for the new lock: it is the first pid
+            # after the original one in the lock file - take ownership of the lock
+            # next iteration only this process will get through
+            echo $MY_PID >"$LOCK_FILE"
+        fi
+    else
+        # still waiting for the original owner to finish
+        sleep 1
+    fi
+done
+
+release_lock() {
+    rm "$MY_LOCK_FILE"
+    rm "$LOCK_FILE"
+}
+# ensure the release the lock at exit
+trap "release_lock" EXIT
+
+# make sure we clear previous tmp output file
+if [ -e "$TMP_OUTPUT_FILE" ]; then
+    rm "$TMP_OUTPUT_FILE"
+fi
+
+pid=$(cat "$BASE_DIR/pid")
+[ -f "$BASE_DIR/position" ] || echo 1 > "$BASE_DIR/position"
+position=$(cat "$BASE_DIR/position")
+
+prepare_output() {
+    if [ -e "$BASE_DIR/manual_mode" ] || ([ -n "$pid" ] && pgrep -P "$pid" >/dev/null 2>&1); then
+        echo RUNNING
+    else
+        echo "DONE $(cat "$BASE_DIR/exit_code" 2>/dev/null)"
+    fi
+    [ -f "$BASE_DIR/output" ] || exit 0
+    tail --bytes "+${position}" "$BASE_DIR/output" > "$TMP_OUTPUT_FILE"
+    cat "$TMP_OUTPUT_FILE"
+}
+
+# prepare the callback payload
+payload() {
+    if [ -n "$1" ]; then
+        exit_code="$1"
+    else
+        exit_code=null
+    fi
+
+    if [ -e "$BASE_DIR/manual_mode" ]; then
+        manual_mode=true
+        output=$(prepare_output | base64 -w0)
+    else
+        manual_mode=false
+    fi
+
+    echo "{ \"exit_code\": $exit_code,"\
+         "  \"step_id\": \"$STEP_ID\","\
+         "  \"manual_mode\": $manual_mode,"\
+         "  \"output\": \"$output\" }"
+}
+
+if [ "$1" = "push_update" ]; then
+    if [ -e "$BASE_DIR/exit_code" ]; then
+        exit_code="$(cat "$BASE_DIR/exit_code")"
+        action="done"
+    else
+        exit_code=""
+        action="update"
+    fi
+    $CURL -X POST -d "$(payload $exit_code)" -u "$AUTH" "$URL_PREFIX"/$action 2>>"$BASE_DIR/curl_stderr"
+    success=$?
+else
+    prepare_output
+    success=$?
+fi
+
+if [ "$success" = 0 ] && [ -e "$TMP_OUTPUT_FILE" ]; then
+    # in case the retrieval was successful, move the position of the cursor to be read next time
+    bytes=$(wc --bytes < "$TMP_OUTPUT_FILE")
+    expr "${position}" + "${bytes}" > "$BASE_DIR/position"
+fi

data/lib/smart_proxy_remote_execution_ssh/dispatcher.rb

@@ -0,0 +1,10 @@
+require 'foreman_tasks_core/runner/dispatcher'
+
+module Proxy::RemoteExecution::Ssh
+  class Dispatcher < ::ForemanTasksCore::Runner::Dispatcher
+    def refresh_interval
+      @refresh_interval ||= Plugin.settings[:runner_refresh_interval] ||
+                            Plugin.runner_class::DEFAULT_REFRESH_INTERVAL
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/log_filter.rb

@@ -0,0 +1,14 @@
+module Proxy::RemoteExecution::Ssh
+  class LogFilter < ::Logger
+    def initialize(base_logger)
+      @base_logger = base_logger
+    end
+
+    def add(severity, *args, &block)
+      severity ||= ::Logger::UNKNOWN
+      return true if @base_logger.nil? || severity < @level
+
+      @base_logger.add(severity, *args, &block)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -1,5 +1,7 @@
 module Proxy::RemoteExecution::Ssh
   class Plugin < Proxy::Plugin
+    SSH_LOG_LEVELS = %w[debug info warn error fatal].freeze
+
     http_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
 
@@ -9,7 +11,11 @@ module Proxy::RemoteExecution::Ssh
                      :remote_working_dir      => '/var/tmp',
                      :local_working_dir       => '/var/tmp',
                      :kerberos_auth           => false,
-                     :async_ssh               => false
+                     :async_ssh               => false,
+                     # When set to nil, makes REX use the runner's default interval
+                     # :runner_refresh_interval => nil,
+                     :ssh_log_level           => :fatal,
+                     :cleanup_working_dirs    => true
 
     plugin :ssh, Proxy::RemoteExecution::Ssh::VERSION
     after_activation do
@@ -17,17 +23,27 @@ module Proxy::RemoteExecution::Ssh
       require 'smart_proxy_remote_execution_ssh/version'
       require 'smart_proxy_remote_execution_ssh/cockpit'
       require 'smart_proxy_remote_execution_ssh/api'
-
-      begin
-        require 'smart_proxy_dynflow_core'
-        require 'foreman_remote_execution_core'
-        ForemanRemoteExecutionCore.initialize_settings(Proxy::RemoteExecution::Ssh::Plugin.settings.to_h)
-      rescue LoadError # rubocop:disable Lint/HandleExceptions
-        # Dynflow core is not available in the proxy, will be handled
-        # by standalone Dynflow core
-      end
+      require 'smart_proxy_remote_execution_ssh/actions/run_script'
+      require 'smart_proxy_remote_execution_ssh/dispatcher'
+      require 'smart_proxy_remote_execution_ssh/log_filter'
+      require 'smart_proxy_remote_execution_ssh/runners'
+      require 'smart_proxy_dynflow_core'
 
       Proxy::RemoteExecution::Ssh.validate!
     end
+
+    def self.simulate?
+      @simulate ||= %w[yes true 1].include? ENV.fetch('REX_SIMULATE', '').downcase
+    end
+
+    def self.runner_class
+      @runner_class ||= if simulate?
+                          Runners::FakeScriptRunner
+                        elsif settings[:async_ssh]
+                          Runners::PollingScriptRunner
+                        else
+                          Runners::ScriptRunner
+                        end
+    end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/runners.rb

@@ -0,0 +1,7 @@
+module Proxy::RemoteExecution::Ssh
+  module Runners
+    require 'smart_proxy_remote_execution_ssh/runners/script_runner'
+    require 'smart_proxy_remote_execution_ssh/runners/polling_script_runner'
+    require 'smart_proxy_remote_execution_ssh/runners/fake_script_runner'
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/runners/fake_script_runner.rb

@@ -0,0 +1,87 @@
+module Proxy::RemoteExecution::Ssh::Runners
+  class FakeScriptRunner < ForemanTasksCore::Runner::Base
+    DEFAULT_REFRESH_INTERVAL = 1
+
+    @data = []
+
+    class << self
+      attr_accessor :data
+
+      def load_data(path = nil)
+        if path.nil?
+          @data = <<-BANNER.gsub(/^\s+\| ?/, '').lines
+            | ====== Simulated Remote Execution ======
+            |
+            | This is an output of a simulated remote
+            | execution run. It should run for about
+            | 5 seconds and finish successfully.
+          BANNER
+        else
+          File.open(File.expand_path(path), 'r') do |f|
+            @data = f.readlines.map(&:chomp)
+          end
+        end
+        @data.freeze
+      end
+
+      def build(options, suspended_action:)
+        new(options, suspended_action: suspended_action)
+      end
+    end
+
+    def initialize(*args)
+      super
+      # Load the fake output the first time its needed
+      self.class.load_data(ENV['REX_SIMULATE_PATH']) unless self.class.data.frozen?
+      @position = 0
+    end
+
+    def start
+      refresh
+    end
+
+    # Do one step
+    def refresh
+      if done?
+        finish
+      else
+        step
+      end
+    end
+
+    def kill
+      finish
+    end
+
+    private
+
+    def finish
+      publish_exit_status exit_code
+    end
+
+    def step
+      publish_data(next_chunk, 'stdout')
+    end
+
+    def done?
+      @position == self.class.data.count
+    end
+
+    def next_chunk
+      output = self.class.data[@position]
+      @position += 1
+      output
+    end
+
+    # Decide if the execution should fail or not
+    def exit_code
+      fail_chance   = ENV.fetch('REX_SIMULATE_FAIL_CHANCE', 0).to_i
+      fail_exitcode = ENV.fetch('REX_SIMULATE_EXIT', 0).to_i
+      if fail_exitcode.zero? || fail_chance < (Random.rand * 100).round
+        0
+      else
+        fail_exitcode
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb

@@ -0,0 +1,140 @@
+require 'base64'
+
+module Proxy::RemoteExecution::Ssh::Runners
+  class PollingScriptRunner < ScriptRunner
+    DEFAULT_REFRESH_INTERVAL = 60
+
+    def self.load_script(name)
+      script_dir = File.expand_path('../async_scripts', __dir__)
+      File.read(File.join(script_dir, name))
+    end
+
+    # The script that controls the flow of the job, able to initiate update or
+    # finish on the task, or take over the control over script lifecycle
+    CONTROL_SCRIPT = load_script('control.sh')
+
+    # The script always outputs at least one line
+    # First line of the output either has to begin with
+    # "RUNNING" or "DONE $EXITCODE"
+    # The following lines are treated as regular output
+    RETRIEVE_SCRIPT = load_script('retrieve.sh')
+
+    def initialize(options, user_method, suspended_action: nil)
+      super(options, user_method, suspended_action: suspended_action)
+      @callback_host = options[:callback_host]
+      @task_id = options[:uuid]
+      @step_id = options[:step_id]
+      @otp = ForemanTasksCore::OtpManager.generate_otp(@task_id)
+    end
+
+    def prepare_start
+      super
+      @base_dir = File.dirname @remote_script
+      upload_control_scripts
+    end
+
+    def initialization_script
+      close_stdin = '</dev/null'
+      close_fds = close_stdin + ' >/dev/null 2>/dev/null'
+      main_script = "(#{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
+      control_script_finish = "#{@control_script_path} init-script-finish"
+      <<-SCRIPT.gsub(/^ +\| /, '')
+      | export CONTROL_SCRIPT="#{@control_script_path}"
+      | sh -c '#{main_script}; #{control_script_finish}' #{close_fds} &
+      | echo $! > '#{@base_dir}/pid'
+      SCRIPT
+    end
+
+    def trigger(*args)
+      run_sync(*args)
+    end
+
+    def refresh
+      err = output = nil
+      begin
+        _, output, err = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
+      rescue StandardError => e
+        @logger.info("Error while connecting to the remote host on refresh: #{e.message}")
+      end
+
+      process_retrieved_data(output, err)
+    ensure
+      destroy_session
+    end
+
+    def process_retrieved_data(output, err)
+      return if output.nil? || output.empty?
+
+      lines = output.lines
+      result = lines.shift.match(/^DONE (\d+)?/)
+      publish_data(lines.join, 'stdout') unless lines.empty?
+      publish_data(err, 'stderr') unless err.empty?
+      if result
+        exitcode = result[1] || 0
+        publish_exit_status(exitcode.to_i)
+        cleanup
+      end
+    end
+
+    def external_event(event)
+      data = event.data
+      if data['manual_mode']
+        load_event_updates(data)
+      else
+        # getting the update from automatic mode - reaching to the host to get the latest update
+        return run_refresh
+      end
+    ensure
+      destroy_session
+    end
+
+    def close
+      super
+      ForemanTasksCore::OtpManager.drop_otp(@task_id, @otp) if @otp
+    end
+
+    def upload_control_scripts
+      return if @control_scripts_uploaded
+
+      cp_script_to_remote(env_script, 'env.sh')
+      @control_script_path = cp_script_to_remote(CONTROL_SCRIPT, 'control.sh')
+      @retrieval_script = cp_script_to_remote(RETRIEVE_SCRIPT, 'retrieve.sh')
+      @control_scripts_uploaded = true
+    end
+
+    # Script setting the dynamic values to env variables: it's sourced from other control scripts
+    def env_script
+      <<-SCRIPT.gsub(/^ +\| /, '')
+      | CALLBACK_HOST="#{@callback_host}"
+      | TASK_ID="#{@task_id}"
+      | STEP_ID="#{@step_id}"
+      | OTP="#{@otp}"
+      SCRIPT
+    end
+
+    private
+
+    # Generates updates based on the callback data from the manual mode
+    def load_event_updates(event_data)
+      continuous_output = ForemanTasksCore::ContinuousOutput.new
+      if event_data.key?('output')
+        lines = Base64.decode64(event_data['output']).sub(/\A(RUNNING|DONE).*\n/, '')
+        continuous_output.add_output(lines, 'stdout')
+      end
+      cleanup if event_data['exit_code']
+      new_update(continuous_output, event_data['exit_code'])
+    end
+
+    def cleanup
+      run_sync("rm -rf \"#{remote_command_dir}\"") if @cleanup_working_dirs
+    end
+
+    def destroy_session
+      if @session
+        @logger.debug("Closing session with #{@ssh_user}@#{@host}")
+        @session.close
+        @session = nil
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb

@@ -0,0 +1,469 @@
+require 'net/ssh'
+require 'fileutils'
+
+# Rubocop can't make up its mind what it wants
+# rubocop:disable Lint/SuppressedException, Lint/RedundantCopDisableDirective
+begin
+  require 'net/ssh/krb'
+rescue LoadError; end
+# rubocop:enable Lint/SuppressedException, Lint/RedundantCopDisableDirective
+
+module Proxy::RemoteExecution::Ssh::Runners
+  class EffectiveUserMethod
+    attr_reader :effective_user, :ssh_user, :effective_user_password, :password_sent
+
+    def initialize(effective_user, ssh_user, effective_user_password)
+      @effective_user = effective_user
+      @ssh_user = ssh_user
+      @effective_user_password = effective_user_password.to_s
+      @password_sent = false
+    end
+
+    def on_data(received_data, ssh_channel)
+      if received_data.match(login_prompt)
+        ssh_channel.send_data(effective_user_password + "\n")
+        @password_sent = true
+      end
+    end
+
+    def filter_password?(received_data)
+      !@effective_user_password.empty? && @password_sent && received_data.match(Regexp.escape(@effective_user_password))
+    end
+
+    def sent_all_data?
+      effective_user_password.empty? || password_sent
+    end
+
+    def reset
+      @password_sent = false
+    end
+
+    def cli_command_prefix; end
+
+    def login_prompt; end
+  end
+
+  class SudoUserMethod < EffectiveUserMethod
+    LOGIN_PROMPT = 'rex login: '.freeze
+
+    def login_prompt
+      LOGIN_PROMPT
+    end
+
+    def cli_command_prefix
+      "sudo -p '#{LOGIN_PROMPT}' -u #{effective_user} "
+    end
+  end
+
+  class DzdoUserMethod < EffectiveUserMethod
+    LOGIN_PROMPT = /password/i.freeze
+
+    def login_prompt
+      LOGIN_PROMPT
+    end
+
+    def cli_command_prefix
+      "dzdo -u #{effective_user} "
+    end
+  end
+
+  class SuUserMethod < EffectiveUserMethod
+    LOGIN_PROMPT = /Password: /i.freeze
+
+    def login_prompt
+      LOGIN_PROMPT
+    end
+
+    def cli_command_prefix
+      "su - #{effective_user} -c "
+    end
+  end
+
+  class NoopUserMethod
+    def on_data(_, _); end
+
+    def filter_password?(received_data)
+      false
+    end
+
+    def sent_all_data?
+      true
+    end
+
+    def cli_command_prefix; end
+
+    def reset; end
+  end
+
+  # rubocop:disable Metrics/ClassLength
+  class ScriptRunner < ForemanTasksCore::Runner::Base
+    attr_reader :execution_timeout_interval
+
+    EXPECTED_POWER_ACTION_MESSAGES = ['restart host', 'shutdown host'].freeze
+    DEFAULT_REFRESH_INTERVAL = 1
+    MAX_PROCESS_RETRIES = 4
+
+    def initialize(options, user_method, suspended_action: nil)
+      super suspended_action: suspended_action
+      @host = options.fetch(:hostname)
+      @script = options.fetch(:script)
+      @ssh_user = options.fetch(:ssh_user, 'root')
+      @ssh_port = options.fetch(:ssh_port, 22)
+      @ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
+      @key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
+      @host_public_key = options.fetch(:host_public_key, nil)
+      @verify_host = options.fetch(:verify_host, nil)
+      @execution_timeout_interval = options.fetch(:execution_timeout_interval, nil)
+
+      @client_private_key_file = settings.ssh_identity_key_file
+      @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
+      @remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir)
+      @cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
+      @user_method = user_method
+    end
+
+    def self.build(options, suspended_action:)
+      effective_user = options.fetch(:effective_user, nil)
+      ssh_user = options.fetch(:ssh_user, 'root')
+      effective_user_method = options.fetch(:effective_user_method, 'sudo')
+
+      user_method = if effective_user.nil? || effective_user == ssh_user
+                      NoopUserMethod.new
+                    elsif effective_user_method == 'sudo'
+                      SudoUserMethod.new(effective_user, ssh_user,
+                                         options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
+                    elsif effective_user_method == 'dzdo'
+                      DzdoUserMethod.new(effective_user, ssh_user,
+                                         options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
+                    elsif effective_user_method == 'su'
+                      SuUserMethod.new(effective_user, ssh_user,
+                                       options.fetch(:secrets, {}).fetch(:effective_user_password, nil))
+                    else
+                      raise "effective_user_method '#{effective_user_method}' not supported"
+                    end
+
+      new(options, user_method, suspended_action: suspended_action)
+    end
+
+    def start
+      prepare_start
+      script = initialization_script
+      logger.debug("executing script:\n#{indent_multiline(script)}")
+      trigger(script)
+    rescue StandardError => e
+      logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
+      publish_exception('Error initializing command', e)
+    end
+
+    def trigger(*args)
+      run_async(*args)
+    end
+
+    def prepare_start
+      @remote_script = cp_script_to_remote
+      @output_path = File.join(File.dirname(@remote_script), 'output')
+      @exit_code_path = File.join(File.dirname(@remote_script), 'exit_code')
+    end
+
+    # the script that initiates the execution
+    def initialization_script
+      su_method = @user_method.instance_of?(SuUserMethod)
+      # pipe the output to tee while capturing the exit code in a file
+      <<-SCRIPT.gsub(/^\s+\| /, '')
+      | sh -c "(#{@user_method.cli_command_prefix}#{su_method ? "'#{@remote_script} < /dev/null '" : "#{@remote_script} < /dev/null"}; echo \\$?>#{@exit_code_path}) | /usr/bin/tee #{@output_path}
+      | exit \\$(cat #{@exit_code_path})"
+      SCRIPT
+    end
+
+    def refresh
+      return if @session.nil?
+
+      with_retries do
+        with_disconnect_handling do
+          @session.process(0)
+        end
+      end
+    ensure
+      check_expecting_disconnect
+    end
+
+    def kill
+      if @session
+        run_sync("pkill -f #{remote_command_file('script')}")
+      else
+        logger.debug('connection closed')
+      end
+    rescue StandardError => e
+      publish_exception('Unexpected error', e, false)
+    end
+
+    def timeout
+      @logger.debug('job timed out')
+      super
+    end
+
+    def timeout_interval
+      execution_timeout_interval
+    end
+
+    def with_retries
+      tries = 0
+      begin
+        yield
+      rescue StandardError => e
+        logger.error("Unexpected error: #{e.class} #{e.message}\n #{e.backtrace.join("\n")}")
+        tries += 1
+        if tries <= MAX_PROCESS_RETRIES
+          logger.error('Retrying')
+          retry
+        else
+          publish_exception('Unexpected error', e)
+        end
+      end
+    end
+
+    def with_disconnect_handling
+      yield
+    rescue IOError, Net::SSH::Disconnect => e
+      @session.shutdown!
+      check_expecting_disconnect
+      if @expecting_disconnect
+        publish_exit_status(0)
+      else
+        publish_exception('Unexpected disconnect', e)
+      end
+    end
+
+    def close
+      run_sync("rm -rf \"#{remote_command_dir}\"") if should_cleanup?
+    rescue StandardError => e
+      publish_exception('Error when removing remote working dir', e, false)
+    ensure
+      @session.close if @session && !@session.closed?
+      FileUtils.rm_rf(local_command_dir) if Dir.exist?(local_command_dir) && @cleanup_working_dirs
+    end
+
+    def publish_data(data, type)
+      super(data.force_encoding('UTF-8'), type)
+    end
+
+    private
+
+    def indent_multiline(string)
+      string.lines.map { |line| "  | #{line}" }.join
+    end
+
+    def should_cleanup?
+      @session && !@session.closed? && @cleanup_working_dirs
+    end
+
+    def session
+      @session ||= begin
+                     @logger.debug("opening session to #{@ssh_user}@#{@host}")
+                     Net::SSH.start(@host, @ssh_user, ssh_options)
+                   end
+    end
+
+    def ssh_options
+      ssh_options = {}
+      ssh_options[:port] = @ssh_port if @ssh_port
+      ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
+      ssh_options[:password] = @ssh_password if @ssh_password
+      ssh_options[:passphrase] = @key_passphrase if @key_passphrase
+      ssh_options[:keys_only] = true
+      # if the host public key is contained in the known_hosts_file,
+      # verify it, otherwise, if missing, import it and continue
+      ssh_options[:paranoid] = true
+      ssh_options[:auth_methods] = available_authentication_methods
+      ssh_options[:user_known_hosts_file] = prepare_known_hosts if @host_public_key
+      ssh_options[:number_of_password_prompts] = 1
+      ssh_options[:verbose] = settings[:ssh_log_level]
+      ssh_options[:logger] = Proxy::RemoteExecution::Ssh::LogFilter.new(SmartProxyDynflowCore::Log.instance)
+      return ssh_options
+    end
+
+    def settings
+      Proxy::RemoteExecution::Ssh::Plugin.settings
+    end
+
+    # Initiates run of the remote command and yields the data when
+    # available. The yielding doesn't happen automatically, but as
+    # part of calling the `refresh` method.
+    def run_async(command)
+      raise 'Async command already in progress' if @started
+
+      @started = false
+      @user_method.reset
+
+      session.open_channel do |channel|
+        channel.request_pty
+        channel.on_data do |ch, data|
+          publish_data(data, 'stdout') unless @user_method.filter_password?(data)
+          @user_method.on_data(data, ch)
+        end
+        channel.on_extended_data { |ch, type, data| publish_data(data, 'stderr') }
+        # standard exit of the command
+        channel.on_request('exit-status') { |ch, data| publish_exit_status(data.read_long) }
+        # on signal: sending the signal value (such as 'TERM')
+        channel.on_request('exit-signal') do |ch, data|
+          publish_exit_status(data.read_string)
+          ch.close
+          # wait for the channel to finish so that we know at the end
+          # that the session is inactive
+          ch.wait
+        end
+        channel.exec(command) do |_, success|
+          @started = true
+          raise('Error initializing command') unless success
+        end
+      end
+      session.process(0) { !run_started? }
+      return true
+    end
+
+    def run_started?
+      @started && @user_method.sent_all_data?
+    end
+
+    def run_sync(command, stdin = nil)
+      stdout = ''
+      stderr = ''
+      exit_status = nil
+      started = false
+
+      channel = session.open_channel do |ch|
+        ch.on_data do |c, data|
+          stdout.concat(data)
+        end
+        ch.on_extended_data { |_, _, data| stderr.concat(data) }
+        ch.on_request('exit-status') { |_, data| exit_status = data.read_long }
+        # Send data to stdin if we have some
+        ch.send_data(stdin) unless stdin.nil?
+        # on signal: sending the signal value (such as 'TERM')
+        ch.on_request('exit-signal') do |_, data|
+          exit_status = data.read_string
+          ch.close
+          ch.wait
+        end
+        ch.exec command do |_, success|
+          raise 'could not execute command' unless success
+
+          started = true
+        end
+      end
+      session.process(0) { !started }
+      # Closing the channel without sending any data gives us SIGPIPE
+      channel.close unless stdin.nil?
+      channel.wait
+      return exit_status, stdout, stderr
+    end
+
+    def prepare_known_hosts
+      path = local_command_file('known_hosts')
+      if @host_public_key
+        write_command_file_locally('known_hosts', "#{@host} #{@host_public_key}")
+      end
+      return path
+    end
+
+    def local_command_dir
+      File.join(@local_working_dir, 'foreman-proxy', "foreman-ssh-cmd-#{@id}")
+    end
+
+    def local_command_file(filename)
+      File.join(local_command_dir, filename)
+    end
+
+    def remote_command_dir
+      File.join(@remote_working_dir, "foreman-ssh-cmd-#{id}")
+    end
+
+    def remote_command_file(filename)
+      File.join(remote_command_dir, filename)
+    end
+
+    def ensure_local_directory(path)
+      if File.exist?(path)
+        raise "#{path} expected to be a directory" unless File.directory?(path)
+      else
+        FileUtils.mkdir_p(path)
+      end
+      return path
+    end
+
+    def cp_script_to_remote(script = @script, name = 'script')
+      path = remote_command_file(name)
+      @logger.debug("copying script to #{path}:\n#{indent_multiline(script)}")
+      upload_data(sanitize_script(script), path, 555)
+    end
+
+    def upload_data(data, path, permissions = 555)
+      ensure_remote_directory File.dirname(path)
+      # We use tee here to pipe stdin coming from ssh to a file at $path, while silencing its output
+      # This is used to write to $path with elevated permissions, solutions using cat and output redirection
+      # would not work, because the redirection would happen in the non-elevated shell.
+      command = "tee '#{path}' >/dev/null && chmod '#{permissions}' '#{path}'"
+
+      @logger.debug("Sending data to #{path} on remote host:\n#{data}")
+      status, _out, err = run_sync(command, data)
+
+      @logger.warn("Output on stderr while uploading #{path}:\n#{err}") unless err.empty?
+      if status != 0
+        raise "Unable to upload file to #{path} on remote system: exit code: #{status}"
+      end
+
+      path
+    end
+
+    def upload_file(local_path, remote_path)
+      mode = File.stat(local_path).mode.to_s(8)[-3..-1]
+      @logger.debug("Uploading local file: #{local_path} as #{remote_path} with #{mode} permissions")
+      upload_data(File.read(local_path), remote_path, mode)
+    end
+
+    def ensure_remote_directory(path)
+      exit_code, _output, err = run_sync("mkdir -p #{path}")
+      if exit_code != 0
+        raise "Unable to create directory on remote system #{path}: exit code: #{exit_code}\n #{err}"
+      end
+    end
+
+    def sanitize_script(script)
+      script.tr("\r", '')
+    end
+
+    def write_command_file_locally(filename, content)
+      path = local_command_file(filename)
+      ensure_local_directory(File.dirname(path))
+      File.write(path, content)
+      return path
+    end
+
+    # when a remote server disconnects, it's hard to tell if it was on purpose (when calling reboot)
+    # or it's an error. When it's expected, we expect the script to produce 'restart host' as
+    # its last command output
+    def check_expecting_disconnect
+      last_output = @continuous_output.raw_outputs.find { |d| d['output_type'] == 'stdout' }
+      return unless last_output
+
+      if EXPECTED_POWER_ACTION_MESSAGES.any? { |message| last_output['output'] =~ /^#{message}/ }
+        @expecting_disconnect = true
+      end
+    end
+
+    def available_authentication_methods
+      methods = %w[publickey] # Always use pubkey auth as fallback
+      if settings[:kerberos_auth]
+        if defined? Net::SSH::Kerberos
+          methods << 'gssapi-with-mic'
+        else
+          @logger.warn('Kerberos authentication requested but not available')
+        end
+      end
+      methods.unshift('password') if @ssh_password
+
+      methods
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.3.1'
+      VERSION = '0.4.0'
     end
   end
 end

data/settings.d/remote_execution_ssh.yml.example

@@ -15,3 +15,6 @@
 # one of :debug, :info, :warn, :error, :fatal
 # must be lower than general log level
 # :ssh_log_level: fatal
+
+# Remove working directories on job completion
+# :cleanup_working_dirs: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-01 00:00:00.000000000 Z
+date: 2021-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.82.0
+- !ruby/object:Gem::Dependency
+  name: foreman-tasks-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: smart_proxy_dynflow
   requirement: !ruby/object:Gem::Requirement
@@ -149,10 +163,19 @@ files:
 - README.md
 - bundler.plugins.d/remote_execution_ssh.rb
 - lib/smart_proxy_remote_execution_ssh.rb
+- lib/smart_proxy_remote_execution_ssh/actions/run_script.rb
 - lib/smart_proxy_remote_execution_ssh/api.rb
+- lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh
+- lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh
 - lib/smart_proxy_remote_execution_ssh/cockpit.rb
+- lib/smart_proxy_remote_execution_ssh/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
+- lib/smart_proxy_remote_execution_ssh/log_filter.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb
+- lib/smart_proxy_remote_execution_ssh/runners.rb
+- lib/smart_proxy_remote_execution_ssh/runners/fake_script_runner.rb
+- lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb
+- lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb
 - lib/smart_proxy_remote_execution_ssh/version.rb
 - lib/smart_proxy_remote_execution_ssh/webrick_ext.rb
 - settings.d/remote_execution_ssh.yml.example