smart_proxy_remote_execution_ssh 0.5.3 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abf6517f6a98386e246650454c7deb049b7583c7653040eb63bbdf8e3cc27f0a
-  data.tar.gz: 12b0aa0d067e45c73ee65286f69d1556f7ae8b1c47624845e1e0a7e96c341e95
+  metadata.gz: 1a599d3732d4f6b064a850ae8baced28354980096785a340828533f953f844c4
+  data.tar.gz: 160b80ab983f0eb7b987987191b48c19db42d3b76079b0ba6639ec831dccc6f3
 SHA512:
-  metadata.gz: f873d27dc9b5ba0b9c10d7d227058618d901a48643ba31e491e1530b34356aa88ac8a197e9731d83a2a535ffa7577ba674578af8b90de3d02f7537d6359063d5
-  data.tar.gz: d72dfc490d0f71a076885eeabf435a89bca3658ce31a465666725a19e1d0f3c682d51f899922a2b6dbfdbfe8ee8d880fc0b0d0a13ac2536c5ce8f276f64b059b
+  metadata.gz: 349961aa474809225d9781bda09687eb23f00db51cca9f758a57e075fa3d19fc07b729569b064887f91bb7f07190152d6ec9cb01e8ccf48df0841e9c3f85d10e
+  data.tar.gz: 8ad40859c92f7d1cd089adf0f37bb0aaa8add745cd461c2e1ed5b484d3ae6b856ac0ee53e61c62e89d564a4ba676b22bf5f01618da970986cd60e42591ff6346

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -1,5 +1,6 @@
 require 'mqtt'
 require 'json'
+require 'time'
 
 module Proxy::RemoteExecution::Ssh::Actions
   class PullScript < Proxy::Dynflow::Action::Runner
@@ -25,7 +26,8 @@ module Proxy::RemoteExecution::Ssh::Actions
       otp_password = if input[:with_mqtt]
                        ::Proxy::Dynflow::OtpManager.generate_otp(execution_plan_id)
                      end
-      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script])
+
+      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''))
       output[:state] = :ready_for_pickup
       output[:result] = []
       mqtt_start(otp_password) if input[:with_mqtt]
@@ -40,9 +42,40 @@ module Proxy::RemoteExecution::Ssh::Actions
     def process_external_event(event)
       output[:state] = :running
       data = event.data
+      case data['version']
+      when nil
+        process_external_unversioned(data)
+      when '1'
+        process_external_v1(data)
+      else
+        raise "Unexpected update message version '#{data['version']}'"
+      end
+    end
+
+    def process_external_unversioned(payload)
+      continuous_output = Proxy::Dynflow::ContinuousOutput.new
+      Array(payload['output']).each { |line| continuous_output.add_output(line, payload['type']) } if payload.key?('output')
+      exit_code = payload['exit_code'].to_i if payload['exit_code']
+      process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
+    end
+
+    def process_external_v1(payload)
       continuous_output = Proxy::Dynflow::ContinuousOutput.new
-      Array(data['output']).each { |line| continuous_output.add_output(line, 'stdout') } if data.key?('output')
-      exit_code = data['exit_code'].to_i if data['exit_code']
+      exit_code = nil
+
+      payload['updates'].each do |update|
+        time = Time.parse update['timestamp']
+        type = update['type']
+        case type
+        when 'output'
+          continuous_output.add_output(update['content'], update['stream'], time)
+        when 'exit'
+          exit_code = update['exit_code'].to_i
+        else
+          raise "Unexpected update type '#{update['type']}'"
+        end
+      end
+
       process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
     end
 
@@ -56,37 +89,52 @@ module Proxy::RemoteExecution::Ssh::Actions
         # Client was notified or is already running, dealing with this situation
         # is only supported if mqtt is available
         # Otherwise we have to wait it out
-        # TODO
-        # if input[:with_mqtt]
+        mqtt_cancel if input[:with_mqtt]
       end
       suspend
     end
 
     def mqtt_start(otp_password)
-      payload = {
-        type: 'data',
-        message_id: SecureRandom.uuid,
-        version: 1,
-        sent: DateTime.now.iso8601,
-        directive: 'foreman',
+      payload = mqtt_payload_base.merge(
+        content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}",
         metadata: {
+          'event': 'start',
           'job_uuid': input[:job_uuid],
           'username': execution_plan_id,
           'password': otp_password,
           'return_url': "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/update",
         },
-        content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}",
-      }
+      )
       mqtt_notify payload
       output[:state] = :notified
     end
 
+    def mqtt_cancel
+      cleanup
+      payload = mqtt_payload_base.merge(
+        metadata: {
+          'event': 'cancel',
+          'job_uuid': input[:job_uuid]
+        }
+      )
+      mqtt_notify payload
+    end
+
     def mqtt_notify(payload)
-      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port) do |c|
+      with_mqtt_client do |c|
         c.publish(mqtt_topic, JSON.dump(payload), false, 1)
       end
     end
 
+    def with_mqtt_client(&block)
+      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
+                           :ssl => settings.mqtt_tls,
+                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
+                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
+                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
+                           &block)
+    end
+
     def host_name
       alternative_names = input.fetch(:alternative_names, {})
 
@@ -106,5 +154,15 @@ module Proxy::RemoteExecution::Ssh::Actions
     def job_storage
       Proxy::RemoteExecution::Ssh.job_storage
     end
+
+    def mqtt_payload_base
+      {
+        type: 'data',
+        message_id: SecureRandom.uuid,
+        version: 1,
+        sent: DateTime.now.iso8601,
+        directive: 'foreman'
+      }
+    end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -13,14 +13,16 @@ module Proxy::RemoteExecution
         File.read(Ssh.public_key_file)
       end
 
-      post "/session" do
-        do_authorize_any
-        session = Cockpit::Session.new(env)
-        unless session.valid?
-          return [ 400, "Invalid request: /ssh/session requires connection upgrade to 'raw'" ]
+      if Proxy::RemoteExecution::Ssh::Plugin.settings.cockpit_integration
+        post "/session" do
+          do_authorize_any
+          session = Cockpit::Session.new(env)
+          unless session.valid?
+            return [ 400, "Invalid request: /ssh/session requires connection upgrade to 'raw'" ]
+          end
+          session.hijack!
+          101
         end
-        session.hijack!
-        101
       end
 
       delete '/known_hosts/:name' do |name|

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -1,5 +1,6 @@
 require 'smart_proxy_remote_execution_ssh/net_ssh_compat'
 require 'forwardable'
+require 'securerandom'
 
 module Proxy::RemoteExecution
   module Cockpit
@@ -164,9 +165,8 @@ module Proxy::RemoteExecution
         out_read, out_write = IO.pipe
         err_read, err_write = IO.pipe
 
-        # Force the script runner to initialize its logger
-        script_runner.logger
-        pid = spawn(*script_runner.send(:get_args, command), :in => in_read, :out => out_write, :err => err_write)
+        connection.establish!
+        pid = spawn(*connection.command(command), :in => in_read, :out => out_write, :err => err_write)
         [in_read, out_write, err_write].each(&:close)
 
         send_start
@@ -176,19 +176,22 @@ module Proxy::RemoteExecution
         in_buf  = MiniSSLBufferedSocket.new(in_write)
 
         inner_system_ssh_loop out_buf, err_buf, in_buf, pid
+      ensure
+        connection.disconnect!
       end
 
       def inner_system_ssh_loop(out_buf, err_buf, in_buf, pid)
         err_buf_raw = ''
-        readers = [buf_socket, out_buf, err_buf]
         loop do
+          readers = [buf_socket, out_buf, err_buf].reject { |io| io.closed? }
+          writers = [buf_socket, in_buf].select { |io| io.pending_writes? }
           # Prime the sockets for reading
-          ready_readers, ready_writers = IO.select(readers, [buf_socket, in_buf], nil, 300)
+          ready_readers, ready_writers = IO.select(readers, writers)
           (ready_readers || []).each { |reader| reader.close if reader.fill.zero? }
 
           proxy_data(out_buf, in_buf)
           if buf_socket.closed?
-            script_runner.close_session
+            connection.disconnect!
           end
 
           if out_buf.closed?
@@ -262,10 +265,10 @@ module Proxy::RemoteExecution
         params["hostname"]
       end
 
-      def script_runner
-        @script_runner ||= Proxy::RemoteExecution::Ssh::Runners::ScriptRunner.build(
+      def connection
+        @connection ||= Proxy::RemoteExecution::Ssh::Runners::MultiplexedSSHConnection.new(
           runner_params,
-          suspended_action: nil
+          logger: logger
         )
       end
 
@@ -278,6 +281,7 @@ module Proxy::RemoteExecution
         # For compatibility only
         ret[:script] = nil
         ret[:hostname] = host
+        ret[:id] = SecureRandom.uuid
         ret
       end
     end

data/lib/smart_proxy_remote_execution_ssh/command_logging.rb

@@ -0,0 +1,23 @@
+# lib/command_logging.rb
+
+module Proxy::RemoteExecution::Ssh::Runners
+  module CommandLogging
+    def log_command(command, label: "Running")
+      command = command.join(' ')
+      label = "#{label}: " if label
+      logger.debug("#{label}#{command}")
+    end
+
+    def set_pm_debug_logging(pm, capture: false, user_method: nil)
+      callback = proc do |data|
+        data.each_line do |line|
+          logger.debug(line.chomp) if user_method.nil? || !user_method.filter_password?(line)
+          user_method.on_data(data, pm.stdin) if user_method
+        end
+        ''
+      end
+      pm.on_stdout(&callback)
+      pm.on_stderr(&callback)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb

@@ -0,0 +1,196 @@
+require 'smart_proxy_remote_execution_ssh/command_logging'
+
+module Proxy::RemoteExecution::Ssh::Runners
+  class SensitiveString
+    def initialize(value, mask: '*****')
+      @value = value
+      @mask = mask
+    end
+
+    def inspect
+      '"' + to_s + '"'
+    end
+
+    def to_s
+      @mask
+    end
+
+    def to_str
+      @value
+    end
+  end
+
+  class AuthenticationMethod
+    attr_reader :name
+    def initialize(name, prompt: nil, password: nil)
+      @name = name
+      @prompt = prompt
+      @password = password
+    end
+
+    def ssh_command_prefix
+      return [] unless @password
+
+      prompt = ['-P', @prompt] if @prompt
+      [{'SSHPASS' => SensitiveString.new(@password)}, '/usr/bin/sshpass', '-e', prompt].compact
+    end
+
+    def ssh_options
+      ["-o PreferredAuthentications=#{name}",
+       "-o NumberOfPasswordPrompts=#{@password ? 1 : 0}"]
+    end
+  end
+
+  class MultiplexedSSHConnection
+    include CommandLogging
+
+    attr_reader :logger
+    def initialize(options, logger:)
+      @logger = logger
+
+      @id = options.fetch(:id)
+      @host = options.fetch(:hostname)
+      @script = options.fetch(:script)
+      @ssh_user = options.fetch(:ssh_user, 'root')
+      @ssh_port = options.fetch(:ssh_port, 22)
+      @ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
+      @key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
+      @host_public_key = options.fetch(:host_public_key, nil)
+      @verify_host = options.fetch(:verify_host, nil)
+      @client_private_key_file = settings.ssh_identity_key_file
+
+      @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
+      @socket_working_dir = options.fetch(:socket_working_dir, settings.socket_working_dir)
+      @socket = nil
+    end
+
+    def establish!
+      @available_auth_methods ||= available_authentication_methods
+      method = @available_auth_methods.find do |method|
+        if try_auth_method(method)
+          @available_auth_methods.unshift(method).uniq!
+          true
+        end
+      end
+      method || raise("Could not establish connection to remote host using any available authentication method, tried #{@available_auth_methods.map(&:name).join(', ')}")
+    end
+
+    def disconnect!
+      return unless connected?
+
+      cmd = command(%w[-O exit])
+      log_command(cmd, label: "Closing shared connection")
+      pm = Proxy::Dynflow::ProcessManager.new(cmd)
+      set_pm_debug_logging(pm)
+      pm.run!
+      @socket = nil
+    end
+
+    def connected?
+      !@socket.nil?
+    end
+
+    def command(cmd)
+      raise "Cannot build command to run over multiplexed connection without having an established connection" unless connected?
+
+      ['/usr/bin/ssh', reuse_ssh_options, cmd].flatten
+    end
+
+    private
+
+    def try_auth_method(method)
+      # running "ssh -f -N" instead of "ssh true" would be cleaner, but ssh
+      # does not close its stderr which trips up the process manager which
+      # expects all FDs to be closed
+
+      full_command = [method.ssh_command_prefix, '/usr/bin/ssh', establish_ssh_options, method.ssh_options, @host, 'true'].flatten
+      log_command(full_command)
+      pm = Proxy::Dynflow::ProcessManager.new(full_command)
+      pm.start!
+      if pm.status
+        raise pm.stderr.to_s
+      else
+        set_pm_debug_logging(pm)
+        pm.stdin.io.close
+        pm.run!
+      end
+
+      if pm.status.zero?
+        logger.debug("Established connection using authentication method #{method.name}")
+        @socket = socket_file
+        true
+      else
+        logger.debug("Failed to establish connection using authentication method #{method.name}")
+        false
+      end
+    end
+
+    def settings
+      Proxy::RemoteExecution::Ssh::Plugin.settings
+    end
+
+    def available_authentication_methods
+      methods = []
+      methods << AuthenticationMethod.new('password', password: @ssh_password) if @ssh_password
+      if verify_key_passphrase
+        methods << AuthenticationMethod.new('publickey', password: @key_passphrase, prompt: 'passphrase')
+      end
+      methods << AuthenticationMethod.new('gssapi-with-mic') if settings[:kerberos_auth]
+      raise "There are no available authentication methods" if methods.empty?
+      methods
+    end
+
+    def establish_ssh_options
+      return @establish_ssh_options if @establish_ssh_options
+      ssh_options = []
+      ssh_options << "-o User=#{@ssh_user}"
+      ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
+      ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
+      ssh_options << "-o IdentitiesOnly=yes"
+      ssh_options << "-o StrictHostKeyChecking=no"
+      ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
+      ssh_options << "-o LogLevel=#{ssh_log_level(true)}"
+      ssh_options << "-o ControlMaster=auto"
+      ssh_options << "-o ControlPath=#{socket_file}"
+      ssh_options << "-o ControlPersist=yes"
+      ssh_options << "-o ProxyCommand=none"
+      @establish_ssh_options = ssh_options
+    end
+
+    def reuse_ssh_options
+      ["-o", "ControlPath=#{@socket}", "-o", "LogLevel=#{ssh_log_level(false)}", @host]
+    end
+
+    def socket_file
+      File.join(@socket_working_dir, @id)
+    end
+
+    def verify_key_passphrase
+      command = ['/usr/bin/ssh-keygen', '-y', '-f', File.expand_path(@client_private_key_file)]
+      log_command(command, label: "Checking if private key has passphrase")
+      pm = Proxy::Dynflow::ProcessManager.new(command)
+      pm.start!
+
+      raise pm.stderr.to_s if pm.status
+
+      pm.stdin.io.close
+      pm.run!
+
+      if pm.status.zero?
+        logger.debug("Private key is not protected with a passphrase")
+        @key_passphrase = nil
+      else
+        logger.debug("Private key is protected with a passphrase")
+      end
+
+      return true if pm.status.zero? || @key_passphrase
+
+      logger.debug("Private key is protected with a passphrase, but no passphrase was provided")
+      false
+    end
+
+    def ssh_log_level(new_connection)
+      new_connection ? settings[:ssh_log_level] : 'quiet'
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb

@@ -173,10 +173,14 @@ module Proxy::RemoteExecution
         output.append(data)
       end
 
+      def pending_writes?
+        output.length.positive?
+      end
+
       # Sends as much of the pending output as possible. Returns +true+ if any
       # data was sent, and +false+ otherwise.
       def send_pending
-        if output.length.positive?
+        if pending_writes?
           sent = send(output.to_s, 0)
           output.consume!(sent)
           return sent.positive?
@@ -189,7 +193,7 @@ module Proxy::RemoteExecution
       # buffer is empty.
       def wait_for_pending_sends
         send_pending
-        while output.length.positive?
+        while pending_writes?
           result = IO.select(nil, [self]) || next
           next unless result[1].any?
 

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -1,7 +1,11 @@
 module Proxy::RemoteExecution::Ssh
   class Plugin < Proxy::Plugin
     SSH_LOG_LEVELS = %w[debug info error fatal].freeze
-    MODES = %i[ssh async-ssh pull pull-mqtt].freeze
+    MODES = %i[ssh ssh-async pull pull-mqtt].freeze
+    # Unix domain socket path length is limited to 104 (on some platforms) characters
+    # Socket path is composed of custom path (max 49 characters) + job id (37 characters)
+    # + offset(17 characters) + null terminator
+    SOCKET_PATH_MAX_LENGTH = 49
 
     http_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
@@ -11,16 +15,21 @@ module Proxy::RemoteExecution::Ssh
                      :ssh_user                => 'root',
                      :remote_working_dir      => '/var/tmp',
                      :local_working_dir       => '/var/tmp',
+                     :socket_working_dir      => '/var/tmp',
                      :kerberos_auth           => false,
+                     :cockpit_integration     => true,
                      # When set to nil, makes REX use the runner's default interval
                      # :runner_refresh_interval => nil,
                      :ssh_log_level           => :error,
                      :cleanup_working_dirs    => true,
                      # :mqtt_broker             => nil,
                      # :mqtt_port               => nil,
+                     # :mqtt_tls                => nil,
                      :mode                    => :ssh
 
-    plugin :ssh, Proxy::RemoteExecution::Ssh::VERSION
+    capability(proc { 'cockpit' if settings.cockpit_integration })
+
+    plugin :script, Proxy::RemoteExecution::Ssh::VERSION
     after_activation do
       require 'smart_proxy_dynflow'
       require 'smart_proxy_remote_execution_ssh/version'

data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb

@@ -36,13 +36,12 @@ module Proxy::RemoteExecution::Ssh::Runners
     def initialization_script
       close_stdin = '</dev/null'
       close_fds = close_stdin + ' >/dev/null 2>/dev/null'
-      main_script = "(#{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
+      main_script = "(#{@remote_script_wrapper} #{@remote_script} #{close_stdin} 2>&1; echo $?>#{@base_dir}/init_exit_code) >#{@base_dir}/output"
       control_script_finish = "#{@control_script_path} init-script-finish"
       <<-SCRIPT.gsub(/^ +\| /, '')
       | export CONTROL_SCRIPT="#{@control_script_path}"
-      | #{"chown #{@user_method.effective_user} '#{@base_dir}'" if @user_method.cli_command_prefix}
+      | #{"chown #{@user_method.effective_user} #{@base_dir}" if @user_method.cli_command_prefix}
       | #{@user_method.cli_command_prefix} sh -c '#{main_script}; #{control_script_finish}' #{close_fds} &
-      | echo $! > '#{@base_dir}/pid'
       SCRIPT
     end
 
@@ -51,18 +50,23 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def refresh
-      err = output = nil
+      @connection.establish! unless @connection.connected?
       begin
-        _, output, err = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
+        pm = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
+        process_retrieved_data(pm.stdout.to_s.chomp, pm.stderr.to_s.chomp)
       rescue StandardError => e
         @logger.info("Error while connecting to the remote host on refresh: #{e.message}")
       end
-
-      process_retrieved_data(output, err)
     ensure
       destroy_session
     end
 
+    def kill
+      run_sync("pkill -P $(cat #{@pid_path})")
+    rescue StandardError => e
+      publish_exception('Unexpected error', e, false)
+    end
+
     def process_retrieved_data(output, err)
       return if output.nil? || output.empty?
 
@@ -127,11 +131,15 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def cleanup
-      run_sync("rm -rf \"#{remote_command_dir}\"") if @cleanup_working_dirs
+      if @cleanup_working_dirs
+        ensure_remote_command("rm -rf #{remote_command_dir}",
+                              publish: true,
+                              error: "Unable to remove working directory #{remote_command_dir} on remote system, exit code: %{exit_code}")
+      end
     end
 
     def destroy_session
-      if @session
+      if @connection.connected?
         @logger.debug("Closing session with #{@ssh_user}@#{@host}")
         close_session
       end

data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb

@@ -1,5 +1,7 @@
 require 'fileutils'
-require 'smart_proxy_dynflow/runner/command'
+require 'smart_proxy_dynflow/runner/process_manager_command'
+require 'smart_proxy_dynflow/process_manager'
+require 'smart_proxy_remote_execution_ssh/multiplexed_ssh_connection'
 
 module Proxy::RemoteExecution::Ssh::Runners
   class EffectiveUserMethod
@@ -12,9 +14,9 @@ module Proxy::RemoteExecution::Ssh::Runners
       @password_sent = false
     end
 
-    def on_data(received_data, ssh_channel)
+    def on_data(received_data, io_buffer)
       if received_data.match(login_prompt)
-        ssh_channel.puts(effective_user_password)
+        io_buffer.add_data(effective_user_password + "\n")
         @password_sent = true
       end
     end
@@ -90,7 +92,9 @@ module Proxy::RemoteExecution::Ssh::Runners
 
   # rubocop:disable Metrics/ClassLength
   class ScriptRunner < Proxy::Dynflow::Runner::Base
-    include Proxy::Dynflow::Runner::Command
+    include Proxy::Dynflow::Runner::ProcessManagerCommand
+    include CommandLogging
+
     attr_reader :execution_timeout_interval
 
     EXPECTED_POWER_ACTION_MESSAGES = ['restart host', 'shutdown host'].freeze
@@ -102,18 +106,17 @@ module Proxy::RemoteExecution::Ssh::Runners
       @script = options.fetch(:script)
       @ssh_user = options.fetch(:ssh_user, 'root')
       @ssh_port = options.fetch(:ssh_port, 22)
-      @ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
-      @key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
       @host_public_key = options.fetch(:host_public_key, nil)
-      @verify_host = options.fetch(:verify_host, nil)
       @execution_timeout_interval = options.fetch(:execution_timeout_interval, nil)
 
       @client_private_key_file = settings.ssh_identity_key_file
       @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
-      @remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir)
+      @remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir.shellescape)
+      @socket_working_dir = options.fetch(:socket_working_dir, settings.socket_working_dir)
       @cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
       @first_execution = options.fetch(:first_execution, false)
       @user_method = user_method
+      @options = options
     end
 
     def self.build(options, suspended_action:)
@@ -141,6 +144,10 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def start
       Proxy::RemoteExecution::Utils.prune_known_hosts!(@host, @ssh_port, logger) if @first_execution
+      ensure_local_directory(@socket_working_dir)
+      @connection = MultiplexedSSHConnection.new(@options.merge(:id => @id), logger: logger)
+      @connection.establish!
+      preflight_checks
       prepare_start
       script = initialization_script
       logger.debug("executing script:\n#{indent_multiline(script)}")
@@ -154,32 +161,51 @@ module Proxy::RemoteExecution::Ssh::Runners
       run_async(*args)
     end
 
+    def preflight_checks
+      ensure_remote_command(cp_script_to_remote("#!/bin/sh\nexec true", 'test'),
+        error: 'Failed to execute script on remote machine, exit code: %{exit_code}.'
+      )
+      unless @user_method.is_a? NoopUserMethod
+        path = cp_script_to_remote("#!/bin/sh\nexec #{@user_method.cli_command_prefix} true", 'effective-user-test')
+        ensure_remote_command(path,
+                              error: 'Failed to change to effective user, exit code: %{exit_code}',
+                              tty: true,
+                              user_method: @user_method,
+                              close_stdin: false)
+      end
+    end
+
     def prepare_start
       @remote_script = cp_script_to_remote
       @output_path = File.join(File.dirname(@remote_script), 'output')
       @exit_code_path = File.join(File.dirname(@remote_script), 'exit_code')
+      @pid_path = File.join(File.dirname(@remote_script), 'pid')
+      @remote_script_wrapper = upload_data("echo $$ > #{@pid_path}; exec \"$@\";", File.join(File.dirname(@remote_script), 'script-wrapper'), 555)
     end
 
     # the script that initiates the execution
     def initialization_script
       su_method = @user_method.instance_of?(SuUserMethod)
       # pipe the output to tee while capturing the exit code in a file
-      <<-SCRIPT.gsub(/^\s+\| /, '')
-      | sh -c "(#{@user_method.cli_command_prefix}#{su_method ? "'#{@remote_script} < /dev/null '" : "#{@remote_script} < /dev/null"}; echo \\$?>#{@exit_code_path}) | /usr/bin/tee #{@output_path}
-      | exit \\$(cat #{@exit_code_path})"
+      <<~SCRIPT
+        sh <<EOF | /usr/bin/tee #{@output_path}
+        #{@remote_script_wrapper} #{@user_method.cli_command_prefix}#{su_method ? "'#{@remote_script} < /dev/null '" : "#{@remote_script} < /dev/null"}
+        echo \\$?>#{@exit_code_path}
+        EOF
+        exit $(cat #{@exit_code_path})
       SCRIPT
     end
 
     def refresh
-      return if @session.nil?
+      return if @process_manager.nil?
       super
     ensure
       check_expecting_disconnect
     end
 
     def kill
-      if @session
-        run_sync("pkill -f #{remote_command_file('script')}")
+      if @process_manager&.started?
+        run_sync("pkill -P $(cat #{@pid_path})")
       else
         logger.debug('connection closed')
       end
@@ -197,28 +223,24 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def close_session
-      @session = nil
-      raise 'Control socket file does not exist' unless File.exist?(local_command_file("socket"))
+      raise 'Control socket file does not exist' unless File.exist?(socket_file)
       @logger.debug("Sending exit request for session #{@ssh_user}@#{@host}")
-      args = ['/usr/bin/ssh', @host, "-o", "User=#{@ssh_user}", "-o", "ControlPath=#{local_command_file("socket")}", "-O", "exit"].flatten
-      pid, *, err = session(args, in_stream: false, out_stream: false)
-      result = read_output_debug(err)
-      Process.wait(pid)
-      result
+      @connection.disconnect!
     end
 
     def close
-      run_sync("rm -rf \"#{remote_command_dir}\"") if should_cleanup?
+      run_sync("rm -rf #{remote_command_dir}") if should_cleanup?
     rescue StandardError => e
       publish_exception('Error when removing remote working dir', e, false)
     ensure
-      close_session if @session
+      close_session if @process_manager
       FileUtils.rm_rf(local_command_dir) if Dir.exist?(local_command_dir) && @cleanup_working_dirs
     end
 
-    def publish_data(data, type)
+    def publish_data(data, type, pm = nil)
+      pm ||= @process_manager
       super(data.force_encoding('UTF-8'), type) unless @user_method.filter_password?(data)
-      @user_method.on_data(data, @command_in)
+      @user_method.on_data(data, pm.stdin) if pm
     end
 
     private
@@ -228,94 +250,47 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def should_cleanup?
-      @session && @cleanup_working_dirs
-    end
-
-    # Creates session with three pipes - one for reading and two for
-    # writing. Similar to `Open3.popen3` method but without creating
-    # a separate thread to monitor it.
-    def session(args, in_stream: true, out_stream: true, err_stream: true)
-      @session = true
-
-      in_read, in_write = in_stream ? IO.pipe : '/dev/null'
-      out_read, out_write = out_stream ? IO.pipe : [nil, '/dev/null']
-      err_read, err_write = err_stream ? IO.pipe : [nil, '/dev/null']
-      command_pid = spawn(*args, :in => in_read, :out => out_write, :err => err_write)
-      in_read.close if in_stream
-      out_write.close if out_stream
-      err_write.close if err_stream
-
-      return command_pid, in_write, out_read, err_read
-    end
-
-    def ssh_options(with_pty = false)
-      ssh_options = []
-      ssh_options << "-tt" if with_pty
-      ssh_options << "-o User=#{@ssh_user}"
-      ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
-      ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
-      ssh_options << "-o IdentitiesOnly=yes"
-      ssh_options << "-o StrictHostKeyChecking=no"
-      ssh_options << "-o PreferredAuthentications=#{available_authentication_methods.join(',')}"
-      ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
-      ssh_options << "-o NumberOfPasswordPrompts=1"
-      ssh_options << "-o LogLevel=#{settings[:ssh_log_level]}"
-      ssh_options << "-o ControlMaster=auto"
-      ssh_options << "-o ControlPath=#{local_command_file("socket")}"
-      ssh_options << "-o ControlPersist=yes"
+      @process_manager && @cleanup_working_dirs
     end
 
     def settings
       Proxy::RemoteExecution::Ssh::Plugin.settings
     end
 
-    def get_args(command, with_pty = false)
-      args = []
-      args = [{'SSHPASS' => @key_passphrase}, '/usr/bin/sshpass', '-P', 'passphrase', '-e'] if @key_passphrase
-      args = [{'SSHPASS' => @ssh_password}, '/usr/bin/sshpass', '-e'] if @ssh_password
-      args += ['/usr/bin/ssh', @host, ssh_options(with_pty), command].flatten
-    end
-
     # Initiates run of the remote command and yields the data when
     # available. The yielding doesn't happen automatically, but as
     # part of calling the `refresh` method.
     def run_async(command)
-      raise 'Async command already in progress' if @started
+      raise 'Async command already in progress' if @process_manager&.started?
 
-      @started = false
       @user_method.reset
-      @command_pid, @command_in, @command_out = session(get_args(command, with_pty: true), err_stream: false)
-      @started = true
+      cmd = @connection.command([tty_flag(true), command].flatten.compact)
+      log_command(cmd)
+      initialize_command(*cmd)
 
-      return true
+      true
     end
 
     def run_started?
-      @started && @user_method.sent_all_data?
+      @process_manager&.started? && @user_method.sent_all_data?
     end
 
-    def read_output_debug(err_io, out_io = nil)
-      stdout = ''
-      debug_str = ''
-
-      if out_io
-        stdout += out_io.read until out_io.eof? rescue
-        out_io.close
-      end
-      debug_str += err_io.read until err_io.eof? rescue
-      err_io.close
-      debug_str.lines.each { |line| @logger.debug(line.strip) }
-
-      return stdout, debug_str
+    def tty_flag(tty)
+      '-tt' if tty
     end
 
-    def run_sync(command, stdin = nil)
-      pid, tx, rx, err = session(get_args(command))
-      tx.puts(stdin) unless stdin.nil?
-      tx.close
-      stdout, stderr = read_output_debug(err, rx)
-      exit_status = Process.wait2(pid)[1].exitstatus
-      return exit_status, stdout, stderr
+    def run_sync(command, stdin: nil, close_stdin: true, tty: false, user_method: nil)
+      cmd = @connection.command([tty_flag(tty), command].flatten.compact)
+      log_command(cmd)
+      pm = Proxy::Dynflow::ProcessManager.new(cmd)
+      set_pm_debug_logging(pm, user_method: user_method)
+      pm.start!
+      unless pm.status
+        pm.stdin.io.puts(stdin) if stdin
+        pm.stdin.io.close if close_stdin
+        pm.run!
+      end
+      pm
     end
 
     def prepare_known_hosts
@@ -334,6 +309,10 @@ module Proxy::RemoteExecution::Ssh::Runners
       File.join(ensure_local_directory(local_command_dir), filename)
     end
 
+    def socket_file
+      File.join(ensure_local_directory(@socket_working_dir), @id)
+    end
+
     def remote_command_dir
       File.join(@remote_working_dir, "foreman-ssh-cmd-#{id}")
     end
@@ -362,15 +341,13 @@ module Proxy::RemoteExecution::Ssh::Runners
       # We use tee here to pipe stdin coming from ssh to a file at $path, while silencing its output
       # This is used to write to $path with elevated permissions, solutions using cat and output redirection
       # would not work, because the redirection would happen in the non-elevated shell.
-      command = "tee '#{path}' >/dev/null && chmod '#{permissions}' '#{path}'"
+      command = "tee #{path} >/dev/null && chmod #{permissions} #{path}"
 
       @logger.debug("Sending data to #{path} on remote host:\n#{data}")
-      status, _out, err = run_sync(command, data)
-
-      @logger.warn("Output on stderr while uploading #{path}:\n#{err}") unless err.empty?
-      if status != 0
-        raise "Unable to upload file to #{path} on remote system: exit code: #{status}"
-      end
+      ensure_remote_command(command,
+        stdin: data,
+        error: "Unable to upload file to #{path} on remote system, exit code: %{exit_code}"
+      )
 
       path
     end
@@ -382,9 +359,15 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def ensure_remote_directory(path)
-      exit_code, _output, err = run_sync("mkdir -p #{path}")
-      if exit_code != 0
-        raise "Unable to create directory on remote system #{path}: exit code: #{exit_code}\n #{err}"
+      ensure_remote_command("mkdir -p #{path}",
+        error: "Unable to create directory #{path} on remote system, exit code: %{exit_code}"
+      )
+    end
+
+    def ensure_remote_command(cmd, error: nil, **kwargs)
+      if (pm = run_sync(cmd, **kwargs)).status != 0
+        msg = error || 'Failed to run command %{command} on remote machine, exit code: %{exit_code}'
+        raise(msg % { command: cmd, exit_code: pm.status })
       end
     end
 
@@ -410,13 +393,6 @@ module Proxy::RemoteExecution::Ssh::Runners
         @expecting_disconnect = true
       end
     end
-
-    def available_authentication_methods
-      methods = %w[publickey] # Always use pubkey auth as fallback
-      methods << 'gssapi-with-mic' if settings[:kerberos_auth]
-      methods.unshift('password') if @ssh_password
-      methods
-    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.5.3'
+      VERSION = '0.8.0'
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -7,22 +7,10 @@ module Proxy::RemoteExecution
   module Ssh
     class << self
       def validate!
-        unless private_key_file
-          raise "settings for `ssh_identity_key` not set"
-        end
-
-        unless File.exist?(private_key_file)
-          raise "Ssh public key file #{private_key_file} doesn't exist.\n"\
-            "You can generate one with `ssh-keygen -t rsa -b 4096 -f #{private_key_file} -N ''`"
-        end
-
-        unless File.exist?(public_key_file)
-          raise "Ssh public key file #{public_key_file} doesn't exist"
-        end
-
         validate_mode!
-        validate_ssh_log_level!
+        validate_ssh_settings!
         validate_mqtt_settings!
+        validate_socket_path!
       end
 
       def private_key_file
@@ -60,6 +48,28 @@ module Proxy::RemoteExecution
 
         raise 'mqtt_broker has to be set when pull-mqtt mode is used' if Plugin.settings.mqtt_broker.nil?
         raise 'mqtt_port has to be set when pull-mqtt mode is used' if Plugin.settings.mqtt_port.nil?
+
+        if Plugin.settings.mqtt_tls.nil?
+          Plugin.settings.mqtt_tls = [[:foreman_ssl_cert, :ssl_certificate], [:foreman_ssl_key, :ssl_private_key], [:foreman_ssl_ca, :ssl_ca_file]].all? { |(client, server)| ::Proxy::SETTINGS[client] || ::Proxy::SETTINGS[server] }
+        end
+      end
+
+      def validate_ssh_settings!
+        return unless requires_configured_ssh?
+        unless private_key_file
+          raise "settings for `ssh_identity_key` not set"
+        end
+
+        unless File.exist?(private_key_file)
+          raise "SSH public key file #{private_key_file} doesn't exist.\n"\
+            "You can generate one with `ssh-keygen -t rsa -b 4096 -f #{private_key_file} -N ''`"
+        end
+
+        unless File.exist?(public_key_file)
+          raise "SSH public key file #{public_key_file} doesn't exist"
+        end
+
+        validate_ssh_log_level!
       end
 
       def validate_ssh_log_level!
@@ -83,6 +93,17 @@ module Proxy::RemoteExecution
         Plugin.settings.ssh_log_level = Plugin.settings.ssh_log_level.to_sym
       end
 
+      def requires_configured_ssh?
+        %i[ssh ssh-async].include?(Plugin.settings.mode) || Plugin.settings.cockpit_integration
+      end
+
+      def validate_socket_path!
+        return unless Plugin.settings.mode == :'ssh' || Plugin.settings.mode == :'ssh-async'
+
+        socket_path = File.expand_path(Plugin.settings.socket_working_dir)
+        raise "Socket path #{socket_path} is too long" if socket_path.length > Plugin::SOCKET_PATH_MAX_LENGTH
+      end
+
       def job_storage
         @job_storage ||= Proxy::RemoteExecution::Ssh::JobStorage.new
       end

data/settings.d/remote_execution_ssh.yml.example

@@ -3,8 +3,11 @@
 :ssh_identity_key_file: '~/.ssh/id_rsa_foreman_proxy'
 :local_working_dir: '/var/tmp'
 :remote_working_dir: '/var/tmp'
+:socket_working_dir: '/var/tmp'
 # :kerberos_auth: false
 
+# :cockpit_integration: true
+
 # Mode of operation, one of ssh, ssh-async, pull, pull-mqtt
 :mode: ssh
 
@@ -24,3 +27,8 @@
 # MQTT configuration, need to be set if mode is set to pull-mqtt
 # :mqtt_broker: localhost
 # :mqtt_port: 1883
+
+# Use of SSL can be forced either way by explicitly setting mqtt_tls setting. If
+# unset, SSL gets used if smart-proxy's foreman_ssl_cert, foreman_ssl_key and
+# foreman_ssl_ca settings are set available.
+# :mqtt_tls:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.8.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-04 00:00:00.000000000 Z
+date: 1980-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -170,10 +170,12 @@ files:
 - lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh
 - lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh
 - lib/smart_proxy_remote_execution_ssh/cockpit.rb
+- lib/smart_proxy_remote_execution_ssh/command_logging.rb
 - lib/smart_proxy_remote_execution_ssh/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
 - lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
 - lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb
 - lib/smart_proxy_remote_execution_ssh/runners.rb
@@ -203,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.26
 signing_key:
 specification_version: 4
 summary: Ssh remote execution provider for Foreman Smart-Proxy