smart_proxy_remote_execution_ssh 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e58a285a16b7d9a173ee4c29639b3367517c5d84f7979bdc84ab2c502a51edb1
-  data.tar.gz: 57892ef63dc0a6bd156ae6fef562b8a4096a7a3cd6a2a03828b4598b1b1d37c7
+  metadata.gz: 1a599d3732d4f6b064a850ae8baced28354980096785a340828533f953f844c4
+  data.tar.gz: 160b80ab983f0eb7b987987191b48c19db42d3b76079b0ba6639ec831dccc6f3
 SHA512:
-  metadata.gz: 89cc41ef35aa11210f322afc3b7ae35aa269cc4ed8fcef8faff11e32ac37fe765cc083d0bdef2b9c58a54f96be5c8224c5916b85cbb393040f6e715816bf51b7
-  data.tar.gz: ec608d3b666443e6afbb97d28f1014f459d746d65adcf0e1f0cd43d168961e05884f8d3e393814f594527a294d6ac56d2af432509b8ecb99cfcf0df7101f501b
+  metadata.gz: 349961aa474809225d9781bda09687eb23f00db51cca9f758a57e075fa3d19fc07b729569b064887f91bb7f07190152d6ec9cb01e8ccf48df0841e9c3f85d10e
+  data.tar.gz: 8ad40859c92f7d1cd089adf0f37bb0aaa8add745cd461c2e1ed5b484d3ae6b856ac0ee53e61c62e89d564a4ba676b22bf5f01618da970986cd60e42591ff6346

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -1,5 +1,6 @@
 require 'smart_proxy_remote_execution_ssh/net_ssh_compat'
 require 'forwardable'
+require 'securerandom'
 
 module Proxy::RemoteExecution
   module Cockpit
@@ -164,9 +165,8 @@ module Proxy::RemoteExecution
         out_read, out_write = IO.pipe
         err_read, err_write = IO.pipe
 
-        # Force the script runner to initialize its logger
-        script_runner.logger
-        pid = spawn(*script_runner.send(:get_args, command), :in => in_read, :out => out_write, :err => err_write)
+        connection.establish!
+        pid = spawn(*connection.command(command), :in => in_read, :out => out_write, :err => err_write)
         [in_read, out_write, err_write].each(&:close)
 
         send_start
@@ -176,19 +176,22 @@ module Proxy::RemoteExecution
         in_buf  = MiniSSLBufferedSocket.new(in_write)
 
         inner_system_ssh_loop out_buf, err_buf, in_buf, pid
+      ensure
+        connection.disconnect!
       end
 
       def inner_system_ssh_loop(out_buf, err_buf, in_buf, pid)
         err_buf_raw = ''
-        readers = [buf_socket, out_buf, err_buf]
         loop do
+          readers = [buf_socket, out_buf, err_buf].reject { |io| io.closed? }
+          writers = [buf_socket, in_buf].select { |io| io.pending_writes? }
           # Prime the sockets for reading
-          ready_readers, ready_writers = IO.select(readers, [buf_socket, in_buf], nil, 300)
+          ready_readers, ready_writers = IO.select(readers, writers)
           (ready_readers || []).each { |reader| reader.close if reader.fill.zero? }
 
           proxy_data(out_buf, in_buf)
           if buf_socket.closed?
-            script_runner.close_session
+            connection.disconnect!
           end
 
           if out_buf.closed?
@@ -262,10 +265,10 @@ module Proxy::RemoteExecution
         params["hostname"]
       end
 
-      def script_runner
-        @script_runner ||= Proxy::RemoteExecution::Ssh::Runners::ScriptRunner.build(
+      def connection
+        @connection ||= Proxy::RemoteExecution::Ssh::Runners::MultiplexedSSHConnection.new(
           runner_params,
-          suspended_action: nil
+          logger: logger
         )
       end
 
@@ -278,6 +281,7 @@ module Proxy::RemoteExecution
         # For compatibility only
         ret[:script] = nil
         ret[:hostname] = host
+        ret[:id] = SecureRandom.uuid
         ret
       end
     end

data/lib/smart_proxy_remote_execution_ssh/command_logging.rb

@@ -0,0 +1,23 @@
+# lib/command_logging.rb
+
+module Proxy::RemoteExecution::Ssh::Runners
+  module CommandLogging
+    def log_command(command, label: "Running")
+      command = command.join(' ')
+      label = "#{label}: " if label
+      logger.debug("#{label}#{command}")
+    end
+
+    def set_pm_debug_logging(pm, capture: false, user_method: nil)
+      callback = proc do |data|
+        data.each_line do |line|
+          logger.debug(line.chomp) if user_method.nil? || !user_method.filter_password?(line)
+          user_method.on_data(data, pm.stdin) if user_method
+        end
+        ''
+      end
+      pm.on_stdout(&callback)
+      pm.on_stderr(&callback)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb

@@ -0,0 +1,196 @@
+require 'smart_proxy_remote_execution_ssh/command_logging'
+
+module Proxy::RemoteExecution::Ssh::Runners
+  class SensitiveString
+    def initialize(value, mask: '*****')
+      @value = value
+      @mask = mask
+    end
+
+    def inspect
+      '"' + to_s + '"'
+    end
+
+    def to_s
+      @mask
+    end
+
+    def to_str
+      @value
+    end
+  end
+
+  class AuthenticationMethod
+    attr_reader :name
+    def initialize(name, prompt: nil, password: nil)
+      @name = name
+      @prompt = prompt
+      @password = password
+    end
+
+    def ssh_command_prefix
+      return [] unless @password
+
+      prompt = ['-P', @prompt] if @prompt
+      [{'SSHPASS' => SensitiveString.new(@password)}, '/usr/bin/sshpass', '-e', prompt].compact
+    end
+
+    def ssh_options
+      ["-o PreferredAuthentications=#{name}",
+       "-o NumberOfPasswordPrompts=#{@password ? 1 : 0}"]
+    end
+  end
+
+  class MultiplexedSSHConnection
+    include CommandLogging
+
+    attr_reader :logger
+    def initialize(options, logger:)
+      @logger = logger
+
+      @id = options.fetch(:id)
+      @host = options.fetch(:hostname)
+      @script = options.fetch(:script)
+      @ssh_user = options.fetch(:ssh_user, 'root')
+      @ssh_port = options.fetch(:ssh_port, 22)
+      @ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
+      @key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
+      @host_public_key = options.fetch(:host_public_key, nil)
+      @verify_host = options.fetch(:verify_host, nil)
+      @client_private_key_file = settings.ssh_identity_key_file
+
+      @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
+      @socket_working_dir = options.fetch(:socket_working_dir, settings.socket_working_dir)
+      @socket = nil
+    end
+
+    def establish!
+      @available_auth_methods ||= available_authentication_methods
+      method = @available_auth_methods.find do |method|
+        if try_auth_method(method)
+          @available_auth_methods.unshift(method).uniq!
+          true
+        end
+      end
+      method || raise("Could not establish connection to remote host using any available authentication method, tried #{@available_auth_methods.map(&:name).join(', ')}")
+    end
+
+    def disconnect!
+      return unless connected?
+
+      cmd = command(%w[-O exit])
+      log_command(cmd, label: "Closing shared connection")
+      pm = Proxy::Dynflow::ProcessManager.new(cmd)
+      set_pm_debug_logging(pm)
+      pm.run!
+      @socket = nil
+    end
+
+    def connected?
+      !@socket.nil?
+    end
+
+    def command(cmd)
+      raise "Cannot build command to run over multiplexed connection without having an established connection" unless connected?
+
+      ['/usr/bin/ssh', reuse_ssh_options, cmd].flatten
+    end
+
+    private
+
+    def try_auth_method(method)
+      # running "ssh -f -N" instead of "ssh true" would be cleaner, but ssh
+      # does not close its stderr which trips up the process manager which
+      # expects all FDs to be closed
+
+      full_command = [method.ssh_command_prefix, '/usr/bin/ssh', establish_ssh_options, method.ssh_options, @host, 'true'].flatten
+      log_command(full_command)
+      pm = Proxy::Dynflow::ProcessManager.new(full_command)
+      pm.start!
+      if pm.status
+        raise pm.stderr.to_s
+      else
+        set_pm_debug_logging(pm)
+        pm.stdin.io.close
+        pm.run!
+      end
+
+      if pm.status.zero?
+        logger.debug("Established connection using authentication method #{method.name}")
+        @socket = socket_file
+        true
+      else
+        logger.debug("Failed to establish connection using authentication method #{method.name}")
+        false
+      end
+    end
+
+    def settings
+      Proxy::RemoteExecution::Ssh::Plugin.settings
+    end
+
+    def available_authentication_methods
+      methods = []
+      methods << AuthenticationMethod.new('password', password: @ssh_password) if @ssh_password
+      if verify_key_passphrase
+        methods << AuthenticationMethod.new('publickey', password: @key_passphrase, prompt: 'passphrase')
+      end
+      methods << AuthenticationMethod.new('gssapi-with-mic') if settings[:kerberos_auth]
+      raise "There are no available authentication methods" if methods.empty?
+      methods
+    end
+
+    def establish_ssh_options
+      return @establish_ssh_options if @establish_ssh_options
+      ssh_options = []
+      ssh_options << "-o User=#{@ssh_user}"
+      ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
+      ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
+      ssh_options << "-o IdentitiesOnly=yes"
+      ssh_options << "-o StrictHostKeyChecking=no"
+      ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
+      ssh_options << "-o LogLevel=#{ssh_log_level(true)}"
+      ssh_options << "-o ControlMaster=auto"
+      ssh_options << "-o ControlPath=#{socket_file}"
+      ssh_options << "-o ControlPersist=yes"
+      ssh_options << "-o ProxyCommand=none"
+      @establish_ssh_options = ssh_options
+    end
+
+    def reuse_ssh_options
+      ["-o", "ControlPath=#{@socket}", "-o", "LogLevel=#{ssh_log_level(false)}", @host]
+    end
+
+    def socket_file
+      File.join(@socket_working_dir, @id)
+    end
+
+    def verify_key_passphrase
+      command = ['/usr/bin/ssh-keygen', '-y', '-f', File.expand_path(@client_private_key_file)]
+      log_command(command, label: "Checking if private key has passphrase")
+      pm = Proxy::Dynflow::ProcessManager.new(command)
+      pm.start!
+
+      raise pm.stderr.to_s if pm.status
+
+      pm.stdin.io.close
+      pm.run!
+
+      if pm.status.zero?
+        logger.debug("Private key is not protected with a passphrase")
+        @key_passphrase = nil
+      else
+        logger.debug("Private key is protected with a passphrase")
+      end
+
+      return true if pm.status.zero? || @key_passphrase
+
+      logger.debug("Private key is protected with a passphrase, but no passphrase was provided")
+      false
+    end
+
+    def ssh_log_level(new_connection)
+      new_connection ? settings[:ssh_log_level] : 'quiet'
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb

@@ -173,10 +173,14 @@ module Proxy::RemoteExecution
         output.append(data)
       end
 
+      def pending_writes?
+        output.length.positive?
+      end
+
       # Sends as much of the pending output as possible. Returns +true+ if any
       # data was sent, and +false+ otherwise.
       def send_pending
-        if output.length.positive?
+        if pending_writes?
           sent = send(output.to_s, 0)
           output.consume!(sent)
           return sent.positive?
@@ -189,7 +193,7 @@ module Proxy::RemoteExecution
       # buffer is empty.
       def wait_for_pending_sends
         send_pending
-        while output.length.positive?
+        while pending_writes?
           result = IO.select(nil, [self]) || next
           next unless result[1].any?
 

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -1,7 +1,7 @@
 module Proxy::RemoteExecution::Ssh
   class Plugin < Proxy::Plugin
     SSH_LOG_LEVELS = %w[debug info error fatal].freeze
-    MODES = %i[ssh async-ssh pull pull-mqtt].freeze
+    MODES = %i[ssh ssh-async pull pull-mqtt].freeze
     # Unix domain socket path length is limited to 104 (on some platforms) characters
     # Socket path is composed of custom path (max 49 characters) + job id (37 characters)
     # + offset(17 characters) + null terminator

data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb

@@ -50,13 +50,13 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def refresh
+      @connection.establish! unless @connection.connected?
       begin
         pm = run_sync("#{@user_method.cli_command_prefix} #{@retrieval_script}")
+        process_retrieved_data(pm.stdout.to_s.chomp, pm.stderr.to_s.chomp)
       rescue StandardError => e
         @logger.info("Error while connecting to the remote host on refresh: #{e.message}")
       end
-
-      process_retrieved_data(pm.stdout.to_s.chomp, pm.stderr.to_s.chomp)
     ensure
       destroy_session
     end
@@ -139,7 +139,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def destroy_session
-      if @session
+      if @connection.connected?
         @logger.debug("Closing session with #{@ssh_user}@#{@host}")
         close_session
       end

data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb

@@ -1,6 +1,7 @@
 require 'fileutils'
 require 'smart_proxy_dynflow/runner/process_manager_command'
 require 'smart_proxy_dynflow/process_manager'
+require 'smart_proxy_remote_execution_ssh/multiplexed_ssh_connection'
 
 module Proxy::RemoteExecution::Ssh::Runners
   class EffectiveUserMethod
@@ -92,6 +93,8 @@ module Proxy::RemoteExecution::Ssh::Runners
   # rubocop:disable Metrics/ClassLength
   class ScriptRunner < Proxy::Dynflow::Runner::Base
     include Proxy::Dynflow::Runner::ProcessManagerCommand
+    include CommandLogging
+
     attr_reader :execution_timeout_interval
 
     EXPECTED_POWER_ACTION_MESSAGES = ['restart host', 'shutdown host'].freeze
@@ -103,10 +106,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       @script = options.fetch(:script)
       @ssh_user = options.fetch(:ssh_user, 'root')
       @ssh_port = options.fetch(:ssh_port, 22)
-      @ssh_password = options.fetch(:secrets, {}).fetch(:ssh_password, nil)
-      @key_passphrase = options.fetch(:secrets, {}).fetch(:key_passphrase, nil)
       @host_public_key = options.fetch(:host_public_key, nil)
-      @verify_host = options.fetch(:verify_host, nil)
       @execution_timeout_interval = options.fetch(:execution_timeout_interval, nil)
 
       @client_private_key_file = settings.ssh_identity_key_file
@@ -116,6 +116,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       @cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
       @first_execution = options.fetch(:first_execution, false)
       @user_method = user_method
+      @options = options
     end
 
     def self.build(options, suspended_action:)
@@ -143,7 +144,9 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def start
       Proxy::RemoteExecution::Utils.prune_known_hosts!(@host, @ssh_port, logger) if @first_execution
-      establish_connection
+      ensure_local_directory(@socket_working_dir)
+      @connection = MultiplexedSSHConnection.new(@options.merge(:id => @id), logger: logger)
+      @connection.establish!
       preflight_checks
       prepare_start
       script = initialization_script
@@ -172,16 +175,6 @@ module Proxy::RemoteExecution::Ssh::Runners
       end
     end
 
-    def establish_connection
-      # run_sync ['-f', '-N'] would be cleaner, but ssh does not close its
-      # stderr which trips up the process manager which expects all FDs to be
-      # closed
-      ensure_remote_command(
-        'true',
-        error: 'Failed to establish connection to remote host, exit code: %{exit_code}'
-      )
-    end
-
     def prepare_start
       @remote_script = cp_script_to_remote
       @output_path = File.join(File.dirname(@remote_script), 'output')
@@ -232,11 +225,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     def close_session
       raise 'Control socket file does not exist' unless File.exist?(socket_file)
       @logger.debug("Sending exit request for session #{@ssh_user}@#{@host}")
-      args = ['/usr/bin/ssh', @host, "-o", "ControlPath=#{socket_file}", "-O", "exit"].flatten
-      pm = Proxy::Dynflow::ProcessManager.new(args)
-      pm.on_stdout { |data| @logger.debug "[close_session]: #{data.chomp}"; data }
-      pm.on_stderr { |data| @logger.debug "[close_session]: #{data.chomp}"; data }
-      pm.run!
+      @connection.disconnect!
     end
 
     def close
@@ -264,34 +253,10 @@ module Proxy::RemoteExecution::Ssh::Runners
       @process_manager && @cleanup_working_dirs
     end
 
-    def ssh_options(with_pty = false, quiet: false)
-      ssh_options = []
-      ssh_options << "-tt" if with_pty
-      ssh_options << "-o User=#{@ssh_user}"
-      ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
-      ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
-      ssh_options << "-o IdentitiesOnly=yes"
-      ssh_options << "-o StrictHostKeyChecking=no"
-      ssh_options << "-o PreferredAuthentications=#{available_authentication_methods.join(',')}"
-      ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
-      ssh_options << "-o NumberOfPasswordPrompts=1"
-      ssh_options << "-o LogLevel=#{quiet ? 'quiet' : settings[:ssh_log_level]}"
-      ssh_options << "-o ControlMaster=auto"
-      ssh_options << "-o ControlPath=#{socket_file}"
-      ssh_options << "-o ControlPersist=yes"
-    end
-
     def settings
       Proxy::RemoteExecution::Ssh::Plugin.settings
     end
 
-    def get_args(command, with_pty = false, quiet: false)
-      args = []
-      args = [{'SSHPASS' => @key_passphrase}, '/usr/bin/sshpass', '-P', 'passphrase', '-e'] if @key_passphrase
-      args = [{'SSHPASS' => @ssh_password}, '/usr/bin/sshpass', '-e'] if @ssh_password
-      args += ['/usr/bin/ssh', @host, ssh_options(with_pty, quiet: quiet), command].flatten
-    end
-
     # Initiates run of the remote command and yields the data when
     # available. The yielding doesn't happen automatically, but as
     # part of calling the `refresh` method.
@@ -299,7 +264,9 @@ module Proxy::RemoteExecution::Ssh::Runners
       raise 'Async command already in progress' if @process_manager&.started?
 
       @user_method.reset
-      initialize_command(*get_args(command, true, quiet: true))
+      cmd = @connection.command([tty_flag(true), command].flatten.compact)
+      log_command(cmd)
+      initialize_command(*cmd)
 
       true
     end
@@ -308,17 +275,15 @@ module Proxy::RemoteExecution::Ssh::Runners
       @process_manager&.started? && @user_method.sent_all_data?
     end
 
+    def tty_flag(tty)
+      '-tt' if tty
+    end
+
     def run_sync(command, stdin: nil, close_stdin: true, tty: false, user_method: nil)
-      pm = Proxy::Dynflow::ProcessManager.new(get_args(command, tty))
-      callback = proc do |data|
-        data.each_line do |line|
-          logger.debug(line.chomp) if user_method.nil? || !user_method.filter_password?(line)
-          user_method.on_data(data, pm.stdin) if user_method
-        end
-        ''
-      end
-      pm.on_stdout(&callback)
-      pm.on_stderr(&callback)
+      cmd = @connection.command([tty_flag(tty), command].flatten.compact)
+      log_command(cmd)
+      pm = Proxy::Dynflow::ProcessManager.new(cmd)
+      set_pm_debug_logging(pm, user_method: user_method)
       pm.start!
       unless pm.status
         pm.stdin.io.puts(stdin) if stdin
@@ -428,13 +393,6 @@ module Proxy::RemoteExecution::Ssh::Runners
         @expecting_disconnect = true
       end
     end
-
-    def available_authentication_methods
-      methods = %w[publickey] # Always use pubkey auth as fallback
-      methods << 'gssapi-with-mic' if settings[:kerberos_auth]
-      methods.unshift('password') if @ssh_password
-      methods
-    end
   end
   # rubocop:enable Metrics/ClassLength
 end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.7.0'
+      VERSION = '0.8.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-13 00:00:00.000000000 Z
+date: 1980-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -170,10 +170,12 @@ files:
 - lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh
 - lib/smart_proxy_remote_execution_ssh/async_scripts/retrieve.sh
 - lib/smart_proxy_remote_execution_ssh/cockpit.rb
+- lib/smart_proxy_remote_execution_ssh/command_logging.rb
 - lib/smart_proxy_remote_execution_ssh/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
 - lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
 - lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb
 - lib/smart_proxy_remote_execution_ssh/runners.rb
@@ -203,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.26
 signing_key:
 specification_version: 4
 summary: Ssh remote execution provider for Foreman Smart-Proxy