smart_proxy_remote_execution_ssh 0.6.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3e83401d99929917142c15969fce19bf17e6fffc82000da554c457567016f8d
-  data.tar.gz: 9dbdec23d2203557204fb60f45cadc884d640d49b436eb291ad0cc4769182e2c
+  metadata.gz: f032dc665671e9aa070cc127201147fc2577d5252bf8b9afc6bd9e04c77e96e5
+  data.tar.gz: 9f8c886f1c8b18646e5ef0e85a967e398cf25fd13b96883eaa9e2275882531a2
 SHA512:
-  metadata.gz: 3cc070d29a185dc80cee33c3b69bf06772c226721ed9f42fba167a702a917ac04b50729ee3354d430b2a7c840581e98b75236cb4298e77aeee749fd9af6f56f2
-  data.tar.gz: a040bf6621ea2e35b421bc9098a5ec2569bc24e51ea6ae5346eec8c9e05d91e2560b96843d553589f8bbab10f009725947e76b306c5bb785c16ae0f3006baed8
+  metadata.gz: 6c8635ee16b4928f3e82bd5701a2492e434ea08b200719d5232874ee63b5b89f2d86a51913cd959eecba9c6ecc292be2070cd4473c8ced818aa7ede45e2d53eb
+  data.tar.gz: e6bbc045c00a4857aac5421953c6915bb0ba95238bad2a0ce0a2eed1269b538a704f474306107ca23dbf25b416ae19d98605fd4b93c7b4566941079e047cdc19

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -1,5 +1,6 @@
 require 'mqtt'
 require 'json'
+require 'time'
 
 module Proxy::RemoteExecution::Ssh::Actions
   class PullScript < Proxy::Dynflow::Action::Runner
@@ -25,7 +26,8 @@ module Proxy::RemoteExecution::Ssh::Actions
       otp_password = if input[:with_mqtt]
                        ::Proxy::Dynflow::OtpManager.generate_otp(execution_plan_id)
                      end
-      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script])
+
+      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''))
       output[:state] = :ready_for_pickup
       output[:result] = []
       mqtt_start(otp_password) if input[:with_mqtt]
@@ -40,9 +42,40 @@ module Proxy::RemoteExecution::Ssh::Actions
     def process_external_event(event)
       output[:state] = :running
       data = event.data
+      case data['version']
+      when nil
+        process_external_unversioned(data)
+      when '1'
+        process_external_v1(data)
+      else
+        raise "Unexpected update message version '#{data['version']}'"
+      end
+    end
+
+    def process_external_unversioned(payload)
       continuous_output = Proxy::Dynflow::ContinuousOutput.new
-      Array(data['output']).each { |line| continuous_output.add_output(line, 'stdout') } if data.key?('output')
-      exit_code = data['exit_code'].to_i if data['exit_code']
+      Array(payload['output']).each { |line| continuous_output.add_output(line, payload['type']) } if payload.key?('output')
+      exit_code = payload['exit_code'].to_i if payload['exit_code']
+      process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
+    end
+
+    def process_external_v1(payload)
+      continuous_output = Proxy::Dynflow::ContinuousOutput.new
+      exit_code = nil
+
+      payload['updates'].each do |update|
+        time = Time.parse update['timestamp']
+        type = update['type']
+        case type
+        when 'output'
+          continuous_output.add_output(update['content'], update['stream'], time)
+        when 'exit'
+          exit_code = update['exit_code'].to_i
+        else
+          raise "Unexpected update type '#{update['type']}'"
+        end
+      end
+
       process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
     end
 
@@ -96,9 +129,9 @@ module Proxy::RemoteExecution::Ssh::Actions
     def with_mqtt_client(&block)
       MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
                            :ssl => settings.mqtt_tls,
-                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert,
-                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key,
-                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca,
+                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
+                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
+                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
                            &block)
     end
 

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -180,10 +180,11 @@ module Proxy::RemoteExecution
 
       def inner_system_ssh_loop(out_buf, err_buf, in_buf, pid)
         err_buf_raw = ''
-        readers = [buf_socket, out_buf, err_buf]
         loop do
+          readers = [buf_socket, out_buf, err_buf].reject { |io| io.closed? }
+          writers = [buf_socket, in_buf].select { |io| io.pending_writes? }
           # Prime the sockets for reading
-          ready_readers, ready_writers = IO.select(readers, [buf_socket, in_buf], nil, 300)
+          ready_readers, ready_writers = IO.select(readers, writers)
           (ready_readers || []).each { |reader| reader.close if reader.fill.zero? }
 
           proxy_data(out_buf, in_buf)

data/lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb

@@ -173,10 +173,14 @@ module Proxy::RemoteExecution
         output.append(data)
       end
 
+      def pending_writes?
+        output.length.positive?
+      end
+
       # Sends as much of the pending output as possible. Returns +true+ if any
       # data was sent, and +false+ otherwise.
       def send_pending
-        if output.length.positive?
+        if pending_writes?
           sent = send(output.to_s, 0)
           output.consume!(sent)
           return sent.positive?
@@ -189,7 +193,7 @@ module Proxy::RemoteExecution
       # buffer is empty.
       def wait_for_pending_sends
         send_pending
-        while output.length.positive?
+        while pending_writes?
           result = IO.select(nil, [self]) || next
           next unless result[1].any?
 

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -1,7 +1,11 @@
 module Proxy::RemoteExecution::Ssh
   class Plugin < Proxy::Plugin
     SSH_LOG_LEVELS = %w[debug info error fatal].freeze
-    MODES = %i[ssh async-ssh pull pull-mqtt].freeze
+    MODES = %i[ssh ssh-async pull pull-mqtt].freeze
+    # Unix domain socket path length is limited to 104 (on some platforms) characters
+    # Socket path is composed of custom path (max 49 characters) + job id (37 characters)
+    # + offset(17 characters) + null terminator
+    SOCKET_PATH_MAX_LENGTH = 49
 
     http_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
@@ -11,6 +15,7 @@ module Proxy::RemoteExecution::Ssh
                      :ssh_user                => 'root',
                      :remote_working_dir      => '/var/tmp',
                      :local_working_dir       => '/var/tmp',
+                     :socket_working_dir      => '/var/tmp',
                      :kerberos_auth           => false,
                      :cockpit_integration     => true,
                      # When set to nil, makes REX use the runner's default interval

data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb

@@ -112,6 +112,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       @client_private_key_file = settings.ssh_identity_key_file
       @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
       @remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir.shellescape)
+      @socket_working_dir = options.fetch(:socket_working_dir, settings.socket_working_dir)
       @cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
       @first_execution = options.fetch(:first_execution, false)
       @user_method = user_method
@@ -159,15 +160,14 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def preflight_checks
       ensure_remote_command(cp_script_to_remote("#!/bin/sh\nexec true", 'test'),
-        publish: true,
         error: 'Failed to execute script on remote machine, exit code: %{exit_code}.'
       )
       unless @user_method.is_a? NoopUserMethod
         path = cp_script_to_remote("#!/bin/sh\nexec #{@user_method.cli_command_prefix} true", 'effective-user-test')
         ensure_remote_command(path,
                               error: 'Failed to change to effective user, exit code: %{exit_code}',
-                              publish: true,
                               tty: true,
+                              user_method: @user_method,
                               close_stdin: false)
       end
     end
@@ -178,7 +178,6 @@ module Proxy::RemoteExecution::Ssh::Runners
       # closed
       ensure_remote_command(
         'true',
-        publish: true,
         error: 'Failed to establish connection to remote host, exit code: %{exit_code}'
       )
     end
@@ -231,9 +230,9 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def close_session
-      raise 'Control socket file does not exist' unless File.exist?(local_command_file("socket"))
+      raise 'Control socket file does not exist' unless File.exist?(socket_file)
       @logger.debug("Sending exit request for session #{@ssh_user}@#{@host}")
-      args = ['/usr/bin/ssh', @host, "-o", "ControlPath=#{local_command_file("socket")}", "-O", "exit"].flatten
+      args = ['/usr/bin/ssh', @host, "-o", "ControlPath=#{socket_file}", "-O", "exit"].flatten
       pm = Proxy::Dynflow::ProcessManager.new(args)
       pm.on_stdout { |data| @logger.debug "[close_session]: #{data.chomp}"; data }
       pm.on_stderr { |data| @logger.debug "[close_session]: #{data.chomp}"; data }
@@ -265,7 +264,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       @process_manager && @cleanup_working_dirs
     end
 
-    def ssh_options(with_pty = false)
+    def ssh_options(with_pty = false, quiet: false)
       ssh_options = []
       ssh_options << "-tt" if with_pty
       ssh_options << "-o User=#{@ssh_user}"
@@ -276,21 +275,22 @@ module Proxy::RemoteExecution::Ssh::Runners
       ssh_options << "-o PreferredAuthentications=#{available_authentication_methods.join(',')}"
       ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
       ssh_options << "-o NumberOfPasswordPrompts=1"
-      ssh_options << "-o LogLevel=#{settings[:ssh_log_level]}"
+      ssh_options << "-o LogLevel=#{quiet ? 'quiet' : settings[:ssh_log_level]}"
       ssh_options << "-o ControlMaster=auto"
-      ssh_options << "-o ControlPath=#{local_command_file("socket")}"
+      ssh_options << "-o ControlPath=#{socket_file}"
       ssh_options << "-o ControlPersist=yes"
+      ssh_options << "-o ProxyCommand=none"
     end
 
     def settings
       Proxy::RemoteExecution::Ssh::Plugin.settings
     end
 
-    def get_args(command, with_pty = false)
+    def get_args(command, with_pty = false, quiet: false)
       args = []
       args = [{'SSHPASS' => @key_passphrase}, '/usr/bin/sshpass', '-P', 'passphrase', '-e'] if @key_passphrase
       args = [{'SSHPASS' => @ssh_password}, '/usr/bin/sshpass', '-e'] if @ssh_password
-      args += ['/usr/bin/ssh', @host, ssh_options(with_pty), command].flatten
+      args += ['/usr/bin/ssh', @host, ssh_options(with_pty, quiet: quiet), command].flatten
     end
 
     # Initiates run of the remote command and yields the data when
@@ -300,7 +300,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       raise 'Async command already in progress' if @process_manager&.started?
 
       @user_method.reset
-      initialize_command(*get_args(command, true))
+      initialize_command(*get_args(command, true, quiet: true))
 
       true
     end
@@ -309,12 +309,17 @@ module Proxy::RemoteExecution::Ssh::Runners
       @process_manager&.started? && @user_method.sent_all_data?
     end
 
-    def run_sync(command, stdin: nil, publish: false, close_stdin: true, tty: false)
+    def run_sync(command, stdin: nil, close_stdin: true, tty: false, user_method: nil)
       pm = Proxy::Dynflow::ProcessManager.new(get_args(command, tty))
-      if publish
-        pm.on_stdout { |data| publish_data(data, 'stdout', pm); '' }
-        pm.on_stderr { |data| publish_data(data, 'stderr', pm); '' }
+      callback = proc do |data|
+        data.each_line do |line|
+          logger.debug(line.chomp) if user_method.nil? || !user_method.filter_password?(line)
+          user_method.on_data(data, pm.stdin) if user_method
+        end
+        ''
       end
+      pm.on_stdout(&callback)
+      pm.on_stderr(&callback)
       pm.start!
       unless pm.status
         pm.stdin.io.puts(stdin) if stdin
@@ -340,6 +345,10 @@ module Proxy::RemoteExecution::Ssh::Runners
       File.join(ensure_local_directory(local_command_dir), filename)
     end
 
+    def socket_file
+      File.join(ensure_local_directory(@socket_working_dir), @id)
+    end
+
     def remote_command_dir
       File.join(@remote_working_dir, "foreman-ssh-cmd-#{id}")
     end
@@ -372,7 +381,6 @@ module Proxy::RemoteExecution::Ssh::Runners
 
       @logger.debug("Sending data to #{path} on remote host:\n#{data}")
       ensure_remote_command(command,
-        publish: true,
         stdin: data,
         error: "Unable to upload file to #{path} on remote system, exit code: %{exit_code}"
       )
@@ -388,7 +396,6 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def ensure_remote_directory(path)
       ensure_remote_command("mkdir -p #{path}",
-        publish: true,
         error: "Unable to create directory #{path} on remote system, exit code: %{exit_code}"
       )
     end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.6.0'
+      VERSION = '0.7.2'
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -10,6 +10,7 @@ module Proxy::RemoteExecution
         validate_mode!
         validate_ssh_settings!
         validate_mqtt_settings!
+        validate_socket_path!
       end
 
       def private_key_file
@@ -49,7 +50,7 @@ module Proxy::RemoteExecution
         raise 'mqtt_port has to be set when pull-mqtt mode is used' if Plugin.settings.mqtt_port.nil?
 
         if Plugin.settings.mqtt_tls.nil?
-          Plugin.settings.mqtt_tls = [:foreman_ssl_cert, :foreman_ssl_key, :foreman_ssl_ca].all? { |key| ::Proxy::SETTINGS[key] }
+          Plugin.settings.mqtt_tls = [[:foreman_ssl_cert, :ssl_certificate], [:foreman_ssl_key, :ssl_private_key], [:foreman_ssl_ca, :ssl_ca_file]].all? { |(client, server)| ::Proxy::SETTINGS[client] || ::Proxy::SETTINGS[server] }
         end
       end
 
@@ -96,6 +97,13 @@ module Proxy::RemoteExecution
         %i[ssh ssh-async].include?(Plugin.settings.mode) || Plugin.settings.cockpit_integration
       end
 
+      def validate_socket_path!
+        return unless Plugin.settings.mode == :'ssh' || Plugin.settings.mode == :'ssh-async'
+
+        socket_path = File.expand_path(Plugin.settings.socket_working_dir)
+        raise "Socket path #{socket_path} is too long" if socket_path.length > Plugin::SOCKET_PATH_MAX_LENGTH
+      end
+
       def job_storage
         @job_storage ||= Proxy::RemoteExecution::Ssh::JobStorage.new
       end

data/settings.d/remote_execution_ssh.yml.example

@@ -3,6 +3,7 @@
 :ssh_identity_key_file: '~/.ssh/id_rsa_foreman_proxy'
 :local_working_dir: '/var/tmp'
 :remote_working_dir: '/var/tmp'
+:socket_working_dir: '/var/tmp'
 # :kerberos_auth: false
 
 # :cockpit_integration: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-19 00:00:00.000000000 Z
+date: 2022-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -203,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.20
 signing_key:
 specification_version: 4
 summary: Ssh remote execution provider for Foreman Smart-Proxy