smart_proxy_remote_execution_ssh 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f90b757baa9c530c89d5c6c960ebf69601b173886c2f74560513592ab78697e4
-  data.tar.gz: edf9bc88acd2f859f86dd15ddee26deb948f6b963ddcd6aee2c6d839e12198ed
+  metadata.gz: e2f91ec259ca105bcc18a5cecf4342019f4023d44cbff5695998b5e57c25e325
+  data.tar.gz: 51ccf299324d4adf10c07475dca2749d885c258913c8d9e7593d8b8149e6d18d
 SHA512:
-  metadata.gz: 56cb17a0d7ac5eb456533e72e83aaffb07892a725609685f84c95571afa4fb0868dbd4017c4cf5ceadafa35aaf01beca27bdadb424b5bf0c0a5280c704f21e2c
-  data.tar.gz: 04c1feec777172574e45369f48219a6d2ea80e7e704a695198ef56bb11b5d4c178c68650414a190f4e57751521a065ad5c0c649ad8842f99be03259f79f3c9f3
+  metadata.gz: 74313e91e51ff5d58fce0bc880131b12984a750110a2d76c88875a6b824eec40a2ba9d7336dfda34df7c11d465ded62c14d4d33ba10cf4bed1b2e0309aff9a2b
+  data.tar.gz: 92f85e971f10d9e45418e75d0325f3e548e65954160be9f86cb154adde6bc3a512b4e3a5429d162291704b49a50a5570fcdeb4b129f735b8f8f737938a0194d0

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -0,0 +1,110 @@
+require 'mqtt'
+require 'json'
+
+module Proxy::RemoteExecution::Ssh::Actions
+  class PullScript < Proxy::Dynflow::Action::Runner
+    JobDelivered = Class.new
+
+    execution_plan_hooks.use :cleanup, :on => :stopped
+
+    def plan(action_input, mqtt: false)
+      super(action_input)
+      input[:with_mqtt] = mqtt
+    end
+
+    def run(event = nil)
+      if event == JobDelivered
+        output[:state] = :delivered
+        suspend
+      else
+        super
+      end
+    end
+
+    def init_run
+      otp_password = if input[:with_mqtt]
+                       ::Proxy::Dynflow::OtpManager.generate_otp(execution_plan_id)
+                     end
+      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script])
+      output[:state] = :ready_for_pickup
+      output[:result] = []
+      mqtt_start(otp_password) if input[:with_mqtt]
+      suspend
+    end
+
+    def cleanup(_plan = nil)
+      job_storage.drop_job(execution_plan_id, run_step_id)
+      Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
+    end
+
+    def process_external_event(event)
+      output[:state] = :running
+      data = event.data
+      continuous_output = Proxy::Dynflow::ContinuousOutput.new
+      Array(data['output']).each { |line| continuous_output.add_output(line, 'stdout') } if data.key?('output')
+      exit_code = data['exit_code'].to_i if data['exit_code']
+      process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
+    end
+
+    def kill_run
+      case output[:state]
+      when :ready_for_pickup
+        # If the job is not running yet on the client, wipe it from storage
+        cleanup
+        # TODO: Stop the action
+      when :notified, :running
+        # Client was notified or is already running, dealing with this situation
+        # is only supported if mqtt is available
+        # Otherwise we have to wait it out
+        # TODO
+        # if input[:with_mqtt]
+      end
+      suspend
+    end
+
+    def mqtt_start(otp_password)
+      payload = {
+        type: 'data',
+        message_id: SecureRandom.uuid,
+        version: 1,
+        sent: DateTime.now.iso8601,
+        directive: 'foreman',
+        metadata: {
+          'job_uuid': input[:job_uuid],
+          'username': execution_plan_id,
+          'password': otp_password,
+          'return_url': "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/update",
+        },
+        content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}",
+      }
+      mqtt_notify payload
+      output[:state] = :notified
+    end
+
+    def mqtt_notify(payload)
+      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port) do |c|
+        c.publish(mqtt_topic, JSON.dump(payload), false, 1)
+      end
+    end
+
+    def host_name
+      alternative_names = input.fetch(:alternative_names, {})
+
+      alternative_names[:consumer_uuid] ||
+        alternative_names[:fqdn] ||
+        input[:hostname]
+    end
+
+    def mqtt_topic
+      "yggdrasil/#{host_name}/data/in"
+    end
+
+    def settings
+      Proxy::RemoteExecution::Ssh::Plugin.settings
+    end
+
+    def job_storage
+      Proxy::RemoteExecution::Ssh.job_storage
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/actions/run_script.rb

@@ -3,7 +3,20 @@ require 'smart_proxy_dynflow/action/runner'
 
 module Proxy::RemoteExecution::Ssh
   module Actions
-    class RunScript < Proxy::Dynflow::Action::Runner
+    class RunScript < ::Dynflow::Action
+      def plan(*args)
+        mode = Proxy::RemoteExecution::Ssh::Plugin.settings.mode
+        case mode
+        when :ssh, :'ssh-async'
+          plan_action(ScriptRunner, *args)
+        when :pull, :'pull-mqtt'
+          plan_action(PullScript, *args,
+                      mqtt: mode == :'pull-mqtt')
+        end
+      end
+    end
+
+    class ScriptRunner < Proxy::Dynflow::Action::Runner
       def initiate_runner
         additional_options = {
           :step_id => run_step_id,

data/lib/smart_proxy_remote_execution_ssh/actions.rb

@@ -0,0 +1,6 @@
+module Proxy::RemoteExecution::Ssh
+  module Actions
+    require 'smart_proxy_remote_execution_ssh/actions/run_script'
+    require 'smart_proxy_remote_execution_ssh/actions/pull_script'
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -1,11 +1,13 @@
 require 'net/ssh'
 require 'base64'
+require 'smart_proxy_dynflow/runner'
 
 module Proxy::RemoteExecution
   module Ssh
 
     class Api < ::Sinatra::Base
       include Sinatra::Authorization::Helpers
+      include Proxy::Dynflow::Helpers
 
       get "/pubkey" do
         File.read(Ssh.public_key_file)
@@ -37,6 +39,53 @@ module Proxy::RemoteExecution
           end
         204
       end
+
+      # Payload is a hash where
+      # exit_code: Integer | NilClass
+      # output: String
+      post '/jobs/:job_uuid/update' do |job_uuid|
+        do_authorize_with_ssl_client
+
+        with_authorized_job(job_uuid) do |job_record|
+          data = MultiJson.load(request.body.read)
+          notify_job(job_record, ::Proxy::Dynflow::Runner::ExternalEvent.new(data))
+        end
+      end
+
+      get '/jobs' do
+        do_authorize_with_ssl_client
+
+        MultiJson.dump(Proxy::RemoteExecution::Ssh.job_storage.job_uuids_for_host(https_cert_cn))
+      end
+
+      get "/jobs/:job_uuid" do |job_uuid|
+        do_authorize_with_ssl_client
+
+        with_authorized_job(job_uuid) do |job_record|
+          notify_job(job_record, Actions::PullScript::JobDelivered)
+          job_record[:job]
+        end
+      end
+
+      private
+
+      def notify_job(job_record, event)
+        world.event(job_record[:execution_plan_uuid], job_record[:run_step_id], event)
+      end
+
+      def with_authorized_job(uuid)
+        if (job = authorized_job(uuid))
+          yield job
+        else
+          halt 404
+        end
+      end
+
+      def authorized_job(uuid)
+        job_record = Proxy::RemoteExecution::Ssh.job_storage.find_job(uuid) || {}
+        return job_record if authorize_with_token(clear: false, task_id: job_record[:execution_plan_uuid]) ||
+                             job_record[:hostname] == https_cert_cn
+      end
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/cockpit.rb

@@ -1,11 +1,11 @@
-require 'net/ssh'
+require 'smart_proxy_remote_execution_ssh/net_ssh_compat'
 require 'forwardable'
 
 module Proxy::RemoteExecution
   module Cockpit
     # A wrapper class around different kind of sockets to comply with Net::SSH event loop
     class BufferedSocket
-      include Net::SSH::BufferedIo
+      include Proxy::RemoteExecution::NetSSHCompat::BufferedIO
       extend Forwardable
 
       # The list of methods taken from OpenSSL::SSL::SocketForwarder for the object to act like a socket
@@ -52,14 +52,14 @@ module Proxy::RemoteExecution
       end
       def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
 
-      def recv(n)
+      def recv(count)
         res = ""
         begin
           # To drain a SSLSocket before we can go back to the event
           # loop, we need to repeatedly call read_nonblock; a single
           # call is not enough.
           loop do
-            res += @socket.read_nonblock(n)
+            res += @socket.read_nonblock(count)
           end
         rescue IO::WaitReadable
           # Sometimes there is no payload after reading everything
@@ -95,8 +95,8 @@ module Proxy::RemoteExecution
       end
       def_delegators(:@socket, :read_nonblock, :write_nonblock, :close)
 
-      def recv(n)
-        @socket.read_nonblock(n)
+      def recv(count)
+        @socket.read_nonblock(count)
       end
 
       def send(mesg, flags)
@@ -113,6 +113,7 @@ module Proxy::RemoteExecution
 
       def initialize(env)
         @env = env
+        @open_ios = []
       end
 
       def valid?
@@ -127,6 +128,7 @@ module Proxy::RemoteExecution
           begin
             @env['rack.hijack'].call
           rescue NotImplementedError
+            # This is fine
           end
           @socket = @env['rack.hijack_io']
         end
@@ -137,15 +139,11 @@ module Proxy::RemoteExecution
       private
 
       def ssh_on_socket
-        with_error_handling { start_ssh_loop }
+        with_error_handling { system_ssh_loop }
       end
 
       def with_error_handling
         yield
-      rescue Net::SSH::AuthenticationFailed => e
-        send_error(401, e.message)
-      rescue Errno::EHOSTUNREACH
-        send_error(400, "No route to #{host}")
       rescue SystemCallError => e
         send_error(400, e.message)
       rescue SocketError => e
@@ -161,50 +159,62 @@ module Proxy::RemoteExecution
         end
       end
 
-      def start_ssh_loop
-        err_buf = ""
-
-        Net::SSH.start(host, ssh_user, ssh_options) do |ssh|
-          channel = ssh.open_channel do |ch|
-            ch.exec(command) do |ch, success|
-              raise "could not execute command" unless success
-
-              ssh.listen_to(buf_socket)
-
-              ch.on_process do
-                if buf_socket.available.positive?
-                  ch.send_data(buf_socket.read_available)
-                end
-                if buf_socket.closed?
-                  ch.close
-                end
-              end
-
-              ch.on_data do |ch2, data|
-                send_start
-                buf_socket.enqueue(data)
-              end
-
-              ch.on_request('exit-status') do |ch, data|
-                code = data.read_long
-                send_start if code.zero?
-                err_buf += "Process exited with code #{code}.\r\n"
-                ch.close
-              end
-
-              ch.on_request('exit-signal') do |ch, data|
-                err_buf += "Process was terminated with signal #{data.read_string}.\r\n"
-                ch.close
-              end
-
-              ch.on_extended_data do |ch2, type, data|
-                err_buf += data
-              end
-            end
+      def system_ssh_loop
+        in_read, in_write   = IO.pipe
+        out_read, out_write = IO.pipe
+        err_read, err_write = IO.pipe
+
+        pid = spawn(*script_runner.send(:get_args, command), :in => in_read, :out => out_write, :err => err_write)
+        [in_read, out_write, err_write].each(&:close)
+
+        send_start
+        # Not SSL buffer, but the interface kinda matches
+        out_buf = MiniSSLBufferedSocket.new(out_read)
+        err_buf = MiniSSLBufferedSocket.new(err_read)
+        in_buf  = MiniSSLBufferedSocket.new(in_write)
+
+        inner_system_ssh_loop out_buf, err_buf, in_buf, pid
+      end
+
+      def inner_system_ssh_loop2(out_buf, err_buf, in_buf, pid)
+        err_buf_raw = ''
+        readers = [buf_socket, out_buf, err_buf]
+        loop do
+          # Prime the sockets for reading
+          ready_readers, ready_writers = IO.select(readers, [buf_socket, in_buf], nil, 300)
+          (ready_readers || []).each { |reader| reader.close if reader.fill.zero? }
+
+          proxy_data(out_buf, in_buf)
+
+          if out_buf.closed?
+            code = Process.wait2(pid).last.exitstatus
+            send_start if code.zero? # TODO: Why?
+            err_buf_raw += "Process exited with code #{code}.\r\n"
+            break
           end
 
-          channel.wait
-          send_error(400, err_buf) unless @started
+          if err_buf.available.positive?
+            err_buf_raw += err_buf.read_available
+          end
+
+          flush_pending_writes(ready_writers || [])
+        end
+      rescue # rubocop:disable Style/RescueStandardError
+        send_error(400, err_buf_raw) unless @started
+      ensure
+        [out_buff, err_buf, in_buf].each(&:close)
+      end
+
+      def proxy_data(out_buf, in_buf)
+        { out_buf => buf_socket, buf_socket => in_buf }.each do |src, dst|
+          dst.enqueue(src.read_available) if src.available.positive?
+          dst.close if src.closed?
+        end
+      end
+
+      def flush_pending_writes(writers)
+        writers.each do |writer|
+          writer.respond_to?(:send_pending) ? writer.send_pending : writer.flush
         end
       end
 
@@ -215,6 +225,7 @@ module Proxy::RemoteExecution
           buf_socket.enqueue("Connection: upgrade\r\n")
           buf_socket.enqueue("Upgrade: raw\r\n")
           buf_socket.enqueue("\r\n")
+          buf_socket.send_pending
         end
       end
 
@@ -223,6 +234,7 @@ module Proxy::RemoteExecution
         buf_socket.enqueue("Connection: close\r\n")
         buf_socket.enqueue("\r\n")
         buf_socket.enqueue(msg)
+        buf_socket.send_pending
       end
 
       def params
@@ -234,34 +246,33 @@ module Proxy::RemoteExecution
       end
 
       def buf_socket
-        @buffered_socket ||= BufferedSocket.build(@socket)
+        @buf_socket ||= BufferedSocket.build(@socket)
       end
 
       def command
         params["command"]
       end
 
-      def ssh_user
-        params["ssh_user"]
-      end
-
       def host
         params["hostname"]
       end
 
-      def ssh_options
-        auth_methods = %w[publickey]
-        auth_methods.unshift('password') if params["ssh_password"]
-
-        ret = {}
-        ret[:port] = params["ssh_port"] if params["ssh_port"]
-        ret[:keys] = [key_file] if key_file
-        ret[:password] = params["ssh_password"] if params["ssh_password"]
-        ret[:passphrase] = params["ssh_key_passphrase"] if params["ssh_key_passphrase"]
-        ret[:keys_only] = true
-        ret[:auth_methods] = auth_methods
-        ret[:verify_host_key] = true
-        ret[:number_of_password_prompts] = 1
+      def script_runner
+        @script_runner ||= Proxy::RemoteExecution::Ssh::Runners::ScriptRunner.build(
+          runner_params,
+          suspended_action: nil
+        )
+      end
+
+      def runner_params
+        ret = { secrets: {} }
+        ret[:secrets][:ssh_password] = params["ssh_password"] if params["ssh_password"]
+        ret[:secrets][:key_passphrase] = params["ssh_key_passphrase"] if params["ssh_key_passphrase"]
+        ret[:ssh_port] = params["ssh_port"] if params["ssh_port"]
+        ret[:ssh_user] = params["ssh_user"]
+        # For compatibility only
+        ret[:script] = nil
+        ret[:hostname] = host
         ret
       end
     end

data/lib/smart_proxy_remote_execution_ssh/job_storage.rb

@@ -0,0 +1,51 @@
+# lib/job_storage.rb
+require 'sequel'
+
+module Proxy::RemoteExecution::Ssh
+  class JobStorage
+    def initialize
+      @db = Sequel.sqlite
+      @db.create_table :jobs do
+        DateTime :timestamp, null: false, default: Sequel::CURRENT_TIMESTAMP
+        String :uuid, fixed: true, size: 36, primary_key: true, null: false
+        String :hostname, null: false, index: true
+        String :execution_plan_uuid, fixed: true, size: 36, null: false, index: true
+        Integer :run_step_id, null: false
+        String :job, text: true
+      end
+    end
+
+    def find_job(uuid)
+      jobs.where(uuid: uuid).first
+    end
+
+    def job_uuids_for_host(hostname)
+      jobs_for_host(hostname).order(:timestamp)
+                             .select_map(:uuid)
+    end
+
+    def store_job(hostname, execution_plan_uuid, run_step_id, job, uuid: SecureRandom.uuid, timestamp: Time.now.utc)
+      jobs.insert(timestamp: timestamp,
+                  uuid: uuid,
+                  hostname: hostname,
+                  execution_plan_uuid: execution_plan_uuid,
+                  run_step_id: run_step_id,
+                  job: job)
+      uuid
+    end
+
+    def drop_job(execution_plan_uuid, run_step_id)
+      jobs.where(execution_plan_uuid: execution_plan_uuid, run_step_id: run_step_id).delete
+    end
+
+    private
+
+    def jobs_for_host(hostname)
+      jobs.where(hostname: hostname)
+    end
+
+    def jobs
+      @db[:jobs]
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb

@@ -0,0 +1,228 @@
+module Proxy::RemoteExecution
+  module NetSSHCompat
+    class Buffer
+      # exposes the raw content of the buffer
+      attr_reader :content
+
+      # the current position of the pointer in the buffer
+      attr_accessor :position
+
+      # Creates a new buffer, initialized to the given content. The position
+      # is initialized to the beginning of the buffer.
+      def initialize(content = +'')
+        @content = content.to_s
+        @position = 0
+      end
+
+      # Returns the length of the buffer's content.
+      def length
+        @content.length
+      end
+
+      # Returns the number of bytes available to be read (e.g., how many bytes
+      # remain between the current position and the end of the buffer).
+      def available
+        length - position
+      end
+
+      # Returns a copy of the buffer's content.
+      def to_s
+        (@content || "").dup
+      end
+
+      # Returns +true+ if the buffer contains no data (e.g., it is of zero length).
+      def empty?
+        @content.empty?
+      end
+
+      # Resets the pointer to the start of the buffer. Subsequent reads will
+      # begin at position 0.
+      def reset!
+        @position = 0
+      end
+
+      # Returns true if the pointer is at the end of the buffer. Subsequent
+      # reads will return nil, in this case.
+      def eof?
+        @position >= length
+      end
+
+      # Resets the buffer, making it empty. Also, resets the read position to
+      # 0.
+      def clear!
+        @content = +''
+        @position = 0
+      end
+
+      # Consumes n bytes from the buffer, where n is the current position
+      # unless otherwise specified. This is useful for removing data from the
+      # buffer that has previously been read, when you are expecting more data
+      # to be appended. It helps to keep the size of buffers down when they
+      # would otherwise tend to grow without bound.
+      #
+      # Returns the buffer object itself.
+      def consume!(count = position)
+        if count >= length
+          # OPTIMIZE: a fairly common case
+          clear!
+        elsif count.positive?
+          @content = @content[count..-1] || +''
+          @position -= count
+          @position = 0 if @position.negative?
+        end
+        self
+      end
+
+      # Appends the given text to the end of the buffer. Does not alter the
+      # read position. Returns the buffer object itself.
+      def append(text)
+        @content << text
+        self
+      end
+
+      # Reads and returns the next +count+ bytes from the buffer, starting from
+      # the read position. If +count+ is +nil+, this will return all remaining
+      # text in the buffer. This method will increment the pointer.
+      def read(count = nil)
+        count ||= length
+        count = length - @position if @position + count > length
+        @position += count
+        @content[@position - count, count]
+      end
+
+      # Writes the given data literally into the string. Does not alter the
+      # read position. Returns the buffer object.
+      def write(*data)
+        data.each { |datum| @content << datum.dup.force_encoding('BINARY') }
+        self
+      end
+    end
+
+    module BufferedIO
+      # This module is used to extend sockets and other IO objects, to allow
+      # them to be buffered for both read and write. This abstraction makes it
+      # quite easy to write a select-based event loop
+      # (see Net::SSH::Connection::Session#listen_to).
+      #
+      # The general idea is that instead of calling #read directly on an IO that
+      # has been extended with this module, you call #fill (to add pending input
+      # to the internal read buffer), and then #read_available (to read from that
+      # buffer). Likewise, you don't call #write directly, you call #enqueue to
+      # add data to the write buffer, and then #send_pending or #wait_for_pending_sends
+      # to actually send the data across the wire.
+      #
+      # In this way you can easily use the object as an argument to IO.select,
+      # calling #fill when it is available for read, or #send_pending when it is
+      # available for write, and then call #enqueue and #read_available during
+      # the idle times.
+      #
+      #   socket = TCPSocket.new(address, port)
+      #   socket.extend(Net::SSH::BufferedIo)
+      #
+      #   ssh.listen_to(socket)
+      #
+      #   ssh.loop do
+      #     if socket.available > 0
+      #       puts socket.read_available
+      #       socket.enqueue("response\n")
+      #     end
+      #   end
+      #
+      # Note that this module must be used to extend an instance, and should not
+      # be included in a class. If you do want to use it via an include, then you
+      # must make sure to invoke the private #initialize_buffered_io method in
+      # your class' #initialize method:
+      #
+      #   class Foo < IO
+      #     include Net::SSH::BufferedIo
+      #
+      #     def initialize
+      #       initialize_buffered_io
+      #       # ...
+      #     end
+      #   end
+
+      # Tries to read up to +n+ bytes of data from the remote end, and appends
+      # the data to the input buffer. It returns the number of bytes read, or 0
+      # if no data was available to be read.
+      def fill(count = 8192)
+        input.consume!
+        data = recv(count)
+        input.append(data)
+        return data.length
+      rescue EOFError => e
+        @input_errors << e
+        return 0
+      end
+
+      # Read up to +length+ bytes from the input buffer. If +length+ is nil,
+      # all available data is read from the buffer. (See #available.)
+      def read_available(length = nil)
+        input.read(length || available)
+      end
+
+      # Returns the number of bytes available to be read from the input buffer.
+      # (See #read_available.)
+      def available
+        input.available
+      end
+
+      # Enqueues data in the output buffer, to be written when #send_pending
+      # is called. Note that the data is _not_ sent immediately by this method!
+      def enqueue(data)
+        output.append(data)
+      end
+
+      # Sends as much of the pending output as possible. Returns +true+ if any
+      # data was sent, and +false+ otherwise.
+      def send_pending
+        if output.length.positive?
+          sent = send(output.to_s, 0)
+          output.consume!(sent)
+          return sent.positive?
+        else
+          return false
+        end
+      end
+
+      # Calls #send_pending repeatedly, if necessary, blocking until the output
+      # buffer is empty.
+      def wait_for_pending_sends
+        send_pending
+        while output.length.positive?
+          result = IO.select(nil, [self]) || next
+          next unless result[1].any?
+
+          send_pending
+        end
+      end
+
+      private
+
+      #--
+      # Can't use attr_reader here (after +private+) without incurring the
+      # wrath of "ruby -w". We hates it.
+      #++
+
+      def input
+        @input
+      end
+
+      def output
+        @output
+      end
+
+      # Initializes the intput and output buffers for this object. This method
+      # is called automatically when the module is mixed into an object via
+      # Object#extend (see Net::SSH::BufferedIo.extended), but must be called
+      # explicitly in the +initialize+ method of any class that uses
+      # Module#include to add this module.
+      def initialize_buffered_io
+        @input = Buffer.new
+        @input_errors = []
+        @output = Buffer.new
+        @output_errors = []
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -1,6 +1,7 @@
 module Proxy::RemoteExecution::Ssh
   class Plugin < Proxy::Plugin
-    SSH_LOG_LEVELS = %w[debug info warn error fatal].freeze
+    SSH_LOG_LEVELS = %w[debug info error fatal].freeze
+    MODES = %i[ssh async-ssh pull pull-mqtt].freeze
 
     http_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("http_config.ru", File.expand_path("../", __FILE__))
@@ -11,11 +12,13 @@ module Proxy::RemoteExecution::Ssh
                      :remote_working_dir      => '/var/tmp',
                      :local_working_dir       => '/var/tmp',
                      :kerberos_auth           => false,
-                     :async_ssh               => false,
                      # When set to nil, makes REX use the runner's default interval
                      # :runner_refresh_interval => nil,
                      :ssh_log_level           => :fatal,
-                     :cleanup_working_dirs    => true
+                     :cleanup_working_dirs    => true,
+                     # :mqtt_broker             => nil,
+                     # :mqtt_port               => nil,
+                     :mode                    => :ssh
 
     plugin :ssh, Proxy::RemoteExecution::Ssh::VERSION
     after_activation do
@@ -23,10 +26,12 @@ module Proxy::RemoteExecution::Ssh
       require 'smart_proxy_remote_execution_ssh/version'
       require 'smart_proxy_remote_execution_ssh/cockpit'
       require 'smart_proxy_remote_execution_ssh/api'
-      require 'smart_proxy_remote_execution_ssh/actions/run_script'
+      require 'smart_proxy_remote_execution_ssh/actions'
       require 'smart_proxy_remote_execution_ssh/dispatcher'
       require 'smart_proxy_remote_execution_ssh/log_filter'
       require 'smart_proxy_remote_execution_ssh/runners'
+      require 'smart_proxy_remote_execution_ssh/utils'
+      require 'smart_proxy_remote_execution_ssh/job_storage'
 
       Proxy::RemoteExecution::Ssh.validate!
 
@@ -40,7 +45,7 @@ module Proxy::RemoteExecution::Ssh
     def self.runner_class
       @runner_class ||= if simulate?
                           Runners::FakeScriptRunner
-                        elsif settings[:async_ssh]
+                        elsif settings.mode == :'ssh-async'
                           Runners::PollingScriptRunner
                         else
                           Runners::ScriptRunner

data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb

@@ -132,8 +132,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     def destroy_session
       if @session
         @logger.debug("Closing session with #{@ssh_user}@#{@host}")
-        @session.close
-        @session = nil
+        close_session
       end
     end
   end

data/lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb

@@ -1,12 +1,5 @@
-require 'net/ssh'
 require 'fileutils'
-
-# Rubocop can't make up its mind what it wants
-# rubocop:disable Lint/SuppressedException, Lint/RedundantCopDisableDirective
-begin
-  require 'net/ssh/krb'
-rescue LoadError; end
-# rubocop:enable Lint/SuppressedException, Lint/RedundantCopDisableDirective
+require 'smart_proxy_dynflow/runner/command'
 
 module Proxy::RemoteExecution::Ssh::Runners
   class EffectiveUserMethod
@@ -21,7 +14,7 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def on_data(received_data, ssh_channel)
       if received_data.match(login_prompt)
-        ssh_channel.send_data(effective_user_password + "\n")
+        ssh_channel.puts(effective_user_password)
         @password_sent = true
       end
     end
@@ -97,11 +90,11 @@ module Proxy::RemoteExecution::Ssh::Runners
 
   # rubocop:disable Metrics/ClassLength
   class ScriptRunner < Proxy::Dynflow::Runner::Base
+    include Proxy::Dynflow::Runner::Command
     attr_reader :execution_timeout_interval
 
     EXPECTED_POWER_ACTION_MESSAGES = ['restart host', 'shutdown host'].freeze
     DEFAULT_REFRESH_INTERVAL = 1
-    MAX_PROCESS_RETRIES = 4
 
     def initialize(options, user_method, suspended_action: nil)
       super suspended_action: suspended_action
@@ -119,6 +112,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       @local_working_dir = options.fetch(:local_working_dir, settings.local_working_dir)
       @remote_working_dir = options.fetch(:remote_working_dir, settings.remote_working_dir)
       @cleanup_working_dirs = options.fetch(:cleanup_working_dirs, settings.cleanup_working_dirs)
+      @first_execution = options.fetch(:first_execution, false)
       @user_method = user_method
     end
 
@@ -146,12 +140,13 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def start
+      Proxy::RemoteExecution::Utils.prune_known_hosts!(@host, @ssh_port, logger) if @first_execution
       prepare_start
       script = initialization_script
       logger.debug("executing script:\n#{indent_multiline(script)}")
       trigger(script)
-    rescue StandardError => e
-      logger.error("error while initalizing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
+    rescue StandardError, NotImplementedError => e
+      logger.error("error while initializing command #{e.class} #{e.message}:\n #{e.backtrace.join("\n")}")
       publish_exception('Error initializing command', e)
     end
 
@@ -177,12 +172,7 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def refresh
       return if @session.nil?
-
-      with_retries do
-        with_disconnect_handling do
-          @session.process(0)
-        end
-      end
+      super
     ensure
       check_expecting_disconnect
     end
@@ -206,32 +196,13 @@ module Proxy::RemoteExecution::Ssh::Runners
       execution_timeout_interval
     end
 
-    def with_retries
-      tries = 0
-      begin
-        yield
-      rescue StandardError => e
-        logger.error("Unexpected error: #{e.class} #{e.message}\n #{e.backtrace.join("\n")}")
-        tries += 1
-        if tries <= MAX_PROCESS_RETRIES
-          logger.error('Retrying')
-          retry
-        else
-          publish_exception('Unexpected error', e)
-        end
-      end
-    end
-
-    def with_disconnect_handling
-      yield
-    rescue IOError, Net::SSH::Disconnect => e
-      @session.shutdown!
-      check_expecting_disconnect
-      if @expecting_disconnect
-        publish_exit_status(0)
-      else
-        publish_exception('Unexpected disconnect', e)
-      end
+    def close_session
+      @session = nil
+      raise 'Control socket file does not exist' unless File.exist?(local_command_file("socket"))
+      @logger.debug("Sending exit request for session #{@ssh_user}@#{@host}")
+      args = ['/usr/bin/ssh', @host, "-o", "User=#{@ssh_user}", "-o", "ControlPath=#{local_command_file("socket")}", "-O", "exit"].flatten
+      *, err = session(args, in_stream: false, out_stream: false)
+      read_output_debug(err)
     end
 
     def close
@@ -239,12 +210,13 @@ module Proxy::RemoteExecution::Ssh::Runners
     rescue StandardError => e
       publish_exception('Error when removing remote working dir', e, false)
     ensure
-      @session.close if @session && !@session.closed?
+      close_session if @session
       FileUtils.rm_rf(local_command_dir) if Dir.exist?(local_command_dir) && @cleanup_working_dirs
     end
 
     def publish_data(data, type)
-      super(data.force_encoding('UTF-8'), type)
+      super(data.force_encoding('UTF-8'), type) unless @user_method.filter_password?(data)
+      @user_method.on_data(data, @command_in)
     end
 
     private
@@ -254,38 +226,54 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def should_cleanup?
-      @session && !@session.closed? && @cleanup_working_dirs
-    end
-
-    def session
-      @session ||= begin
-                     @logger.debug("opening session to #{@ssh_user}@#{@host}")
-                     Net::SSH.start(@host, @ssh_user, ssh_options)
-                   end
-    end
-
-    def ssh_options
-      ssh_options = {}
-      ssh_options[:port] = @ssh_port if @ssh_port
-      ssh_options[:keys] = [@client_private_key_file] if @client_private_key_file
-      ssh_options[:password] = @ssh_password if @ssh_password
-      ssh_options[:passphrase] = @key_passphrase if @key_passphrase
-      ssh_options[:keys_only] = true
-      # if the host public key is contained in the known_hosts_file,
-      # verify it, otherwise, if missing, import it and continue
-      ssh_options[:paranoid] = true
-      ssh_options[:auth_methods] = available_authentication_methods
-      ssh_options[:user_known_hosts_file] = prepare_known_hosts if @host_public_key
-      ssh_options[:number_of_password_prompts] = 1
-      ssh_options[:verbose] = settings[:ssh_log_level]
-      ssh_options[:logger] = Proxy::RemoteExecution::Ssh::LogFilter.new(Proxy::Dynflow::Log.instance)
-      return ssh_options
+      @session && @cleanup_working_dirs
+    end
+
+    # Creates session with three pipes - one for reading and two for
+    # writing. Similar to `Open3.popen3` method but without creating
+    # a separate thread to monitor it.
+    def session(args, in_stream: true, out_stream: true, err_stream: true)
+      @session = true
+
+      in_read, in_write = in_stream ? IO.pipe : '/dev/null'
+      out_read, out_write = out_stream ? IO.pipe : [nil, '/dev/null']
+      err_read, err_write = err_stream ? IO.pipe : [nil, '/dev/null']
+      command_pid = spawn(*args, :in => in_read, :out => out_write, :err => err_write)
+      in_read.close if in_stream
+      out_write.close if out_stream
+      err_write.close if err_stream
+
+      return command_pid, in_write, out_read, err_read
+    end
+
+    def ssh_options(with_pty = false)
+      ssh_options = []
+      ssh_options << "-tt" if with_pty
+      ssh_options << "-o User=#{@ssh_user}"
+      ssh_options << "-o Port=#{@ssh_port}" if @ssh_port
+      ssh_options << "-o IdentityFile=#{@client_private_key_file}" if @client_private_key_file
+      ssh_options << "-o IdentitiesOnly=yes"
+      ssh_options << "-o StrictHostKeyChecking=no"
+      ssh_options << "-o PreferredAuthentications=#{available_authentication_methods.join(',')}"
+      ssh_options << "-o UserKnownHostsFile=#{prepare_known_hosts}" if @host_public_key
+      ssh_options << "-o NumberOfPasswordPrompts=1"
+      ssh_options << "-o LogLevel=#{settings[:ssh_log_level]}"
+      ssh_options << "-o ControlMaster=auto"
+      ssh_options << "-o ControlPath=#{local_command_file("socket")}"
+      ssh_options << "-o ControlPersist=yes"
     end
 
     def settings
       Proxy::RemoteExecution::Ssh::Plugin.settings
     end
 
+    def get_args(command, with_pty = false)
+      args = []
+      args = [{'SSHPASS' => @key_passphrase}, '/usr/bin/sshpass', '-P', 'passphrase', '-e'] if @key_passphrase
+      args = [{'SSHPASS' => @ssh_password}, '/usr/bin/sshpass', '-e'] if @ssh_password
+      args += ['/usr/bin/ssh', @host, ssh_options(with_pty), command].flatten
+    end
+
     # Initiates run of the remote command and yields the data when
     # available. The yielding doesn't happen automatically, but as
     # part of calling the `refresh` method.
@@ -294,30 +282,9 @@ module Proxy::RemoteExecution::Ssh::Runners
 
       @started = false
       @user_method.reset
+      @command_pid, @command_in, @command_out = session(get_args(command, with_pty: true), err_stream: false)
+      @started = true
 
-      session.open_channel do |channel|
-        channel.request_pty
-        channel.on_data do |ch, data|
-          publish_data(data, 'stdout') unless @user_method.filter_password?(data)
-          @user_method.on_data(data, ch)
-        end
-        channel.on_extended_data { |ch, type, data| publish_data(data, 'stderr') }
-        # standard exit of the command
-        channel.on_request('exit-status') { |ch, data| publish_exit_status(data.read_long) }
-        # on signal: sending the signal value (such as 'TERM')
-        channel.on_request('exit-signal') do |ch, data|
-          publish_exit_status(data.read_string)
-          ch.close
-          # wait for the channel to finish so that we know at the end
-          # that the session is inactive
-          ch.wait
-        end
-        channel.exec(command) do |_, success|
-          @started = true
-          raise('Error initializing command') unless success
-        end
-      end
-      session.process(0) { !run_started? }
       return true
     end
 
@@ -325,36 +292,27 @@ module Proxy::RemoteExecution::Ssh::Runners
       @started && @user_method.sent_all_data?
     end
 
-    def run_sync(command, stdin = nil)
+    def read_output_debug(err_io, out_io = nil)
       stdout = ''
-      stderr = ''
-      exit_status = nil
-      started = false
-
-      channel = session.open_channel do |ch|
-        ch.on_data do |c, data|
-          stdout.concat(data)
-        end
-        ch.on_extended_data { |_, _, data| stderr.concat(data) }
-        ch.on_request('exit-status') { |_, data| exit_status = data.read_long }
-        # Send data to stdin if we have some
-        ch.send_data(stdin) unless stdin.nil?
-        # on signal: sending the signal value (such as 'TERM')
-        ch.on_request('exit-signal') do |_, data|
-          exit_status = data.read_string
-          ch.close
-          ch.wait
-        end
-        ch.exec command do |_, success|
-          raise 'could not execute command' unless success
-
-          started = true
-        end
+      debug_str = ''
+
+      if out_io
+        stdout += out_io.read until out_io.eof? rescue
+        out_io.close
       end
-      session.process(0) { !started }
-      # Closing the channel without sending any data gives us SIGPIPE
-      channel.close unless stdin.nil?
-      channel.wait
+      debug_str += err_io.read until err_io.eof? rescue
+      err_io.close
+      debug_str.lines.each { |line| @logger.debug(line.strip) }
+
+      return stdout, debug_str
+    end
+
+    def run_sync(command, stdin = nil)
+      pid, tx, rx, err = session(get_args(command))
+      tx.puts(stdin) unless stdin.nil?
+      tx.close
+      stdout, stderr = read_output_debug(err, rx)
+      exit_status = Process.wait2(pid)[1].exitstatus
       return exit_status, stdout, stderr
     end
 
@@ -371,7 +329,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def local_command_file(filename)
-      File.join(local_command_dir, filename)
+      File.join(ensure_local_directory(local_command_dir), filename)
     end
 
     def remote_command_dir
@@ -453,15 +411,8 @@ module Proxy::RemoteExecution::Ssh::Runners
 
     def available_authentication_methods
       methods = %w[publickey] # Always use pubkey auth as fallback
-      if settings[:kerberos_auth]
-        if defined? Net::SSH::Kerberos
-          methods << 'gssapi-with-mic'
-        else
-          @logger.warn('Kerberos authentication requested but not available')
-        end
-      end
+      methods << 'gssapi-with-mic' if settings[:kerberos_auth]
       methods.unshift('password') if @ssh_password
-
       methods
     end
   end

data/lib/smart_proxy_remote_execution_ssh/utils.rb

@@ -0,0 +1,24 @@
+require 'open3'
+
+module Proxy::RemoteExecution
+  module Utils
+    class << self
+      def prune_known_hosts!(hostname, port, logger = Logger.new($stdout))
+        return if Net::SSH::KnownHosts.search_for(hostname).empty?
+
+        target = if port == 22
+                   hostname
+                 else
+                   "[#{hostname}]:#{port}"
+                 end
+
+        Open3.popen3('ssh-keygen', '-R', target) do |_stdin, stdout, _stderr, wait_thr|
+          wait_thr.join
+          stdout.read
+        end
+      rescue Errno::ENOENT => e
+        logger.warn("Could not remove #{hostname} from know_hosts: #{e}")
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.4.1'
+      VERSION = '0.5.0'
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh.rb

@@ -20,7 +20,9 @@ module Proxy::RemoteExecution
           raise "Ssh public key file #{public_key_file} doesn't exist"
         end
 
+        validate_mode!
         validate_ssh_log_level!
+        validate_mqtt_settings!
       end
 
       def private_key_file
@@ -31,6 +33,35 @@ module Proxy::RemoteExecution
         File.expand_path("#{private_key_file}.pub")
       end
 
+      def validate_mode!
+        Plugin.settings.mode = Plugin.settings.mode.to_sym
+
+        unless Plugin::MODES.include? Plugin.settings.mode
+          raise "Mode has to be one of #{Plugin::MODES.join(', ')}, given #{Plugin.settings.mode}"
+        end
+
+        if Plugin.settings.async_ssh
+          Plugin.logger.warn('Option async_ssh is deprecated, use ssh-async mode instead.')
+
+          case Plugin.settings.mode
+          when :ssh
+            Plugin.logger.warn('Deprecated option async_ssh used together with ssh mode, switching mode to ssh-async.')
+            Plugin.settings.mode = :'ssh-async'
+          when :'async-ssh'
+            # This is a noop
+          else
+            Plugin.logger.warn('Deprecated option async_ssh used together with incompatible mode, ignoring.')
+          end
+        end
+      end
+
+      def validate_mqtt_settings!
+        return unless Plugin.settings.mode == :'pull-mqtt'
+
+        raise 'mqtt_broker has to be set when pull-mqtt mode is used' if Plugin.settings.mqtt_broker.nil?
+        raise 'mqtt_port has to be set when pull-mqtt mode is used' if Plugin.settings.mqtt_port.nil?
+      end
+
       def validate_ssh_log_level!
         wanted_level = Plugin.settings.ssh_log_level.to_s
         levels = Plugin::SSH_LOG_LEVELS
@@ -51,6 +82,10 @@ module Proxy::RemoteExecution
 
         Plugin.settings.ssh_log_level = Plugin.settings.ssh_log_level.to_sym
       end
+
+      def job_storage
+        @job_storage ||= Proxy::RemoteExecution::Ssh::JobStorage.new
+      end
     end
   end
 end

data/settings.d/remote_execution_ssh.yml.example

@@ -4,17 +4,23 @@
 :local_working_dir: '/var/tmp'
 :remote_working_dir: '/var/tmp'
 # :kerberos_auth: false
-# :async_ssh: false
+
+# Mode of operation, one of ssh, ssh-async, pull, pull-mqtt
+:mode: ssh
 
 # Defines how often (in seconds) should the runner check
 # for new data leave empty to use the runner's default
 # (1 second for regular, 60 seconds with async_ssh enabled)
 # :runner_refresh_interval:
 
-# Defines the verbosity of logging coming from Net::SSH
-# one of :debug, :info, :warn, :error, :fatal
+# Defines the verbosity of logging coming from ssh command
+# one of :debug, :info, :error, :fatal
 # must be lower than general log level
 # :ssh_log_level: fatal
 
 # Remove working directories on job completion
 # :cleanup_working_dirs: true
+
+# MQTT configuration, need to be set if mode is set to pull-mqtt
+# :mqtt_broker: localhost
+# :mqtt_port: 1883

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-09 00:00:00.000000000 Z
+date: 2021-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -124,6 +124,20 @@ dependencies:
         version: '0.5'
 - !ruby/object:Gem::Dependency
   name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+- !ruby/object:Gem::Dependency
+  name: mqtt
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -149,6 +163,8 @@ files:
 - README.md
 - bundler.d/remote_execution_ssh.rb
 - lib/smart_proxy_remote_execution_ssh.rb
+- lib/smart_proxy_remote_execution_ssh/actions.rb
+- lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb
 - lib/smart_proxy_remote_execution_ssh/actions/run_script.rb
 - lib/smart_proxy_remote_execution_ssh/api.rb
 - lib/smart_proxy_remote_execution_ssh/async_scripts/control.sh
@@ -156,12 +172,15 @@ files:
 - lib/smart_proxy_remote_execution_ssh/cockpit.rb
 - lib/smart_proxy_remote_execution_ssh/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
+- lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb
 - lib/smart_proxy_remote_execution_ssh/runners.rb
 - lib/smart_proxy_remote_execution_ssh/runners/fake_script_runner.rb
 - lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb
 - lib/smart_proxy_remote_execution_ssh/runners/script_runner.rb
+- lib/smart_proxy_remote_execution_ssh/utils.rb
 - lib/smart_proxy_remote_execution_ssh/version.rb
 - lib/smart_proxy_remote_execution_ssh/webrick_ext.rb
 - settings.d/remote_execution_ssh.yml.example