smart_proxy_remote_execution_ssh 0.8.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a599d3732d4f6b064a850ae8baced28354980096785a340828533f953f844c4
-  data.tar.gz: 160b80ab983f0eb7b987987191b48c19db42d3b76079b0ba6639ec831dccc6f3
+  metadata.gz: '048f8694430967060fae6bd7790fe56930b5c2bac37f3bc26e7b13db69081da6'
+  data.tar.gz: 5a3b1e344a47aa4c97dcce97f43dc9e424d398f79c67a6f1e066d670758c99f0
 SHA512:
-  metadata.gz: 349961aa474809225d9781bda09687eb23f00db51cca9f758a57e075fa3d19fc07b729569b064887f91bb7f07190152d6ec9cb01e8ccf48df0841e9c3f85d10e
-  data.tar.gz: 8ad40859c92f7d1cd089adf0f37bb0aaa8add745cd461c2e1ed5b484d3ae6b856ac0ee53e61c62e89d564a4ba676b22bf5f01618da970986cd60e42591ff6346
+  metadata.gz: a4bc90b3dda6d190c8f94aeb6731a3a6679788ea4f950e1bb4196c1d5da4630b170fe152e9c96a5e4cb7dcd25ba1d6371292e52b4059773759c1fbe8461bd347
+  data.tar.gz: f4849b32f7529cbb29d127ae1c4c9fd9642b0700c834b1e3eebf59b5d9cfca282fca674188a734a37b36d45ff3e423096aa3512ceae118aef7ff831cb1600a38

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -5,6 +5,7 @@ require 'time'
 module Proxy::RemoteExecution::Ssh::Actions
   class PullScript < Proxy::Dynflow::Action::Runner
     JobDelivered = Class.new
+    PickupTimeout = Class.new
 
     execution_plan_hooks.use :cleanup, :on => :stopped
 
@@ -17,9 +18,14 @@ module Proxy::RemoteExecution::Ssh::Actions
       if event == JobDelivered
         output[:state] = :delivered
         suspend
+      elsif event == PickupTimeout
+        process_pickup_timeout
       else
         super
       end
+    rescue => e
+      action_logger.error(e)
+      process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
     end
 
     def init_run
@@ -27,9 +33,12 @@ module Proxy::RemoteExecution::Ssh::Actions
                        ::Proxy::Dynflow::OtpManager.generate_otp(execution_plan_id)
                      end
 
-      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''))
+      plan_event(PickupTimeout, input[:time_to_pickup], optional: true) if input[:time_to_pickup]
+
+      input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''), effective_user: input[:effective_user])
       output[:state] = :ready_for_pickup
       output[:result] = []
+
       mqtt_start(otp_password) if input[:with_mqtt]
       suspend
     end
@@ -37,6 +46,7 @@ module Proxy::RemoteExecution::Ssh::Actions
     def cleanup(_plan = nil)
       job_storage.drop_job(execution_plan_id, run_step_id)
       Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.done(input[:job_uuid])
     end
 
     def process_external_event(event)
@@ -103,9 +113,11 @@ module Proxy::RemoteExecution::Ssh::Actions
           'username': execution_plan_id,
           'password': otp_password,
           'return_url': "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/update",
+          'version': 'v1',
+          'effective_user': input[:effective_user]
         },
       )
-      mqtt_notify payload
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.new(input[:job_uuid], mqtt_topic, payload)
       output[:state] = :notified
     end
 
@@ -121,18 +133,7 @@ module Proxy::RemoteExecution::Ssh::Actions
     end
 
     def mqtt_notify(payload)
-      with_mqtt_client do |c|
-        c.publish(mqtt_topic, JSON.dump(payload), false, 1)
-      end
-    end
-
-    def with_mqtt_client(&block)
-      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
-                           :ssl => settings.mqtt_tls,
-                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
-                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
-                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
-                           &block)
+      Proxy::RemoteExecution::Ssh::MQTT.publish(mqtt_topic, JSON.dump(payload))
     end
 
     def host_name
@@ -147,10 +148,6 @@ module Proxy::RemoteExecution::Ssh::Actions
       "yggdrasil/#{host_name}/data/in"
     end
 
-    def settings
-      Proxy::RemoteExecution::Ssh::Plugin.settings
-    end
-
     def job_storage
       Proxy::RemoteExecution::Ssh.job_storage
     end
@@ -164,5 +161,13 @@ module Proxy::RemoteExecution::Ssh::Actions
         directive: 'foreman'
       }
     end
+
+    def process_pickup_timeout
+      if output[:state] != :delivered
+        raise "The job was not picked up in time"
+      else
+        suspend
+      end
+    end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -64,7 +64,9 @@ module Proxy::RemoteExecution
         do_authorize_with_ssl_client
 
         with_authorized_job(job_uuid) do |job_record|
+          Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.running(job_record[:uuid])
           notify_job(job_record, Actions::PullScript::JobDelivered)
+          response.headers['X-Foreman-Effective-User'] = job_record[:effective_user]
           job_record[:job]
         end
       end

data/lib/smart_proxy_remote_execution_ssh/command_logging.rb

@@ -14,7 +14,7 @@ module Proxy::RemoteExecution::Ssh::Runners
           logger.debug(line.chomp) if user_method.nil? || !user_method.filter_password?(line)
           user_method.on_data(data, pm.stdin) if user_method
         end
-        ''
+        data
       end
       pm.on_stdout(&callback)
       pm.on_stderr(&callback)

data/lib/smart_proxy_remote_execution_ssh/job_storage.rb

@@ -11,6 +11,7 @@ module Proxy::RemoteExecution::Ssh
         String :hostname, null: false, index: true
         String :execution_plan_uuid, fixed: true, size: 36, null: false, index: true
         Integer :run_step_id, null: false
+        String :effective_user
         String :job, text: true
       end
     end
@@ -24,13 +25,14 @@ module Proxy::RemoteExecution::Ssh
                              .select_map(:uuid)
     end
 
-    def store_job(hostname, execution_plan_uuid, run_step_id, job, uuid: SecureRandom.uuid, timestamp: Time.now.utc)
+    def store_job(hostname, execution_plan_uuid, run_step_id, job, uuid: SecureRandom.uuid, timestamp: Time.now.utc, effective_user: nil)
       jobs.insert(timestamp: timestamp,
                   uuid: uuid,
                   hostname: hostname,
                   execution_plan_uuid: execution_plan_uuid,
                   run_step_id: run_step_id,
-                  job: job)
+                  job: job,
+                  effective_user: effective_user)
       uuid
     end
 

data/lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb

@@ -0,0 +1,169 @@
+require 'concurrent'
+require 'mqtt'
+
+class Proxy::RemoteExecution::Ssh::MQTT
+  class Dispatcher
+    include Singleton
+
+    attr_reader :reference
+    def initialize
+      limit = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_rate_limit]
+      @reference = DispatcherActor.spawn('MQTT dispatcher',
+                                         Proxy::Dynflow::Core.world.clock,
+                                         limit)
+    end
+
+    def new(uuid, topic, payload)
+      reference.tell([:new, uuid, topic, payload])
+    end
+
+    def running(uuid)
+      reference.tell([:running, uuid])
+    end
+
+    def resend(uuid)
+      reference.tell([:resend, uuid])
+    end
+
+    def done(uuid)
+      reference.tell([:done, uuid])
+    end
+  end
+
+  class DispatcherActor < Concurrent::Actor::RestartingContext
+    JobDefinition = Struct.new :uuid, :topic, :payload
+
+    class Tracker
+      def initialize(limit, clock)
+        @clock = clock
+        @limit = limit
+        @jobs = {}
+        @pending = []
+        @running = Set.new
+        @hot = Set.new
+        @cold = Set.new
+      end
+
+      def new(uuid, topic, payload)
+        @jobs[uuid] = JobDefinition.new(uuid, topic, payload)
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def running(uuid)
+        [@pending, @hot, @cold].each { |source| source.delete(uuid) }
+        @running << uuid
+      end
+
+      def resend(uuid)
+        return unless @jobs[uuid]
+
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def done(uuid)
+        @jobs.delete(uuid)
+        [@pending, @running, @hot, @cold].each do |source|
+          source.delete(uuid)
+        end
+        dispatch_pending
+      end
+
+      def needs_processing?
+        pending_count.positive? || @hot.any? || @cold.any?
+      end
+
+      def pending_count
+        pending = @pending.count
+        return pending if @limit.nil?
+
+        running = [@running, @hot, @cold].map(&:count).sum
+        capacity = @limit - running
+        pending > capacity ? capacity : pending
+      end
+
+      def dispatch_pending
+        pending_count.times do
+          mqtt_notify(@pending.first)
+          @hot << @pending.shift
+        end
+      end
+
+      def process
+        @cold.each { |uuid| schedule_resend(uuid) }
+        @cold = @hot
+        @hot = Set.new
+
+        dispatch_pending
+      end
+
+      def mqtt_notify(uuid)
+        job = @jobs[uuid]
+        return if job.nil?
+
+        Proxy::RemoteExecution::Ssh::MQTT.publish(job.topic, JSON.dump(job.payload))
+      end
+
+      def settings
+        Proxy::RemoteExecution::Ssh::Plugin.settings
+      end
+
+      def schedule_resend(uuid)
+        @clock.ping(Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance, resend_interval, uuid, :resend)
+      end
+
+      def resend_interval
+        settings[:mqtt_resend_interval]
+      end
+    end
+
+    def initialize(clock, limit = nil)
+      @tracker = Tracker.new(limit, clock)
+
+      interval = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_ttl]
+      @timer = Concurrent::TimerTask.new(execution_interval: interval) do
+        reference.tell(:tick)
+      end
+    end
+
+    def on_message(message)
+      action, arg = message
+      # Enable the timer just in case anything in tracker raises an exception so
+      # we can continue
+      timer_on
+      case action
+      when :new
+        _, uuid, topic, payload = message
+        @tracker.new(uuid, topic, payload)
+      when :resend
+        @tracker.resend(arg)
+      when :running
+        @tracker.running(arg)
+      when :done
+        @tracker.done(arg)
+      when :tick
+        @tracker.process
+      end
+      timer_set(@tracker.needs_processing?)
+    end
+
+    def timer_set(on)
+      on ? timer_on : timer_off
+    end
+
+    def timer_on
+      @timer.execute
+    end
+
+    def timer_off
+      @timer.shutdown
+    end
+
+    # In case an exception is raised during processing, instruct concurrent-ruby
+    # to keep going without losing state
+    def behaviour_definition
+      Concurrent::Actor::Behaviour.restarting_behaviour_definition(:resume!)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/mqtt.rb

@@ -0,0 +1,23 @@
+module Proxy::RemoteExecution::Ssh
+  class MQTT
+    require 'smart_proxy_remote_execution_ssh/mqtt/dispatcher'
+
+    class << self
+      def publish(topic, payload, retain: false, qos: 1)
+        with_mqtt_client do |c|
+          c.publish(topic, payload, retain, qos)
+        end
+      end
+
+      def with_mqtt_client(&block)
+        ::MQTT::Client.connect(Plugin.settings.mqtt_broker,
+                               Plugin.settings.mqtt_port,
+                               :ssl => Plugin.settings.mqtt_tls,
+                               :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
+                               :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
+                               :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
+                               &block)
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb

@@ -32,7 +32,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       return [] unless @password
 
       prompt = ['-P', @prompt] if @prompt
-      [{'SSHPASS' => SensitiveString.new(@password)}, '/usr/bin/sshpass', '-e', prompt].compact
+      [{'SSHPASS' => SensitiveString.new(@password)}, 'sshpass', '-e', prompt].compact
     end
 
     def ssh_options
@@ -93,7 +93,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     def command(cmd)
       raise "Cannot build command to run over multiplexed connection without having an established connection" unless connected?
 
-      ['/usr/bin/ssh', reuse_ssh_options, cmd].flatten
+      ['ssh', reuse_ssh_options, cmd].flatten
     end
 
     private
@@ -103,7 +103,7 @@ module Proxy::RemoteExecution::Ssh::Runners
       # does not close its stderr which trips up the process manager which
       # expects all FDs to be closed
 
-      full_command = [method.ssh_command_prefix, '/usr/bin/ssh', establish_ssh_options, method.ssh_options, @host, 'true'].flatten
+      full_command = [method.ssh_command_prefix, 'ssh', establish_ssh_options, method.ssh_options, @host, 'true'].flatten
       log_command(full_command)
       pm = Proxy::Dynflow::ProcessManager.new(full_command)
       pm.start!
@@ -166,7 +166,7 @@ module Proxy::RemoteExecution::Ssh::Runners
     end
 
     def verify_key_passphrase
-      command = ['/usr/bin/ssh-keygen', '-y', '-f', File.expand_path(@client_private_key_file)]
+      command = ['ssh-keygen', '-y', '-f', File.expand_path(@client_private_key_file)]
       log_command(command, label: "Checking if private key has passphrase")
       pm = Proxy::Dynflow::ProcessManager.new(command)
       pm.start!

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -25,7 +25,10 @@ module Proxy::RemoteExecution::Ssh
                      # :mqtt_broker             => nil,
                      # :mqtt_port               => nil,
                      # :mqtt_tls                => nil,
-                     :mode                    => :ssh
+                     # :mqtt_rate_limit         => nil
+                     :mode                    => :ssh,
+                     :mqtt_resend_interval    => 900,
+                     :mqtt_ttl                => 5
 
     capability(proc { 'cockpit' if settings.cockpit_integration })
 
@@ -45,6 +48,11 @@ module Proxy::RemoteExecution::Ssh
       Proxy::RemoteExecution::Ssh.validate!
 
       Proxy::Dynflow::TaskLauncherRegistry.register('ssh', Proxy::Dynflow::TaskLauncher::Batch)
+      if settings.mode == :'pull-mqtt'
+        require 'smart_proxy_remote_execution_ssh/mqtt'
+        # Force initialization
+        Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance
+      end
     end
 
     def self.simulate?

data/lib/smart_proxy_remote_execution_ssh/runners/polling_script_runner.rb

@@ -133,7 +133,6 @@ module Proxy::RemoteExecution::Ssh::Runners
     def cleanup
       if @cleanup_working_dirs
         ensure_remote_command("rm -rf #{remote_command_dir}",
-                              publish: true,
                               error: "Unable to remove working directory #{remote_command_dir} on remote system, exit code: %{exit_code}")
       end
     end

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.8.0'
+      VERSION = '0.10.0'
     end
   end
 end

data/settings.d/remote_execution_ssh.yml.example

@@ -32,3 +32,7 @@
 # unset, SSL gets used if smart-proxy's foreman_ssl_cert, foreman_ssl_key and
 # foreman_ssl_ca settings are set available.
 # :mqtt_tls:
+
+# The notification is sent over mqtt every $mqtt_resend_interval seconds, until
+# the job is picked up by the host or cancelled
+# :mqtt_resend_interval: 900

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-01 00:00:00.000000000 Z
+date: 2022-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -175,6 +175,8 @@ files:
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
 - lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
 - lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb
@@ -205,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.26
+rubygems_version: 3.3.20
 signing_key:
 specification_version: 4
 summary: Ssh remote execution provider for Foreman Smart-Proxy