smart_proxy_remote_execution_ssh 0.9.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f28e2eb48888626dad82eb56e7a5309da15d584e6d4c53eaf648e5ddfc6f3d1e
-  data.tar.gz: a5b604d80264dc78b461e1792100ecd6fc9a4a71791b8df5b064bd80a32f6920
+  metadata.gz: debc9989aade4d23bda1b90e9fa8e85c99948f5b9ca37d6cecca7573bb994e5e
+  data.tar.gz: 38a2731c7f28643daaa33ddd29f93172b7467877f36b12c750b82f86e921c04d
 SHA512:
-  metadata.gz: 3231a3d465b180d1b8b325fd9fc341c009b566452946ee56837f9f37e6502d7095d1a213bc5b4b13078b9d54e5bd592f5430aea502ecdee93edab15901440815
-  data.tar.gz: 62e77d3cf6d37c90b2e6e8e48ad8b2080186a50fd1d3633b38743a099e0897b9b1e4d4dafaf7b02b1855dcd51036d418333c156438c3b20e5fa42feac93ded17
+  metadata.gz: a3f5776ce0f6ad361f2c344d0ff5d5cef9b6b032e23dc42ae0e877298abd9e95c0156c77334b106036f376e62c60c01ab13344d6bda53baa52c93c7caba79b71
+  data.tar.gz: d1de077038e5fce86df1871e4702e2abb4c7de4937f66aa8730803bb4542aa8f26c1c848d24bb74db1ce5594514d3a552b92b7ad315d346ead5e7586cb20da20

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -4,9 +4,19 @@ require 'time'
 
 module Proxy::RemoteExecution::Ssh::Actions
   class PullScript < Proxy::Dynflow::Action::Runner
+    include ::Dynflow::Action::Timeouts
+
     JobDelivered = Class.new
     PickupTimeout = Class.new
-    ResendNotification = Class.new
+
+    # The proxy has the job stored in its job storage
+    READY_FOR_PICKUP = 'ready_for_pickup'
+    # The host was notified over MQTT at least once
+    NOTIFIED = 'notified'
+    # The host has picked up the job
+    DELIVERED = 'delivered'
+    # We received at least one output from the host
+    RUNNING = 'running'
 
     execution_plan_hooks.use :cleanup, :on => :stopped
 
@@ -17,20 +27,22 @@ module Proxy::RemoteExecution::Ssh::Actions
 
     def run(event = nil)
       if event == JobDelivered
-        output[:state] = :delivered
+        output[:state] = DELIVERED
         suspend
       elsif event == PickupTimeout
         process_pickup_timeout
-      elsif event == ResendNotification
-        if input[:with_mqtt] && %w(ready_for_pickup notified).include?(output[:state])
-          schedule_mqtt_resend
-          mqtt_start(::Proxy::Dynflow::OtpManager.passwords[execution_plan_id])
+      elsif event == ::Dynflow::Action::Timeouts::Timeout
+        process_timeout
+      elsif event.nil?
+        if (timeout = input['execution_timeout_interval'])
+          schedule_timeout(timeout, optional: true)
         end
-        suspend
+        super
       else
         super
       end
     rescue => e
+      cleanup
       action_logger.error(e)
       process_update(Proxy::Dynflow::Runner::Update.encode_exception('Proxy error', e))
     end
@@ -43,22 +55,21 @@ module Proxy::RemoteExecution::Ssh::Actions
       plan_event(PickupTimeout, input[:time_to_pickup], optional: true) if input[:time_to_pickup]
 
       input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''), effective_user: input[:effective_user])
-      output[:state] = :ready_for_pickup
+      output[:state] = READY_FOR_PICKUP
       output[:result] = []
-      if input[:with_mqtt]
-        schedule_mqtt_resend
-        mqtt_start(otp_password)
-      end
+
+      mqtt_start(otp_password) if input[:with_mqtt]
       suspend
     end
 
     def cleanup(_plan = nil)
       job_storage.drop_job(execution_plan_id, run_step_id)
       Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.done(input[:job_uuid])
     end
 
     def process_external_event(event)
-      output[:state] = :running
+      output[:state] = RUNNING
       data = event.data
       case data['version']
       when nil
@@ -97,19 +108,34 @@ module Proxy::RemoteExecution::Ssh::Actions
       process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
     end
 
-    def kill_run
+    def process_timeout
+      kill_run "Execution timeout exceeded"
+    end
+
+    def kill_run(fail_msg = 'The job was cancelled by the user')
+      continuous_output = Proxy::Dynflow::ContinuousOutput.new
+      exit_code = nil
+
       case output[:state]
-      when :ready_for_pickup
+      when READY_FOR_PICKUP, NOTIFIED
         # If the job is not running yet on the client, wipe it from storage
         cleanup
-        # TODO: Stop the action
-      when :notified, :running
+        exit_code = 'EXCEPTION'
+      when DELIVERED, RUNNING
         # Client was notified or is already running, dealing with this situation
         # is only supported if mqtt is available
         # Otherwise we have to wait it out
-        mqtt_cancel if input[:with_mqtt]
+        if input[:with_mqtt]
+          mqtt_cancel 
+          fail_msg += ', notifying the host over MQTT'
+        else
+          fail_msg += ', however the job was triggered without MQTT and cannot be stopped'
+        end
       end
-      suspend
+      continuous_output.add_output(fail_msg + "\n")
+      process_update(Proxy::Dynflow::Runner::Update.new(continuous_output, exit_code))
+
+      suspend unless exit_code
     end
 
     def mqtt_start(otp_password)
@@ -125,13 +151,13 @@ module Proxy::RemoteExecution::Ssh::Actions
           'effective_user': input[:effective_user]
         },
       )
-      mqtt_notify payload
-      output[:state] = :notified
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.new(input[:job_uuid], mqtt_topic, payload)
+      output[:state] = NOTIFIED
     end
 
     def mqtt_cancel
-      cleanup
       payload = mqtt_payload_base.merge(
+        content: "#{input[:proxy_url]}/ssh/jobs/#{input[:job_uuid]}/cancel",
         metadata: {
           'event': 'cancel',
           'job_uuid': input[:job_uuid]
@@ -141,18 +167,7 @@ module Proxy::RemoteExecution::Ssh::Actions
     end
 
     def mqtt_notify(payload)
-      with_mqtt_client do |c|
-        c.publish(mqtt_topic, JSON.dump(payload), false, 1)
-      end
-    end
-
-    def with_mqtt_client(&block)
-      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
-                           :ssl => settings.mqtt_tls,
-                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
-                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
-                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
-                           &block)
+      Proxy::RemoteExecution::Ssh::MQTT.publish(mqtt_topic, JSON.dump(payload))
     end
 
     def host_name
@@ -167,10 +182,6 @@ module Proxy::RemoteExecution::Ssh::Actions
       "yggdrasil/#{host_name}/data/in"
     end
 
-    def settings
-      Proxy::RemoteExecution::Ssh::Plugin.settings
-    end
-
     def job_storage
       Proxy::RemoteExecution::Ssh.job_storage
     end
@@ -186,15 +197,9 @@ module Proxy::RemoteExecution::Ssh::Actions
     end
 
     def process_pickup_timeout
-      if output[:state] != :delivered
-        raise "The job was not picked up in time"
-      else
-        suspend
-      end
-    end
+      suspend unless [READY_FOR_PICKUP, NOTIFIED].include? output[:state]
 
-    def schedule_mqtt_resend
-      plan_event(ResendNotification, settings[:mqtt_resend_interval], optional: true)
+      kill_run 'The job was not picked up in time'
     end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -64,12 +64,21 @@ module Proxy::RemoteExecution
         do_authorize_with_ssl_client
 
         with_authorized_job(job_uuid) do |job_record|
+          Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.running(job_record[:uuid])
           notify_job(job_record, Actions::PullScript::JobDelivered)
           response.headers['X-Foreman-Effective-User'] = job_record[:effective_user]
           job_record[:job]
         end
       end
 
+      get "/jobs/:job_uuid/cancel" do |job_uuid|
+        do_authorize_with_ssl_client
+
+        with_authorized_job(job_uuid) do |job_record|
+          {}
+        end
+      end
+
       private
 
       def notify_job(job_record, event)

data/lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb

@@ -0,0 +1,169 @@
+require 'concurrent'
+require 'mqtt'
+
+class Proxy::RemoteExecution::Ssh::MQTT
+  class Dispatcher
+    include Singleton
+
+    attr_reader :reference
+    def initialize
+      limit = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_rate_limit]
+      @reference = DispatcherActor.spawn('MQTT dispatcher',
+                                         Proxy::Dynflow::Core.world.clock,
+                                         limit)
+    end
+
+    def new(uuid, topic, payload)
+      reference.tell([:new, uuid, topic, payload])
+    end
+
+    def running(uuid)
+      reference.tell([:running, uuid])
+    end
+
+    def resend(uuid)
+      reference.tell([:resend, uuid])
+    end
+
+    def done(uuid)
+      reference.tell([:done, uuid])
+    end
+  end
+
+  class DispatcherActor < Concurrent::Actor::RestartingContext
+    JobDefinition = Struct.new :uuid, :topic, :payload
+
+    class Tracker
+      def initialize(limit, clock)
+        @clock = clock
+        @limit = limit
+        @jobs = {}
+        @pending = []
+        @running = Set.new
+        @hot = Set.new
+        @cold = Set.new
+      end
+
+      def new(uuid, topic, payload)
+        @jobs[uuid] = JobDefinition.new(uuid, topic, payload)
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def running(uuid)
+        [@pending, @hot, @cold].each { |source| source.delete(uuid) }
+        @running << uuid
+      end
+
+      def resend(uuid)
+        return unless @jobs[uuid]
+
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def done(uuid)
+        @jobs.delete(uuid)
+        [@pending, @running, @hot, @cold].each do |source|
+          source.delete(uuid)
+        end
+        dispatch_pending
+      end
+
+      def needs_processing?
+        pending_count.positive? || @hot.any? || @cold.any?
+      end
+
+      def pending_count
+        pending = @pending.count
+        return pending if @limit.nil?
+
+        running = [@running, @hot, @cold].map(&:count).sum
+        capacity = @limit - running
+        pending > capacity ? capacity : pending
+      end
+
+      def dispatch_pending
+        pending_count.times do
+          mqtt_notify(@pending.first)
+          @hot << @pending.shift
+        end
+      end
+
+      def process
+        @cold.each { |uuid| schedule_resend(uuid) }
+        @cold = @hot
+        @hot = Set.new
+
+        dispatch_pending
+      end
+
+      def mqtt_notify(uuid)
+        job = @jobs[uuid]
+        return if job.nil?
+
+        Proxy::RemoteExecution::Ssh::MQTT.publish(job.topic, JSON.dump(job.payload))
+      end
+
+      def settings
+        Proxy::RemoteExecution::Ssh::Plugin.settings
+      end
+
+      def schedule_resend(uuid)
+        @clock.ping(Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance, resend_interval, uuid, :resend)
+      end
+
+      def resend_interval
+        settings[:mqtt_resend_interval]
+      end
+    end
+
+    def initialize(clock, limit = nil)
+      @tracker = Tracker.new(limit, clock)
+
+      interval = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_ttl]
+      @timer = Concurrent::TimerTask.new(execution_interval: interval) do
+        reference.tell(:tick)
+      end
+    end
+
+    def on_message(message)
+      action, arg = message
+      # Enable the timer just in case anything in tracker raises an exception so
+      # we can continue
+      timer_on
+      case action
+      when :new
+        _, uuid, topic, payload = message
+        @tracker.new(uuid, topic, payload)
+      when :resend
+        @tracker.resend(arg)
+      when :running
+        @tracker.running(arg)
+      when :done
+        @tracker.done(arg)
+      when :tick
+        @tracker.process
+      end
+      timer_set(@tracker.needs_processing?)
+    end
+
+    def timer_set(on)
+      on ? timer_on : timer_off
+    end
+
+    def timer_on
+      @timer.execute
+    end
+
+    def timer_off
+      @timer.shutdown
+    end
+
+    # In case an exception is raised during processing, instruct concurrent-ruby
+    # to keep going without losing state
+    def behaviour_definition
+      Concurrent::Actor::Behaviour.restarting_behaviour_definition(:resume!)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/mqtt.rb

@@ -0,0 +1,23 @@
+module Proxy::RemoteExecution::Ssh
+  class MQTT
+    require 'smart_proxy_remote_execution_ssh/mqtt/dispatcher'
+
+    class << self
+      def publish(topic, payload, retain: false, qos: 1)
+        with_mqtt_client do |c|
+          c.publish(topic, payload, retain, qos)
+        end
+      end
+
+      def with_mqtt_client(&block)
+        ::MQTT::Client.connect(Plugin.settings.mqtt_broker,
+                               Plugin.settings.mqtt_port,
+                               :ssl => Plugin.settings.mqtt_tls,
+                               :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
+                               :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
+                               :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
+                               &block)
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb

@@ -154,6 +154,8 @@ module Proxy::RemoteExecution::Ssh::Runners
       ssh_options << "-o ControlPath=#{socket_file}"
       ssh_options << "-o ControlPersist=yes"
       ssh_options << "-o ProxyCommand=none"
+      ssh_options << "-o ServerAliveInterval=15"
+      ssh_options << "-o ServerAliveCountMax=3" # This is the default, but let's be explicit
       @establish_ssh_options = ssh_options
     end
 

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -25,8 +25,10 @@ module Proxy::RemoteExecution::Ssh
                      # :mqtt_broker             => nil,
                      # :mqtt_port               => nil,
                      # :mqtt_tls                => nil,
+                     # :mqtt_rate_limit         => nil
                      :mode                    => :ssh,
-                     :mqtt_resend_interval    => 900
+                     :mqtt_resend_interval    => 900,
+                     :mqtt_ttl                => 5
 
     capability(proc { 'cockpit' if settings.cockpit_integration })
 
@@ -46,6 +48,11 @@ module Proxy::RemoteExecution::Ssh
       Proxy::RemoteExecution::Ssh.validate!
 
       Proxy::Dynflow::TaskLauncherRegistry.register('ssh', Proxy::Dynflow::TaskLauncher::Batch)
+      if settings.mode == :'pull-mqtt'
+        require 'smart_proxy_remote_execution_ssh/mqtt'
+        # Force initialization
+        Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance
+      end
     end
 
     def self.simulate?

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.9.0'
+      VERSION = '0.10.1'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 2023-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -175,6 +175,8 @@ files:
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
 - lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
 - lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb