smart_proxy_remote_execution_ssh 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f28e2eb48888626dad82eb56e7a5309da15d584e6d4c53eaf648e5ddfc6f3d1e
-  data.tar.gz: a5b604d80264dc78b461e1792100ecd6fc9a4a71791b8df5b064bd80a32f6920
+  metadata.gz: '048f8694430967060fae6bd7790fe56930b5c2bac37f3bc26e7b13db69081da6'
+  data.tar.gz: 5a3b1e344a47aa4c97dcce97f43dc9e424d398f79c67a6f1e066d670758c99f0
 SHA512:
-  metadata.gz: 3231a3d465b180d1b8b325fd9fc341c009b566452946ee56837f9f37e6502d7095d1a213bc5b4b13078b9d54e5bd592f5430aea502ecdee93edab15901440815
-  data.tar.gz: 62e77d3cf6d37c90b2e6e8e48ad8b2080186a50fd1d3633b38743a099e0897b9b1e4d4dafaf7b02b1855dcd51036d418333c156438c3b20e5fa42feac93ded17
+  metadata.gz: a4bc90b3dda6d190c8f94aeb6731a3a6679788ea4f950e1bb4196c1d5da4630b170fe152e9c96a5e4cb7dcd25ba1d6371292e52b4059773759c1fbe8461bd347
+  data.tar.gz: f4849b32f7529cbb29d127ae1c4c9fd9642b0700c834b1e3eebf59b5d9cfca282fca674188a734a37b36d45ff3e423096aa3512ceae118aef7ff831cb1600a38

data/lib/smart_proxy_remote_execution_ssh/actions/pull_script.rb

@@ -6,7 +6,6 @@ module Proxy::RemoteExecution::Ssh::Actions
   class PullScript < Proxy::Dynflow::Action::Runner
     JobDelivered = Class.new
     PickupTimeout = Class.new
-    ResendNotification = Class.new
 
     execution_plan_hooks.use :cleanup, :on => :stopped
 
@@ -21,12 +20,6 @@ module Proxy::RemoteExecution::Ssh::Actions
         suspend
       elsif event == PickupTimeout
         process_pickup_timeout
-      elsif event == ResendNotification
-        if input[:with_mqtt] && %w(ready_for_pickup notified).include?(output[:state])
-          schedule_mqtt_resend
-          mqtt_start(::Proxy::Dynflow::OtpManager.passwords[execution_plan_id])
-        end
-        suspend
       else
         super
       end
@@ -45,16 +38,15 @@ module Proxy::RemoteExecution::Ssh::Actions
       input[:job_uuid] = job_storage.store_job(host_name, execution_plan_id, run_step_id, input[:script].tr("\r", ''), effective_user: input[:effective_user])
       output[:state] = :ready_for_pickup
       output[:result] = []
-      if input[:with_mqtt]
-        schedule_mqtt_resend
-        mqtt_start(otp_password)
-      end
+
+      mqtt_start(otp_password) if input[:with_mqtt]
       suspend
     end
 
     def cleanup(_plan = nil)
       job_storage.drop_job(execution_plan_id, run_step_id)
       Proxy::Dynflow::OtpManager.passwords.delete(execution_plan_id)
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.done(input[:job_uuid])
     end
 
     def process_external_event(event)
@@ -125,7 +117,7 @@ module Proxy::RemoteExecution::Ssh::Actions
           'effective_user': input[:effective_user]
         },
       )
-      mqtt_notify payload
+      Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.new(input[:job_uuid], mqtt_topic, payload)
       output[:state] = :notified
     end
 
@@ -141,18 +133,7 @@ module Proxy::RemoteExecution::Ssh::Actions
     end
 
     def mqtt_notify(payload)
-      with_mqtt_client do |c|
-        c.publish(mqtt_topic, JSON.dump(payload), false, 1)
-      end
-    end
-
-    def with_mqtt_client(&block)
-      MQTT::Client.connect(settings.mqtt_broker, settings.mqtt_port,
-                           :ssl => settings.mqtt_tls,
-                           :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
-                           :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
-                           :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
-                           &block)
+      Proxy::RemoteExecution::Ssh::MQTT.publish(mqtt_topic, JSON.dump(payload))
     end
 
     def host_name
@@ -167,10 +148,6 @@ module Proxy::RemoteExecution::Ssh::Actions
       "yggdrasil/#{host_name}/data/in"
     end
 
-    def settings
-      Proxy::RemoteExecution::Ssh::Plugin.settings
-    end
-
     def job_storage
       Proxy::RemoteExecution::Ssh.job_storage
     end
@@ -192,9 +169,5 @@ module Proxy::RemoteExecution::Ssh::Actions
         suspend
       end
     end
-
-    def schedule_mqtt_resend
-      plan_event(ResendNotification, settings[:mqtt_resend_interval], optional: true)
-    end
   end
 end

data/lib/smart_proxy_remote_execution_ssh/api.rb

@@ -64,6 +64,7 @@ module Proxy::RemoteExecution
         do_authorize_with_ssl_client
 
         with_authorized_job(job_uuid) do |job_record|
+          Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance.running(job_record[:uuid])
           notify_job(job_record, Actions::PullScript::JobDelivered)
           response.headers['X-Foreman-Effective-User'] = job_record[:effective_user]
           job_record[:job]

data/lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb

@@ -0,0 +1,169 @@
+require 'concurrent'
+require 'mqtt'
+
+class Proxy::RemoteExecution::Ssh::MQTT
+  class Dispatcher
+    include Singleton
+
+    attr_reader :reference
+    def initialize
+      limit = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_rate_limit]
+      @reference = DispatcherActor.spawn('MQTT dispatcher',
+                                         Proxy::Dynflow::Core.world.clock,
+                                         limit)
+    end
+
+    def new(uuid, topic, payload)
+      reference.tell([:new, uuid, topic, payload])
+    end
+
+    def running(uuid)
+      reference.tell([:running, uuid])
+    end
+
+    def resend(uuid)
+      reference.tell([:resend, uuid])
+    end
+
+    def done(uuid)
+      reference.tell([:done, uuid])
+    end
+  end
+
+  class DispatcherActor < Concurrent::Actor::RestartingContext
+    JobDefinition = Struct.new :uuid, :topic, :payload
+
+    class Tracker
+      def initialize(limit, clock)
+        @clock = clock
+        @limit = limit
+        @jobs = {}
+        @pending = []
+        @running = Set.new
+        @hot = Set.new
+        @cold = Set.new
+      end
+
+      def new(uuid, topic, payload)
+        @jobs[uuid] = JobDefinition.new(uuid, topic, payload)
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def running(uuid)
+        [@pending, @hot, @cold].each { |source| source.delete(uuid) }
+        @running << uuid
+      end
+
+      def resend(uuid)
+        return unless @jobs[uuid]
+
+        @pending << uuid
+        dispatch_pending
+      end
+
+      def done(uuid)
+        @jobs.delete(uuid)
+        [@pending, @running, @hot, @cold].each do |source|
+          source.delete(uuid)
+        end
+        dispatch_pending
+      end
+
+      def needs_processing?
+        pending_count.positive? || @hot.any? || @cold.any?
+      end
+
+      def pending_count
+        pending = @pending.count
+        return pending if @limit.nil?
+
+        running = [@running, @hot, @cold].map(&:count).sum
+        capacity = @limit - running
+        pending > capacity ? capacity : pending
+      end
+
+      def dispatch_pending
+        pending_count.times do
+          mqtt_notify(@pending.first)
+          @hot << @pending.shift
+        end
+      end
+
+      def process
+        @cold.each { |uuid| schedule_resend(uuid) }
+        @cold = @hot
+        @hot = Set.new
+
+        dispatch_pending
+      end
+
+      def mqtt_notify(uuid)
+        job = @jobs[uuid]
+        return if job.nil?
+
+        Proxy::RemoteExecution::Ssh::MQTT.publish(job.topic, JSON.dump(job.payload))
+      end
+
+      def settings
+        Proxy::RemoteExecution::Ssh::Plugin.settings
+      end
+
+      def schedule_resend(uuid)
+        @clock.ping(Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance, resend_interval, uuid, :resend)
+      end
+
+      def resend_interval
+        settings[:mqtt_resend_interval]
+      end
+    end
+
+    def initialize(clock, limit = nil)
+      @tracker = Tracker.new(limit, clock)
+
+      interval = Proxy::RemoteExecution::Ssh::Plugin.settings[:mqtt_ttl]
+      @timer = Concurrent::TimerTask.new(execution_interval: interval) do
+        reference.tell(:tick)
+      end
+    end
+
+    def on_message(message)
+      action, arg = message
+      # Enable the timer just in case anything in tracker raises an exception so
+      # we can continue
+      timer_on
+      case action
+      when :new
+        _, uuid, topic, payload = message
+        @tracker.new(uuid, topic, payload)
+      when :resend
+        @tracker.resend(arg)
+      when :running
+        @tracker.running(arg)
+      when :done
+        @tracker.done(arg)
+      when :tick
+        @tracker.process
+      end
+      timer_set(@tracker.needs_processing?)
+    end
+
+    def timer_set(on)
+      on ? timer_on : timer_off
+    end
+
+    def timer_on
+      @timer.execute
+    end
+
+    def timer_off
+      @timer.shutdown
+    end
+
+    # In case an exception is raised during processing, instruct concurrent-ruby
+    # to keep going without losing state
+    def behaviour_definition
+      Concurrent::Actor::Behaviour.restarting_behaviour_definition(:resume!)
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/mqtt.rb

@@ -0,0 +1,23 @@
+module Proxy::RemoteExecution::Ssh
+  class MQTT
+    require 'smart_proxy_remote_execution_ssh/mqtt/dispatcher'
+
+    class << self
+      def publish(topic, payload, retain: false, qos: 1)
+        with_mqtt_client do |c|
+          c.publish(topic, payload, retain, qos)
+        end
+      end
+
+      def with_mqtt_client(&block)
+        ::MQTT::Client.connect(Plugin.settings.mqtt_broker,
+                               Plugin.settings.mqtt_port,
+                               :ssl => Plugin.settings.mqtt_tls,
+                               :cert_file => ::Proxy::SETTINGS.foreman_ssl_cert || ::Proxy::SETTINGS.ssl_certificate,
+                               :key_file => ::Proxy::SETTINGS.foreman_ssl_key || ::Proxy::SETTINGS.ssl_private_key,
+                               :ca_file => ::Proxy::SETTINGS.foreman_ssl_ca || ::Proxy::SETTINGS.ssl_ca_file,
+                               &block)
+      end
+    end
+  end
+end

data/lib/smart_proxy_remote_execution_ssh/plugin.rb

@@ -25,8 +25,10 @@ module Proxy::RemoteExecution::Ssh
                      # :mqtt_broker             => nil,
                      # :mqtt_port               => nil,
                      # :mqtt_tls                => nil,
+                     # :mqtt_rate_limit         => nil
                      :mode                    => :ssh,
-                     :mqtt_resend_interval    => 900
+                     :mqtt_resend_interval    => 900,
+                     :mqtt_ttl                => 5
 
     capability(proc { 'cockpit' if settings.cockpit_integration })
 
@@ -46,6 +48,11 @@ module Proxy::RemoteExecution::Ssh
       Proxy::RemoteExecution::Ssh.validate!
 
       Proxy::Dynflow::TaskLauncherRegistry.register('ssh', Proxy::Dynflow::TaskLauncher::Batch)
+      if settings.mode == :'pull-mqtt'
+        require 'smart_proxy_remote_execution_ssh/mqtt'
+        # Force initialization
+        Proxy::RemoteExecution::Ssh::MQTT::Dispatcher.instance
+      end
     end
 
     def self.simulate?

data/lib/smart_proxy_remote_execution_ssh/version.rb

@@ -1,7 +1,7 @@
 module Proxy
   module RemoteExecution
     module Ssh
-      VERSION = '0.9.0'
+      VERSION = '0.10.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_remote_execution_ssh
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 2022-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -175,6 +175,8 @@ files:
 - lib/smart_proxy_remote_execution_ssh/http_config.ru
 - lib/smart_proxy_remote_execution_ssh/job_storage.rb
 - lib/smart_proxy_remote_execution_ssh/log_filter.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt.rb
+- lib/smart_proxy_remote_execution_ssh/mqtt/dispatcher.rb
 - lib/smart_proxy_remote_execution_ssh/multiplexed_ssh_connection.rb
 - lib/smart_proxy_remote_execution_ssh/net_ssh_compat.rb
 - lib/smart_proxy_remote_execution_ssh/plugin.rb