smart_proxy_dynflow 0.2.4 → 0.5.2

data/lib/smart_proxy_dynflow/proxy_adapter.rb

@@ -1,5 +1,5 @@
 module Proxy
-  class Dynflow
+  module Dynflow
     class ProxyAdapter < ::Dynflow::LoggerAdapters::Simple
       def initialize(logger, level = Logger::DEBUG, formatters = [Formatters::Exception])
         super(nil, level, formatters)

data/lib/smart_proxy_dynflow/runner.rb

@@ -0,0 +1,10 @@
+module Proxy::Dynflow
+  module Runner
+  end
+end
+
+require 'smart_proxy_dynflow/runner/update'
+require 'smart_proxy_dynflow/runner/base'
+require 'smart_proxy_dynflow/runner/dispatcher'
+require 'smart_proxy_dynflow/action/runner'
+require 'smart_proxy_dynflow/runner/parent'

data/lib/smart_proxy_dynflow/runner/base.rb

@@ -0,0 +1,98 @@
+module Proxy::Dynflow
+  module Runner
+    # Runner is an object that is able to initiate some action and
+    # provide update data on refresh call.
+    class Base
+      attr_reader :id
+      attr_writer :logger
+
+      def initialize(*_args, suspended_action: nil)
+        @suspended_action = suspended_action
+        @id = SecureRandom.uuid
+        initialize_continuous_outputs
+      end
+
+      def logger
+        @logger ||= Logger.new(STDERR)
+      end
+
+      def run_refresh
+        logger.debug('refreshing runner')
+        refresh
+        generate_updates
+      end
+
+      # by default, external event just causes the refresh to be triggered: this allows the descendants
+      # of the Base to add custom logic to process the external events.
+      # Similarly as `run_refresh`, it's expected to return updates to be dispatched.
+      def external_event(_event)
+        run_refresh
+      end
+
+      def start
+        raise NotImplementedError
+      end
+
+      def refresh
+        raise NotImplementedError
+      end
+
+      def kill
+        # Override when you can kill the runner in the middle
+      end
+
+      def close
+        # if cleanup is needed
+      end
+
+      def timeout
+        # Override when timeouts and regular kills should be handled differently
+        publish_data('Timeout for execution passed, trying to stop the job', 'debug')
+        kill
+      end
+
+      def timeout_interval
+        # A number of seconds after which the runner should receive a #timeout
+        #   or nil for no timeout
+      end
+
+      def publish_data(data, type)
+        @continuous_output.add_output(data, type)
+      end
+
+      def publish_exception(context, exception, fatal = true)
+        logger.error("#{context} - #{exception.class} #{exception.message}:\n" + \
+                     exception.backtrace.join("\n"))
+        dispatch_exception context, exception
+        publish_exit_status('EXCEPTION') if fatal
+      end
+
+      def publish_exit_status(status)
+        @exit_status = status
+      end
+
+      def dispatch_exception(context, exception)
+        @continuous_output.add_exception(context, exception)
+      end
+
+      def generate_updates
+        return no_update if @continuous_output.empty? && @exit_status.nil?
+        new_data = @continuous_output
+        @continuous_output = Proxy::Dynflow::ContinuousOutput.new
+        new_update(new_data, @exit_status)
+      end
+
+      def no_update
+        {}
+      end
+
+      def new_update(data, exit_status)
+        { @suspended_action => Runner::Update.new(data, exit_status) }
+      end
+
+      def initialize_continuous_outputs
+        @continuous_output = ::Proxy::Dynflow::ContinuousOutput.new
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/command.rb

@@ -0,0 +1,40 @@
+module Proxy::Dynflow
+  module Runner
+    module Command
+      def initialize_command(*command)
+        @command_out, @command_in, @command_pid = PTY.spawn(*command)
+      rescue Errno::ENOENT => e
+        publish_exception("Error running command '#{command.join(' ')}'", e)
+      end
+
+      def refresh
+        return if @command_out.nil?
+        ready_outputs, * = IO.select([@command_out], nil, nil, 0.1)
+        if ready_outputs
+          if @command_out.nread.positive?
+            lines = @command_out.read_nonblock(@command_out.nread)
+          else
+            close_io
+            Process.wait(@command_pid)
+            publish_exit_status($CHILD_STATUS.exitstatus)
+          end
+          publish_data(lines, 'stdout') if lines && !lines.empty?
+        end
+      end
+
+      def close
+        close_io
+      end
+
+      private
+
+      def close_io
+        @command_out.close if @command_out && !@command_out.closed?
+        @command_out = nil
+
+        @command_in.close if @command_in && !@command_in.closed?
+        @command_in = nil
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/command_runner.rb

@@ -0,0 +1,11 @@
+require 'io/wait'
+require 'pty'
+require 'smart_proxy_dynflow/runner/command'
+
+module Proxy::Dynflow
+  module Runner
+    class CommandRunner < Base
+      include Command
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/dispatcher.rb

@@ -0,0 +1,191 @@
+require 'smart_proxy_dynflow/ticker'
+
+module Proxy::Dynflow
+  module Runner
+    class Dispatcher
+      def self.instance
+        return @instance if @instance
+        @instance = new(Proxy::Dynflow::Core.world.clock,
+                        Proxy::Dynflow::Core.world.logger)
+      end
+
+      class RunnerActor < ::Dynflow::Actor
+        def initialize(dispatcher, suspended_action, runner, clock, logger, _options = {})
+          @dispatcher = dispatcher
+          @clock = clock
+          @ticker = dispatcher.ticker
+          @logger = logger
+          @suspended_action = suspended_action
+          @runner = runner
+          @finishing = false
+        end
+
+        def on_envelope(*args)
+          super
+        rescue => e
+          handle_exception(e)
+        end
+
+        def start_runner
+          @logger.debug("start runner #{@runner.id}")
+          set_timeout if @runner.timeout_interval
+          @runner.start
+          refresh_runner
+        ensure
+          plan_next_refresh
+        end
+
+        def refresh_runner
+          @logger.debug("refresh runner #{@runner.id}")
+          dispatch_updates(@runner.run_refresh)
+        ensure
+          @refresh_planned = false
+          plan_next_refresh
+        end
+
+        def dispatch_updates(updates)
+          updates.each { |receiver, update| (receiver || @suspended_action) << update }
+
+          # This is a workaround when the runner does not accept the suspended action
+          main_key = updates.keys.any?(&:nil?) ? nil : @suspended_action
+          main_process = updates[main_key]
+          finish if main_process&.exit_status
+        end
+
+        def timeout_runner
+          @logger.debug("timeout runner #{@runner.id}")
+          @runner.timeout
+        rescue => e
+          handle_exception(e, false)
+        end
+
+        def kill
+          @logger.debug("kill runner #{@runner.id}")
+          @runner.kill
+        rescue => e
+          handle_exception(e, false)
+        end
+
+        def finish
+          @logger.debug("finish runner #{@runner.id}")
+          @finishing = true
+          @dispatcher.finish(@runner.id)
+        end
+
+        def start_termination(*args)
+          @logger.debug("terminate #{@runner.id}")
+          super
+          @runner.close
+          finish_termination
+        end
+
+        def external_event(event)
+          dispatch_updates(@runner.external_event(event))
+        end
+
+        private
+
+        def set_timeout
+          timeout_time = Time.now.getlocal + @runner.timeout_interval
+          @logger.debug("setting timeout for #{@runner.id} to #{timeout_time}")
+          @clock.ping(reference, timeout_time, :timeout_runner)
+        end
+
+        def plan_next_refresh
+          if !@finishing && !@refresh_planned
+            @logger.debug("planning to refresh #{@runner.id}")
+            @ticker.tell([:add_event, reference, :refresh_runner])
+            @refresh_planned = true
+          end
+        end
+
+        def handle_exception(exception, fatal = true)
+          @dispatcher.handle_command_exception(@runner.id, exception, fatal)
+        end
+      end
+
+      attr_reader :ticker
+      def initialize(clock, logger)
+        @mutex  = Mutex.new
+        @clock  = clock
+        @logger = logger
+        @ticker = ::Proxy::Dynflow::Ticker.spawn('dispatcher-ticker', @clock, @logger, refresh_interval)
+        @runner_actors = {}
+        @runner_suspended_actions = {}
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def start(suspended_action, runner)
+        synchronize do
+          raise "Actor with runner id #{runner.id} already exists" if @runner_actors[runner.id]
+          runner.logger = @logger
+          runner_actor = RunnerActor.spawn("runner-actor-#{runner.id}", self, suspended_action, runner, @clock, @logger)
+          @runner_actors[runner.id] = runner_actor
+          @runner_suspended_actions[runner.id] = suspended_action
+          runner_actor.tell(:start_runner)
+          return runner.id
+        rescue => exception
+          _handle_command_exception(runner.id, exception)
+          return nil
+        end
+      end
+
+      def kill(runner_id)
+        synchronize do
+          runner_actor = @runner_actors[runner_id]
+          runner_actor&.tell(:kill)
+        rescue => exception
+          _handle_command_exception(runner_id, exception, false)
+        end
+      end
+
+      def finish(runner_id)
+        synchronize do
+          _finish(runner_id)
+        rescue => exception
+          _handle_command_exception(runner_id, exception, false)
+        end
+      end
+
+      def external_event(runner_id, external_event)
+        synchronize do
+          runner_actor = @runner_actors[runner_id]
+          runner_actor&.tell([:external_event, external_event])
+        end
+      end
+
+      def handle_command_exception(*args)
+        synchronize { _handle_command_exception(*args) }
+      end
+
+      def refresh_interval
+        1
+      end
+
+      private
+
+      def _finish(runner_id)
+        runner_actor = @runner_actors.delete(runner_id)
+        return unless runner_actor
+        @logger.debug("closing session for command [#{runner_id}]," \
+                      "#{@runner_actors.size} actors left ")
+        runner_actor.tell([:start_termination, Concurrent::Promises.resolvable_future])
+      ensure
+        @runner_suspended_actions.delete(runner_id)
+      end
+
+      def _handle_command_exception(runner_id, exception, fatal = true)
+        @logger.error("error while dispatching request to runner #{runner_id}:"\
+                      "#{exception.class} #{exception.message}:\n #{exception.backtrace.join("\n")}")
+        suspended_action = @runner_suspended_actions[runner_id]
+        if suspended_action
+          suspended_action << Runner::Update.encode_exception('Runner error', exception, fatal)
+        end
+        _finish(runner_id) if fatal
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/parent.rb

@@ -0,0 +1,57 @@
+module Proxy::Dynflow
+  module Runner
+    class Parent < Base
+      # targets = { identifier => { :execution_plan_id => "...", :run_step_id => id,
+      #                           :input => { ... } }
+      def initialize(targets = {}, suspended_action: nil)
+        @targets = targets
+        @exit_statuses = {}
+        super suspended_action: suspended_action
+      end
+
+      def generate_updates
+        base = {}
+        base[@suspended_action] = Runner::Update.new(Proxy::Dynflow::ContinuousOutput.new, @exit_status) if @exit_status
+        # Operate on all hosts if the main process ended or only on hosts for which we have updates
+        @outputs.reject { |_, output| @exit_status.nil? && output.empty? }
+                .reduce(base) do |acc, (identifier, output)|
+                  @outputs[identifier] = Proxy::Dynflow::ContinuousOutput.new # Create a new ContinuousOutput for next round of updates
+                  exit_status = @exit_statuses[identifier] || @exit_status if @exit_status
+                  acc.merge(host_action(identifier) => Runner::Update.new(output, exit_status))
+                end
+      end
+
+      def initialize_continuous_outputs
+        @outputs = @targets.keys.reduce({}) do |acc, target|
+          acc.merge(target => Proxy::Dynflow::ContinuousOutput.new)
+        end
+      end
+
+      def host_action(identifier)
+        options = @targets[identifier].slice('execution_plan_id', 'run_step_id')
+                                      .merge(:world => Proxy::Dynflow::Core.world)
+        Dynflow::Action::Suspended.new OpenStruct.new(options)
+      end
+
+      def broadcast_data(data, type)
+        @outputs.each_value { |output| output.add_output(data, type) }
+      end
+
+      def publish_data(_data, _type)
+        true
+      end
+
+      def publish_data_for(identifier, data, type)
+        @outputs[identifier].add_output(data, type)
+      end
+
+      def dispatch_exception(context, exception)
+        @outputs.each_value { |output| output.add_exception(context, exception) }
+      end
+
+      def publish_exit_status_for(identifier, exit_status)
+        @exit_statuses[identifier] = exit_status
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/update.rb

@@ -0,0 +1,28 @@
+require 'smart_proxy_dynflow/continuous_output'
+
+module Proxy::Dynflow
+  module Runner
+    # Runner::Update represents chunk of data produced by runner that
+    # can be consumed by other components, such as RunnerAction
+    class Update
+      attr_reader :continuous_output, :exit_status
+      def initialize(continuous_output, exit_status)
+        @continuous_output = continuous_output
+        @exit_status = exit_status
+      end
+
+      def self.encode_exception(context, exception, fatal = true)
+        continuous_output = ::Proxy::Dynflow::ContinuousOutput.new
+        continuous_output.add_exception(context, exception)
+        new(continuous_output, fatal ? 'EXCEPTION' : nil)
+      end
+    end
+
+    class ExternalEvent
+      attr_reader :data
+      def initialize(data = {})
+        @data = data
+      end
+    end
+  end
+end