smart_proxy_dynflow 0.4.0 → 0.6.0

data/lib/smart_proxy_dynflow/runner/command_runner.rb

@@ -0,0 +1,11 @@
+require 'io/wait'
+require 'pty'
+require 'smart_proxy_dynflow/runner/command'
+
+module Proxy::Dynflow
+  module Runner
+    class CommandRunner < Base
+      include Command
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/dispatcher.rb

@@ -0,0 +1,191 @@
+require 'smart_proxy_dynflow/ticker'
+
+module Proxy::Dynflow
+  module Runner
+    class Dispatcher
+      def self.instance
+        return @instance if @instance
+        @instance = new(Proxy::Dynflow::Core.world.clock,
+                        Proxy::Dynflow::Core.world.logger)
+      end
+
+      class RunnerActor < ::Dynflow::Actor
+        def initialize(dispatcher, suspended_action, runner, clock, logger, _options = {})
+          @dispatcher = dispatcher
+          @clock = clock
+          @ticker = dispatcher.ticker
+          @logger = logger
+          @suspended_action = suspended_action
+          @runner = runner
+          @finishing = false
+        end
+
+        def on_envelope(*args)
+          super
+        rescue => e
+          handle_exception(e)
+        end
+
+        def start_runner
+          @logger.debug("start runner #{@runner.id}")
+          set_timeout if @runner.timeout_interval
+          @runner.start
+          refresh_runner
+        ensure
+          plan_next_refresh
+        end
+
+        def refresh_runner
+          @logger.debug("refresh runner #{@runner.id}")
+          dispatch_updates(@runner.run_refresh)
+        ensure
+          @refresh_planned = false
+          plan_next_refresh
+        end
+
+        def dispatch_updates(updates)
+          updates.each { |receiver, update| (receiver || @suspended_action) << update }
+
+          # This is a workaround when the runner does not accept the suspended action
+          main_key = updates.keys.any?(&:nil?) ? nil : @suspended_action
+          main_process = updates[main_key]
+          finish if main_process&.exit_status
+        end
+
+        def timeout_runner
+          @logger.debug("timeout runner #{@runner.id}")
+          @runner.timeout
+        rescue => e
+          handle_exception(e, false)
+        end
+
+        def kill
+          @logger.debug("kill runner #{@runner.id}")
+          @runner.kill
+        rescue => e
+          handle_exception(e, false)
+        end
+
+        def finish
+          @logger.debug("finish runner #{@runner.id}")
+          @finishing = true
+          @dispatcher.finish(@runner.id)
+        end
+
+        def start_termination(*args)
+          @logger.debug("terminate #{@runner.id}")
+          super
+          @runner.close
+          finish_termination
+        end
+
+        def external_event(event)
+          dispatch_updates(@runner.external_event(event))
+        end
+
+        private
+
+        def set_timeout
+          timeout_time = Time.now.getlocal + @runner.timeout_interval
+          @logger.debug("setting timeout for #{@runner.id} to #{timeout_time}")
+          @clock.ping(reference, timeout_time, :timeout_runner)
+        end
+
+        def plan_next_refresh
+          if !@finishing && !@refresh_planned
+            @logger.debug("planning to refresh #{@runner.id}")
+            @ticker.tell([:add_event, reference, :refresh_runner])
+            @refresh_planned = true
+          end
+        end
+
+        def handle_exception(exception, fatal = true)
+          @dispatcher.handle_command_exception(@runner.id, exception, fatal)
+        end
+      end
+
+      attr_reader :ticker
+      def initialize(clock, logger)
+        @mutex  = Mutex.new
+        @clock  = clock
+        @logger = logger
+        @ticker = ::Proxy::Dynflow::Ticker.spawn('dispatcher-ticker', @clock, @logger, refresh_interval)
+        @runner_actors = {}
+        @runner_suspended_actions = {}
+      end
+
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      def start(suspended_action, runner)
+        synchronize do
+          raise "Actor with runner id #{runner.id} already exists" if @runner_actors[runner.id]
+          runner.logger = @logger
+          runner_actor = RunnerActor.spawn("runner-actor-#{runner.id}", self, suspended_action, runner, @clock, @logger)
+          @runner_actors[runner.id] = runner_actor
+          @runner_suspended_actions[runner.id] = suspended_action
+          runner_actor.tell(:start_runner)
+          return runner.id
+        rescue => exception
+          _handle_command_exception(runner.id, exception)
+          return nil
+        end
+      end
+
+      def kill(runner_id)
+        synchronize do
+          runner_actor = @runner_actors[runner_id]
+          runner_actor&.tell(:kill)
+        rescue => exception
+          _handle_command_exception(runner_id, exception, false)
+        end
+      end
+
+      def finish(runner_id)
+        synchronize do
+          _finish(runner_id)
+        rescue => exception
+          _handle_command_exception(runner_id, exception, false)
+        end
+      end
+
+      def external_event(runner_id, external_event)
+        synchronize do
+          runner_actor = @runner_actors[runner_id]
+          runner_actor&.tell([:external_event, external_event])
+        end
+      end
+
+      def handle_command_exception(*args)
+        synchronize { _handle_command_exception(*args) }
+      end
+
+      def refresh_interval
+        1
+      end
+
+      private
+
+      def _finish(runner_id)
+        runner_actor = @runner_actors.delete(runner_id)
+        return unless runner_actor
+        @logger.debug("closing session for command [#{runner_id}]," \
+                      "#{@runner_actors.size} actors left ")
+        runner_actor.tell([:start_termination, Concurrent::Promises.resolvable_future])
+      ensure
+        @runner_suspended_actions.delete(runner_id)
+      end
+
+      def _handle_command_exception(runner_id, exception, fatal = true)
+        @logger.error("error while dispatching request to runner #{runner_id}:"\
+                      "#{exception.class} #{exception.message}:\n #{exception.backtrace.join("\n")}")
+        suspended_action = @runner_suspended_actions[runner_id]
+        if suspended_action
+          suspended_action << Runner::Update.encode_exception('Runner error', exception, fatal)
+        end
+        _finish(runner_id) if fatal
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/parent.rb

@@ -0,0 +1,57 @@
+module Proxy::Dynflow
+  module Runner
+    class Parent < Base
+      # targets = { identifier => { :execution_plan_id => "...", :run_step_id => id,
+      #                           :input => { ... } }
+      def initialize(targets = {}, suspended_action: nil)
+        @targets = targets
+        @exit_statuses = {}
+        super suspended_action: suspended_action
+      end
+
+      def generate_updates
+        base = {}
+        base[@suspended_action] = Runner::Update.new(Proxy::Dynflow::ContinuousOutput.new, @exit_status) if @exit_status
+        # Operate on all hosts if the main process ended or only on hosts for which we have updates
+        @outputs.reject { |_, output| @exit_status.nil? && output.empty? }
+                .reduce(base) do |acc, (identifier, output)|
+                  @outputs[identifier] = Proxy::Dynflow::ContinuousOutput.new # Create a new ContinuousOutput for next round of updates
+                  exit_status = @exit_statuses[identifier] || @exit_status if @exit_status
+                  acc.merge(host_action(identifier) => Runner::Update.new(output, exit_status))
+                end
+      end
+
+      def initialize_continuous_outputs
+        @outputs = @targets.keys.reduce({}) do |acc, target|
+          acc.merge(target => Proxy::Dynflow::ContinuousOutput.new)
+        end
+      end
+
+      def host_action(identifier)
+        options = @targets[identifier].slice('execution_plan_id', 'run_step_id')
+                                      .merge(:world => Proxy::Dynflow::Core.world)
+        Dynflow::Action::Suspended.new OpenStruct.new(options)
+      end
+
+      def broadcast_data(data, type)
+        @outputs.each_value { |output| output.add_output(data, type) }
+      end
+
+      def publish_data(_data, _type)
+        true
+      end
+
+      def publish_data_for(identifier, data, type)
+        @outputs[identifier].add_output(data, type)
+      end
+
+      def dispatch_exception(context, exception)
+        @outputs.each_value { |output| output.add_exception(context, exception) }
+      end
+
+      def publish_exit_status_for(identifier, exit_status)
+        @exit_statuses[identifier] = exit_status
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/update.rb

@@ -0,0 +1,28 @@
+require 'smart_proxy_dynflow/continuous_output'
+
+module Proxy::Dynflow
+  module Runner
+    # Runner::Update represents chunk of data produced by runner that
+    # can be consumed by other components, such as RunnerAction
+    class Update
+      attr_reader :continuous_output, :exit_status
+      def initialize(continuous_output, exit_status)
+        @continuous_output = continuous_output
+        @exit_status = exit_status
+      end
+
+      def self.encode_exception(context, exception, fatal = true)
+        continuous_output = ::Proxy::Dynflow::ContinuousOutput.new
+        continuous_output.add_exception(context, exception)
+        new(continuous_output, fatal ? 'EXCEPTION' : nil)
+      end
+    end
+
+    class ExternalEvent
+      attr_reader :data
+      def initialize(data = {})
+        @data = data
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner.rb

@@ -0,0 +1,10 @@
+module Proxy::Dynflow
+  module Runner
+  end
+end
+
+require 'smart_proxy_dynflow/runner/update'
+require 'smart_proxy_dynflow/runner/base'
+require 'smart_proxy_dynflow/runner/dispatcher'
+require 'smart_proxy_dynflow/action/runner'
+require 'smart_proxy_dynflow/runner/parent'

data/lib/smart_proxy_dynflow/settings.rb

@@ -1,6 +1,6 @@
 # require 'ostruct'
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   class Settings
     def self.instance
       Proxy::Dynflow::Plugin.settings

data/lib/smart_proxy_dynflow/settings_loader.rb

@@ -0,0 +1,53 @@
+module Proxy::Dynflow
+  module SettingsLoader
+    def self.settings_registry
+      @settings_registry ||= {}
+    end
+
+    def self.name_to_settings
+      @name_to_settings ||= {}
+    end
+
+    def self.settings_keys
+      @settings_keys ||= []
+    end
+
+    def self.settings_registered?(name)
+      name_to_settings.key?(name)
+    end
+
+    def self.register_settings(names, object)
+      names = [names] unless names.is_a? Array
+      names.each do |name|
+        raise 'settings name has to be a symbol' unless name.is_a? Symbol
+        raise "settings #{name} already registered" if SettingsLoader.settings_registered?(name)
+        name_to_settings[name] = object
+      end
+      settings_registry[names] = object
+    end
+
+    def self.setup_settings(name, settings)
+      raise "Settings for #{name} were not registered" unless settings_registered?(name)
+      name_to_settings[name].initialize_settings(settings)
+    end
+
+    def register_settings(names, defaults = {})
+      SettingsLoader.register_settings(names, self)
+      @defaults = defaults
+    end
+
+    def initialize_settings(settings = {})
+      @settings = @defaults.merge(settings)
+      validate_settings!
+    end
+
+    def settings
+      raise "Settings for #{self} not initalized" unless @settings
+      @settings
+    end
+
+    def validate_settings!
+      raise 'Only symbols expected in keys' unless @settings.keys.all? { |key| key.is_a? Symbol }
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher/abstract.rb

@@ -0,0 +1,44 @@
+module Proxy::Dynflow
+  module TaskLauncher
+    class Abstract
+      attr_reader :callback, :options, :results, :world
+      def initialize(world, callback, options = {})
+        @world = world
+        @callback = callback
+        @options = options
+        @results = {}
+      end
+
+      def launch!(_input)
+        raise NotImplementedError
+      end
+
+      def self.input_format; end
+
+      private
+
+      def format_result(result)
+        if result.triggered?
+          { :result => 'success', :task_id => result.execution_plan_id }
+        else
+          plan = world.persistence.load_execution_plan(result.id)
+          { :result => 'error', :errors => plan.errors }
+        end
+      end
+
+      def action_class(input)
+        options[:action_class_override] || ::Dynflow::Utils.constantize(input['action_class'])
+      end
+
+      def with_callback(input)
+        input.merge(:callback_host => callback)
+      end
+
+      def trigger(parent, klass, *input)
+        world.trigger do
+          world.plan_with_options(caller_action: parent, action_class: klass, args: input)
+        end
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher/batch.rb

@@ -0,0 +1,37 @@
+module Proxy::Dynflow
+  module TaskLauncher
+    class Batch < Abstract
+      def launch!(input)
+        trigger(nil, Proxy::Dynflow::Action::Batch, self, input)
+      end
+
+      def launch_children(parent, input_hash)
+        input_hash.each do |task_id, input|
+          launcher = child_launcher(parent)
+          launcher.launch!(transform_input(input))
+          results[task_id] = launcher.results
+        end
+      end
+
+      def prepare_batch(input_hash)
+        success_tasks = input_hash.select do |task_id, _input|
+          results[task_id][:result] == 'success'
+        end
+        success_tasks.reduce({}) do |acc, (key, value)|
+          acc.merge(results[key][:task_id] => value['action_input']['callback'])
+        end
+      end
+
+      private
+
+      def child_launcher(parent)
+        Single.new(world, callback, :parent => parent)
+      end
+
+      # Identity by default
+      def transform_input(input)
+        input
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher/group.rb

@@ -0,0 +1,48 @@
+require 'smart_proxy_dynflow/runner'
+
+module Proxy::Dynflow
+  module TaskLauncher
+    class AbstractGroup < Batch
+      def self.runner_class
+        raise NotImplementedError
+      end
+
+      def launch!(input)
+        trigger(nil, Action::SingleRunnerBatch, self, input)
+      end
+
+      def launch_children(parent, input_hash)
+        super(parent, input_hash)
+        trigger(parent, Action::BatchRunner, self, input_hash)
+      end
+
+      def operation
+        raise NotImplementedError
+      end
+
+      def runner_input(input)
+        input.reduce({}) do |acc, (id, input)|
+          input = { :execution_plan_id => results[id][:task_id],
+                    :run_step_id => 2,
+                    :input => input }
+          acc.merge(id => input)
+        end
+      end
+
+      private
+
+      def child_launcher(parent)
+        Single.new(world, callback, :parent => parent, :action_class_override => Action::OutputCollector)
+      end
+
+      def transform_input(input)
+        wipe_callback(input)
+      end
+
+      def wipe_callback(input)
+        callback = input['action_input']['callback']
+        input.merge('action_input' => input['action_input'].merge('callback' => nil, :task_id => callback['task_id']))
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher/single.rb

@@ -0,0 +1,17 @@
+module Proxy::Dynflow
+  module TaskLauncher
+    class Single < Abstract
+      def self.input_format
+        { :action_class => "MyActionClass", :action_input => {} }
+      end
+
+      def launch!(input)
+        triggered = trigger(options[:parent],
+                            action_class(input),
+                            with_callback(input.fetch('action_input', {})))
+        @results = format_result(triggered)
+        triggered
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher.rb

@@ -0,0 +1,9 @@
+module Proxy::Dynflow
+  module TaskLauncher
+  end
+end
+
+require 'smart_proxy_dynflow/task_launcher/abstract'
+require 'smart_proxy_dynflow/task_launcher/single'
+require 'smart_proxy_dynflow/task_launcher/batch'
+require 'smart_proxy_dynflow/task_launcher/group'

data/lib/smart_proxy_dynflow/task_launcher_registry.rb

@@ -1,4 +1,4 @@
-class Proxy::Dynflow
+module Proxy::Dynflow
   class TaskLauncherRegistry
     class << self
       def register(name, launcher)

data/lib/smart_proxy_dynflow/testing.rb

@@ -4,7 +4,7 @@ unless defined? DYNFLOW_TESTING_LOG_LEVEL
   DYNFLOW_TESTING_LOG_LEVEL = 4
 end
 
-class Proxy::Dynflow
+module Proxy::Dynflow
   # Helper for usage in other dependent plugins that need Dynflow
   # related things, such as testing instance of world etc.
   module Testing

data/lib/smart_proxy_dynflow/ticker.rb

@@ -0,0 +1,47 @@
+require 'dynflow'
+
+module Proxy::Dynflow
+  class Ticker < ::Dynflow::Actor
+    attr_reader :clock
+
+    def initialize(clock, logger, refresh_interval)
+      @clock = clock
+      @logger = logger
+      @events = []
+      @refresh_interval = refresh_interval
+      plan_next_tick
+    end
+
+    def tick
+      @logger.debug("Ticker ticking for #{@events.size} events")
+      @events.each do |(target, args)|
+        pass_event(target, args)
+      end
+      @events = []
+    ensure
+      @planned = false
+      plan_next_tick
+    end
+
+    def add_event(target, args)
+      @events << [target, args]
+      plan_next_tick
+    end
+
+    private
+
+    def pass_event(target, args)
+      target.tell(args)
+    rescue => e
+      @logger.error("Failed passing event to #{target} with #{args}")
+      @logger.error(e)
+    end
+
+    def plan_next_tick
+      if !@planned && !@events.empty?
+        @clock.ping(reference, Time.now.getlocal + @refresh_interval, :tick)
+        @planned = true
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/version.rb

@@ -1,5 +1,5 @@
 module Proxy
-  class Dynflow
-    VERSION = '0.4.0'.freeze
+  module Dynflow
+    VERSION = '0.6.0'.freeze
   end
 end

data/lib/smart_proxy_dynflow.rb

@@ -3,10 +3,9 @@ require 'dynflow'
 require 'smart_proxy_dynflow/task_launcher_registry'
 require 'smart_proxy_dynflow/middleware/keep_current_request_id'
 
-require 'foreman_tasks_core'
-
 require 'smart_proxy_dynflow/log'
 require 'smart_proxy_dynflow/settings'
+require 'smart_proxy_dynflow/ticker'
 require 'smart_proxy_dynflow/core'
 require 'smart_proxy_dynflow/callback'
 
@@ -16,10 +15,6 @@ require 'smart_proxy_dynflow/helpers'
 require 'smart_proxy_dynflow/api'
 
 module Proxy
-  class Dynflow
-    Core.after_initialize do |dynflow_core|
-      ForemanTasksCore.dynflow_setup(dynflow_core.world)
-    end
-    Core.register_silencer_matchers ForemanTasksCore.silent_dead_letter_matchers
+  module Dynflow
   end
 end