dynflow 0.8.16 → 0.8.17

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -2,7 +2,7 @@ module Dynflow
   module Dispatcher
     class ExecutorDispatcher < Abstract
       def initialize(world, semaphore)
-        @world        = Type! world, World
+        @world           = Type! world, World
         @current_futures = Set.new
       end
 

data/lib/dynflow/execution_plan.rb

@@ -35,17 +35,17 @@ module Dynflow
 
     # all params with default values are part of *private* api
     def initialize(world,
-        id = SecureRandom.uuid,
-        state = :pending,
-        root_plan_step = nil,
-        run_flow = Flows::Concurrence.new([]),
-        finalize_flow = Flows::Sequence.new([]),
-        steps = {},
-        started_at = nil,
-        ended_at = nil,
-        execution_time = nil,
-        real_time = 0.0,
-        execution_history = ExecutionHistory.new)
+                   id                = SecureRandom.uuid,
+                   state             = :pending,
+                   root_plan_step    = nil,
+                   run_flow          = Flows::Concurrence.new([]),
+                   finalize_flow     = Flows::Sequence.new([]),
+                   steps             = {},
+                   started_at        = nil,
+                   ended_at          = nil,
+                   execution_time    = nil,
+                   real_time         = 0.0,
+                   execution_history = ExecutionHistory.new)
 
       @id                = Type! id, String
       @world             = Type! world, World
@@ -133,6 +133,18 @@ module Dynflow
       end
     end
 
+    def plan_steps
+      steps_of_type(Dynflow::ExecutionPlan::Steps::PlanStep)
+    end
+
+    def run_steps
+      steps_of_type(Dynflow::ExecutionPlan::Steps::RunStep)
+    end
+
+    def finalize_steps
+      steps_of_type(Dynflow::ExecutionPlan::Steps::FinalizeStep)
+    end
+
     def failed_steps
       steps_in_state(:error)
     end
@@ -165,10 +177,10 @@ module Dynflow
       execution_history.add("delay", @world.id)
       serializer = root_plan_step.delay(delay_options, args)
       delayed_plan = DelayedPlan.new(@world,
-                                         id,
-                                         delay_options[:start_at],
-                                         delay_options.fetch(:start_before, nil),
-                                         serializer)
+                                     id,
+                                     delay_options[:start_at],
+                                     delay_options.fetch(:start_before, nil),
+                                     serializer)
       persistence.save_delayed_plan(delayed_plan)
     ensure
       update_state(error? ? :stopped : :scheduled)
@@ -263,6 +275,11 @@ module Dynflow
     end
 
     # @api private
+
+    def steps_of_type(type)
+      steps.values.find_all { |step| step.is_a?(type) }
+    end
+
     def current_run_flow
       @run_flow_stack.last
     end

data/lib/dynflow/execution_plan/steps/abstract.rb

@@ -9,18 +9,18 @@ module Dynflow
       attr_accessor :error
 
       def initialize(execution_plan_id,
-          id,
-          state,
-          action_class,
-          action_id,
-          error,
-          world,
-          started_at = nil,
-          ended_at = nil,
-          execution_time = 0.0,
-          real_time = 0.0,
-          progress_done = nil,
-          progress_weight = nil)
+                     id,
+                     state,
+                     action_class,
+                     action_id,
+                     error,
+                     world,
+                     started_at      = nil,
+                     ended_at        = nil,
+                     execution_time  = 0.0,
+                     real_time       = 0.0,
+                     progress_done   = nil,
+                     progress_weight = nil)
 
         @id                = id || raise(ArgumentError, 'missing id')
         @execution_plan_id = Type! execution_plan_id, String
@@ -155,9 +155,9 @@ module Dynflow
         block.call
       ensure
         @progress_done, @progress_weight = action.calculated_progress
-        @ended_at       = Time.now
+        @ended_at        = Time.now
         @execution_time += @ended_at - start
-        @real_time      = @ended_at - @started_at
+        @real_time       = @ended_at - @started_at
       end
     end
   end

data/lib/dynflow/execution_plan/steps/error.rb

@@ -55,7 +55,7 @@ module Dynflow
 
       def exception
         @exception ||
-            exception_class.exception(message).tap { |e| e.set_backtrace backtrace }
+          exception_class.exception(message).tap { |e| e.set_backtrace backtrace }
       end
     end
   end

data/lib/dynflow/execution_plan/steps/finalize_step.rb

@@ -13,7 +13,6 @@ module Dynflow
         }
       end
 
-
       def phase
         Action::Finalize
       end

data/lib/dynflow/execution_plan/steps/plan_step.rb

@@ -5,17 +5,17 @@ module Dynflow
 
       # @param [Array] children is a private API parameter
       def initialize(execution_plan_id,
-          id,
-          state,
-          action_class,
-          action_id,
-          error,
-          world,
-          started_at = nil,
-          ended_at = nil,
-          execution_time = 0.0,
-          real_time = 0.0,
-          children = [])
+                     id,
+                     state,
+                     action_class,
+                     action_id,
+                     error,
+                     world,
+                     started_at     = nil,
+                     ended_at       = nil,
+                     execution_time = 0.0,
+                     real_time      = 0.0,
+                     children       = [])
 
         super execution_plan_id, id, state, action_class, action_id, error, world, started_at,
               ended_at, execution_time, real_time
@@ -68,7 +68,6 @@ module Dynflow
                                  error:      [] }
       end
 
-
       def self.new_from_hash(hash, execution_plan_id, world)
         check_class_matching hash
         new execution_plan_id,

data/lib/dynflow/execution_plan/steps/run_step.rb

@@ -20,7 +20,7 @@ module Dynflow
 
       def cancellable?
         [:suspended, :running].include?(self.state) &&
-            self.action_class < Action::Cancellable
+          self.action_class < Action::Cancellable
       end
 
       def with_sub_plans?

data/lib/dynflow/executors/abstract.rb

@@ -1,13 +1,6 @@
 module Dynflow
   module Executors
     class Abstract
-      Event = Algebrick.type do
-        fields! execution_plan_id: String,
-                step_id:           Fixnum,
-                event:             Object,
-                result:            Concurrent::Edge::Future
-      end
-
       include Algebrick::TypeCheck
       attr_reader :world, :logger
 
@@ -16,9 +9,13 @@ module Dynflow
         @logger = world.logger
       end
 
+      # @param execution_plan_id [String] id of execution plan
+      # @param finished [Concurrent::Edge::Future]
+      # @param wait_for_acceptance [TrueClass|FalseClass] should the executor confirm receiving
+      # the event, disable if calling executor from within executor
       # @return [Concurrent::Edge::Future]
       # @raise when execution_plan_id is not accepted
-      def execute(execution_plan_id)
+      def execute(execution_plan_id, finished = Concurrent.future, wait_for_acceptance = true)
         raise NotImplementedError
       end
 

data/lib/dynflow/executors/parallel.rb

@@ -1,41 +1,10 @@
 module Dynflow
   module Executors
     class Parallel < Abstract
-
-      require 'dynflow/executors/parallel/sequence_cursor'
-      require 'dynflow/executors/parallel/flow_manager'
-      require 'dynflow/executors/parallel/work_queue'
-      require 'dynflow/executors/parallel/execution_plan_manager'
-      require 'dynflow/executors/parallel/sequential_manager'
-      require 'dynflow/executors/parallel/running_steps_manager'
       require 'dynflow/executors/parallel/core'
       require 'dynflow/executors/parallel/pool'
       require 'dynflow/executors/parallel/worker'
 
-      UnprocessableEvent = Class.new(Dynflow::Error)
-
-      Algebrick.type do |work|
-        Work = work
-
-        Work::Finalize = type do
-          fields! sequential_manager: SequentialManager,
-                  execution_plan_id:  String
-        end
-
-        Work::Step = type do
-          fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
-                  execution_plan_id: String
-        end
-
-        Work::Event = type do
-          fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
-                  execution_plan_id: String,
-                  event:             Event
-        end
-
-        variants Work::Step, Work::Event, Work::Finalize
-      end
-
       def initialize(world, pool_size = 10)
         super(world)
         @core = Core.spawn name:        'parallel-executor-core',
@@ -43,8 +12,9 @@ module Dynflow
                            initialized: @core_initialized = Concurrent.future
       end
 
-      def execute(execution_plan_id, finished = Concurrent.future)
-        @core.ask([:handle_execution, execution_plan_id, finished]).value!
+      def execute(execution_plan_id, finished = Concurrent.future, wait_for_acceptance = true)
+        accepted = @core.ask([:handle_execution, execution_plan_id, finished])
+        accepted.value! if wait_for_acceptance
         finished
       rescue Concurrent::Actor::ActorTerminated => error
         dynflow_error = Dynflow::Error.new('executor terminated')
@@ -56,7 +26,7 @@ module Dynflow
       end
 
       def event(execution_plan_id, step_id, event, future = Concurrent.future)
-        @core.ask([:handle_event, Event[execution_plan_id, step_id, event, future]])
+        @core.ask([:handle_event, Director::Event[execution_plan_id, step_id, event, future]])
         future
       end
 

data/lib/dynflow/executors/parallel/core.rb

@@ -6,38 +6,33 @@ module Dynflow
         attr_reader :logger
 
         def initialize(world, pool_size)
-          @logger                  = world.logger
-          @world                   = Type! world, World
-          @pool                    = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
-          @execution_plan_managers = {}
-          @plan_ids_in_rescue      = Set.new
-          @terminated              = nil
+          @logger     = world.logger
+          @world      = Type! world, World
+          @pool       = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
+          @terminated = nil
+          @director   = Director.new(@world)
         end
 
         def handle_execution(execution_plan_id, finished)
-          start_executing track_execution_plan(execution_plan_id, finished)
+          if terminating?
+            raise Dynflow::Error,
+                  "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
+          end
+
+          feed_pool(@director.start_execution(execution_plan_id, finished))
         end
 
         def handle_event(event)
-          Type! event, Parallel::Event
+          Type! event, Director::Event
           if terminating?
             raise Dynflow::Error,
                   "cannot accept event: #{event} core is terminating"
           end
-          execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
-          if execution_plan_manager
-            feed_pool execution_plan_manager.event(event)
-            true
-          else
-            raise Dynflow::Error, "no manager for #{event.inspect}"
-          end
-        rescue Dynflow::Error => e
-          event.result.fail e.message
-          raise e
+          feed_pool(@director.handle_event(event))
         end
 
-        def finish_step(step)
-          update_manager(step)
+        def work_finished(work)
+          feed_pool(@director.work_finished(work))
         end
 
         def handle_persistence_error(error)
@@ -53,19 +48,7 @@ module Dynflow
         end
 
         def finish_termination
-          unless @execution_plan_managers.empty?
-            logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
-            begin
-              @execution_plan_managers.values.each do |manager|
-                manager.terminate
-              end
-            rescue Errors::PersistenceError
-              logger.error "could not to clean the data properly"
-            end
-            @execution_plan_managers.values.each do |manager|
-              finish_plan(manager.execution_plan.id)
-            end
-          end
+          @director.terminate
           logger.error '... core terminated.'
           super
         end
@@ -78,96 +61,13 @@ module Dynflow
           self.tell(:handle_persistence_error, e)
         end
 
-        # @return
-        def track_execution_plan(execution_plan_id, finished)
-          execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
-
-          if terminating?
-            raise Dynflow::Error,
-                  "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
-          end
-
-          if @execution_plan_managers[execution_plan_id]
-            raise Dynflow::Error,
-                  "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
-          end
-
-          if execution_plan.state == :stopped
-            raise Dynflow::Error,
-                  "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
-          end
-
-          @execution_plan_managers[execution_plan_id] =
-              ExecutionPlanManager.new(@world, execution_plan, finished)
-
-        rescue Dynflow::Error => e
-          finished.fail e
-          nil
-        end
-
-        def update_manager(finished_work)
-          manager   = @execution_plan_managers[finished_work.execution_plan_id]
-          next_work = manager.what_is_next(finished_work)
-          continue_manager manager, next_work
-        end
-
-        def continue_manager(manager, next_work)
-          if manager.done?
-            finish_plan manager.execution_plan.id
-          else
-            feed_pool next_work
-          end
-        end
-
-        def rescue?(manager)
-          return false if terminating?
-          @world.auto_rescue && manager.execution_plan.state == :paused &&
-              !@plan_ids_in_rescue.include?(manager.execution_plan.id)
-        end
-
-        def rescue!(manager)
-          # TODO: after moving to concurrent-ruby actors, there should be better place
-          # to put this logic of making sure we don't run rescues in endless loop
-          @plan_ids_in_rescue << manager.execution_plan.id
-          rescue_plan_id = manager.execution_plan.rescue_plan_id
-          if rescue_plan_id
-            reference.tell([:handle_execution, rescue_plan_id, manager.future])
-          else
-            set_future(manager)
-          end
-        end
-
         def feed_pool(work_items)
           return if terminating?
-          Type! work_items, Array, Work, NilClass
           return if work_items.nil?
-          work_items = [work_items] if work_items.is_a? Work
-          work_items.all? { |i| Type! i, Work }
+          work_items = [work_items] if work_items.is_a? Director::WorkItem
+          work_items.all? { |i| Type! i, Director::WorkItem }
           work_items.each { |new_work| @pool.tell([:schedule_work, new_work]) }
         end
-
-        def finish_plan(execution_plan_id)
-          manager = @execution_plan_managers.delete(execution_plan_id)
-          if rescue?(manager)
-            rescue!(manager)
-          else
-            set_future(manager)
-          end
-        end
-
-        def set_future(manager)
-          @plan_ids_in_rescue.delete(manager.execution_plan.id)
-          manager.future.success manager.execution_plan
-        end
-
-        def start_executing(manager)
-          return if manager.nil?
-          Type! manager, ExecutionPlanManager
-
-          next_work = manager.start
-          continue_manager manager, next_work
-        end
-
       end
     end
   end