dynflow 0.7.9 → 0.8.0

data/lib/dynflow/executors.rb

@@ -3,7 +3,6 @@ module Dynflow
 
     require 'dynflow/executors/abstract'
     require 'dynflow/executors/parallel'
-    require 'dynflow/executors/remote_via_socket'
 
   end
 end

data/lib/dynflow/executors/abstract.rb

@@ -5,12 +5,7 @@ module Dynflow
         fields! execution_plan_id: String,
                 step_id:           Fixnum,
                 event:             Object,
-                result:            Future
-      end
-
-      Execution = Algebrick.type do
-        fields! execution_plan_id: String,
-                finished:          Future
+                result:            Concurrent::Edge::Future
       end
 
       include Algebrick::TypeCheck
@@ -21,21 +16,21 @@ module Dynflow
         @logger = world.logger
       end
 
-      # @return [Future]
+      # @return [Concurrent::Edge::Future]
       # @raise when execution_plan_id is not accepted
       def execute(execution_plan_id)
         raise NotImplementedError
       end
 
-      def event(execution_plan_id, step_id, event, future = Future)
+      def event(execution_plan_id, step_id, event, future = Concurrent.future)
         raise NotImplementedError
       end
 
-      def terminate(future = Future.new)
+      def terminate(future = Concurrent.future)
         raise NotImplementedError
       end
 
-      # @return [Future]
+      # @return [Concurrent::Edge::Future]
       def initialized
         raise NotImplementedError
       end

data/lib/dynflow/executors/parallel.rb

@@ -36,34 +36,37 @@ module Dynflow
         variants Work::Step, Work::Event, Work::Finalize
       end
 
-      PoolDone       = Algebrick.type { fields! work: Work }
-      PoolTerminated = Algebrick.atom
-      WorkerDone     = Algebrick.type { fields! work: Work, worker: Worker }
-
       def initialize(world, pool_size = 10)
         super(world)
-        @core = Core.new world, pool_size
+        @core = Core.spawn name:        'parallel-executor-core',
+                           args:        [world, pool_size],
+                           initialized: @core_initialized = Concurrent.future
       end
 
-      def execute(execution_plan_id, finished = Future.new)
-        @core.ask(Execution[execution_plan_id, finished]).value!
+      def execute(execution_plan_id, finished = Concurrent.future)
+        @core.ask([:handle_execution, execution_plan_id, finished]).value!
         finished
+      rescue Concurrent::Actor::ActorTerminated => error
+        dynflow_error = Dynflow::Error.new('executor terminated')
+        finished.fail dynflow_error unless finished.completed?
+        raise dynflow_error
       rescue => e
-        finished.fail e unless finished.ready?
+        finished.fail e unless finished.completed?
         raise e
       end
 
-      def event(execution_plan_id, step_id, event, future = Future.new)
-        @core << Event[execution_plan_id, step_id, event, future]
+      def event(execution_plan_id, step_id, event, future = Concurrent.future)
+        @core.ask([:handle_event, Event[execution_plan_id, step_id, event, future]])
         future
       end
 
-      def terminate(future = Future.new)
-        @core.ask(MicroActor::Terminate, future)
+      def terminate(future = Concurrent.future)
+        @core.tell([:start_termination, future])
+        future
       end
 
       def initialized
-        @core.initialized
+        @core_initialized
       end
     end
   end

data/lib/dynflow/executors/parallel/core.rb

@@ -2,51 +2,83 @@ module Dynflow
   module Executors
     class Parallel < Abstract
 
-      # TODO add dynflow error handling to avoid getting stuck and report errors to the future
-      class Core < MicroActor
-        def initialize(world, pool_size)
-          super(world.logger, world, pool_size)
-        end
-
-        private
+      class Core < Actor
+        attr_reader :logger
 
-        def delayed_initialize(world, pool_size)
+        def initialize(world, pool_size)
+          @logger                  = world.logger
           @world                   = Type! world, World
-          @pool                    = Pool.new(self, pool_size, world.transaction_adapter)
+          @pool                    = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
           @execution_plan_managers = {}
           @plan_ids_in_rescue      = Set.new
+          @terminated              = nil
         end
 
-        def on_message(message)
-          match message,
-                (on ~Parallel::Execution do |(execution_plan_id, finished)|
-                  start_executing track_execution_plan(execution_plan_id, finished)
-                  true
-                end),
-                (on ~Parallel::Event do |event|
-                  event(event)
-                 end),
-                (on Parallel::PoolTerminated do
-                   finish_termination
-                 end),
-                (on PoolDone.(~any) do |step|
-                  update_manager(step)
-                 end),
-                (on ~Errors::PersistenceError.to_m do |error|
-                   logger.fatal "PersistenceError in executor: terminating"
-                   logger.fatal error
-                   @world.terminate
-                 end)
-        rescue Errors::PersistenceError => e
-          self << e
+        def handle_execution(execution_plan_id, finished)
+          start_executing track_execution_plan(execution_plan_id, finished)
+        end
+
+        def handle_event(event)
+          Type! event, Parallel::Event
+          if terminating?
+            raise Dynflow::Error,
+                  "cannot accept event: #{event} core is terminating"
+          end
+          execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
+          if execution_plan_manager
+            feed_pool execution_plan_manager.event(event)
+            true
+          else
+            raise Dynflow::Error, "no manager for #{event.inspect}"
+          end
+        rescue Dynflow::Error => e
+          event.result.fail e.message
+          raise e
         end
 
-        def termination
+        def finish_step(step)
+          update_manager(step)
+        end
+
+        def handle_persistence_error(error)
+          logger.fatal "PersistenceError in executor: terminating"
+          logger.fatal error
+          @world.terminate
+        end
+
+        def start_termination(*args)
+          super
           logger.info 'shutting down Core ...'
-          @pool << MicroActor::Terminate
+          @pool.tell([:start_termination, Concurrent.future])
+        end
+
+        def finish_termination
+          unless @execution_plan_managers.empty?
+            logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
+            begin
+              @execution_plan_managers.values.each do |manager|
+                manager.terminate
+              end
+            rescue Errors::PersistenceError
+              logger.error "could not to clean the data properly"
+            end
+            @execution_plan_managers.values.each do |manager|
+              finish_plan(manager.execution_plan.id)
+            end
+          end
+          logger.error '... core terminated.'
+          super
+        end
+
+        private
+
+        def on_message(message)
+          super
+        rescue Errors::PersistenceError => e
+          self.tell(:handle_persistence_error, e)
         end
 
-        # @return false on problem
+        # @return
         def track_execution_plan(execution_plan_id, finished)
           execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
 
@@ -70,14 +102,7 @@ module Dynflow
 
         rescue Dynflow::Error => e
           finished.fail e
-          raise e
-        end
-
-        def start_executing(manager)
-          Type! manager, ExecutionPlanManager
-
-          next_work = manager.start
-          continue_manager manager, next_work
+          nil
         end
 
         def update_manager(finished_work)
@@ -106,18 +131,19 @@ module Dynflow
           @plan_ids_in_rescue << manager.execution_plan.id
           rescue_plan_id = manager.execution_plan.rescue_plan_id
           if rescue_plan_id
-            self << Parallel::Execution[rescue_plan_id, manager.future]
+            reference.tell([:handle_execution, rescue_plan_id, manager.future])
           else
             set_future(manager)
           end
         end
 
         def feed_pool(work_items)
+          return if terminating?
           Type! work_items, Array, Work, NilClass
           return if work_items.nil?
           work_items = [work_items] if work_items.is_a? Work
           work_items.all? { |i| Type! i, Work }
-          work_items.each { |new_work| @pool << new_work }
+          work_items.each { |new_work| @pool.tell([:schedule_work, new_work]) }
         end
 
         def finish_plan(execution_plan_id)
@@ -131,45 +157,17 @@ module Dynflow
 
         def set_future(manager)
           @plan_ids_in_rescue.delete(manager.execution_plan.id)
-          manager.future.resolve manager.execution_plan
+          manager.future.success manager.execution_plan
         end
 
+        def start_executing(manager)
+          return if manager.nil?
+          Type! manager, ExecutionPlanManager
 
-        def event(event)
-          Type! event, Parallel::Event
-          if terminating?
-            raise Dynflow::Error,
-                  "cannot accept event: #{event} core is terminating"
-          end
-          execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
-          if execution_plan_manager
-            feed_pool execution_plan_manager.event(event)
-            true
-          else
-            raise Dynflow::Error, "no manager for #{event.execution_plan_id}:#{event.step_id}"
-          end
-        rescue Dynflow::Error => e
-          event.result.fail e.message
-          raise e
+          next_work = manager.start
+          continue_manager manager, next_work
         end
 
-        def finish_termination
-          unless @execution_plan_managers.empty?
-            logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
-            begin
-              @execution_plan_managers.values.each do |manager|
-                manager.terminate
-              end
-            rescue Errors::PersistenceError
-              logger.error "could not to clean the data properly"
-            end
-            @execution_plan_managers.values.each do |manager|
-              finish_plan(manager.execution_plan.id)
-            end
-          end
-          logger.error '... core terminated.'
-          terminate!
-        end
       end
     end
   end

data/lib/dynflow/executors/parallel/execution_plan_manager.rb

@@ -10,17 +10,18 @@ module Dynflow
         def initialize(world, execution_plan, future)
           @world                 = Type! world, World
           @execution_plan        = Type! execution_plan, ExecutionPlan
-          @future                = Type! future, Future
+          @future                = Type! future, Concurrent::Edge::Future
           @running_steps_manager = RunningStepsManager.new(world)
 
           unless [:planned, :paused].include? execution_plan.state
             raise "execution_plan is not in pending or paused state, it's #{execution_plan.state}"
           end
+          execution_plan.execution_history.add('start execution', @world.id)
           execution_plan.update_state(:running)
         end
 
         def start
-          raise "The future was already set" if @future.ready?
+          raise "The future was already set" if @future.completed?
           start_run or start_finalize or finish
         end
 
@@ -73,9 +74,6 @@ module Dynflow
 
         def terminate
           @running_steps_manager.terminate
-          unless @execution_plan.state == :paused
-            @execution_plan.update_state(:paused)
-          end
         end
 
         private
@@ -102,6 +100,7 @@ module Dynflow
         end
 
         def finish
+          execution_plan.execution_history.add('finish execution', @world.id)
           @execution_plan.update_state(execution_plan.error? ? :paused : :stopped)
           return no_work
         end

data/lib/dynflow/executors/parallel/pool.rb

@@ -1,33 +1,7 @@
 module Dynflow
   module Executors
     class Parallel < Abstract
-      class Pool < MicroActor
-        class RoundRobin
-          def initialize
-            @data   = []
-            @cursor = 0
-          end
-
-          def add(item)
-            @data.push item
-            self
-          end
-
-          def delete(item)
-            @data.delete item
-            self
-          end
-
-          def next
-            @cursor = 0 if @cursor > @data.size-1
-            @data[@cursor].tap { @cursor += 1 }
-          end
-
-          def empty?
-            @data.empty?
-          end
-        end
-
+      class Pool < Actor
         class JobStorage
           def initialize
             @round_robin = RoundRobin.new
@@ -62,43 +36,39 @@ module Dynflow
         end
 
         def initialize(core, pool_size, transaction_adapter)
-          super(core.logger, core, pool_size, transaction_adapter)
+          @executor_core = core
+          @pool_size     = pool_size
+          @free_workers  = Array.new(pool_size) { |i| Worker.spawn("worker-#{i}", reference, transaction_adapter) }
+          @jobs          = JobStorage.new
         end
 
-        private
+        def schedule_work(work)
+          @jobs.add work
+          distribute_jobs
+        end
 
-        def delayed_initialize(core, pool_size, transaction_adapter)
-          @core         = core
-          @pool_size    = pool_size
-          @free_workers = Array.new(pool_size) { Worker.new(self, transaction_adapter) }
-          @jobs         = JobStorage.new
+        def worker_done(worker, step)
+          @executor_core.tell([:finish_step, step])
+          @free_workers << worker
+          distribute_jobs
         end
 
-        def on_message(message)
-          match message,
-                (on ~Work do |work|
-                  @jobs.add work
-                  distribute_jobs
-                 end),
-                (on WorkerDone.(~any, ~any) do |step, worker|
-                  @core << PoolDone[step]
-                  @free_workers << worker
-                  distribute_jobs
-                 end),
-                (on Errors::PersistenceError do
-                   @core << message
-                 end)
+        def handle_persistence_error(error)
+          @executor_core.tell(:handle_persistence_error, error)
         end
 
-        def termination
+        def start_termination(*args)
+          super
           try_to_terminate
         end
 
+        private
+
         def try_to_terminate
           if terminating? && @free_workers.size == @pool_size
-            @free_workers.map { |worker| worker.ask(Terminate) }.each(&:wait)
-            @core << PoolTerminated
-            terminate!
+            @free_workers.map { |worker| worker.ask(:terminate!) }.map(&:wait)
+            @executor_core.tell(:finish_termination)
+            finish_termination
           end
         end
 

data/lib/dynflow/executors/parallel/running_steps_manager.rb

@@ -14,7 +14,7 @@ module Dynflow
         end
 
         def terminate
-          pending_work = @events.clear.values.flatten
+          pending_work = @events.clear.values.flatten(1)
           pending_work.each do |w|
             if Work::Event === w
               w.event.result.fail UnprocessableEvent.new("dropping due to termination")
@@ -34,7 +34,7 @@ module Dynflow
         def done(step)
           Type! step, ExecutionPlan::Steps::RunStep
           @events.shift(step.id).tap do |work|
-            work.event.result.resolve true if Work::Event === work
+            work.event.result.success true if Work::Event === work
           end
 
           if step.state == :suspended
@@ -52,6 +52,13 @@ module Dynflow
           end
         end
 
+        def try_to_terminate
+          @running_steps.delete_if do |_, step|
+            step.state != :running
+          end
+          return @running_steps.empty?
+        end
+
         # @returns [Work, nil]
         def event(event)
           Type! event, Parallel::Event