dynflow 0.8.16 → 0.8.17

data/lib/dynflow/director/execution_plan_manager.rb

@@ -0,0 +1,107 @@
+module Dynflow
+  class Director
+    class ExecutionPlanManager
+      include Algebrick::TypeCheck
+      include Algebrick::Matching
+
+      attr_reader :execution_plan, :future
+
+      def initialize(world, execution_plan, future)
+        @world                 = Type! world, World
+        @execution_plan        = Type! execution_plan, ExecutionPlan
+        @future                = Type! future, Concurrent::Edge::Future
+        @running_steps_manager = RunningStepsManager.new(world)
+
+        unless [:planned, :paused].include? execution_plan.state
+          raise "execution_plan is not in pending or paused state, it's #{execution_plan.state}"
+        end
+        execution_plan.execution_history.add('start execution', @world.id)
+        execution_plan.update_state(:running)
+      end
+
+      def start
+        raise "The future was already set" if @future.completed?
+        start_run or start_finalize or finish
+      end
+
+      def prepare_next_step(step)
+        StepWorkItem.new(execution_plan.id, step).tap do |work|
+          @running_steps_manager.add(step, work)
+        end
+      end
+
+      # @return [Array<WorkItem>] of Work items to continue with
+      def what_is_next(work)
+        Type! work, WorkItem
+
+        case work
+        when StepWorkItem
+          step = work.step
+          execution_plan.steps[step.id] = step
+          suspended, work = @running_steps_manager.done(step)
+          work = compute_next_from_step(step) unless suspended
+          work
+        when FinalizeWorkItem
+          raise "Finalize work item without @finalize_manager ready" unless @finalize_manager
+          finish
+        end
+      end
+
+      def event(event)
+        Type! event, Event
+        unless event.execution_plan_id == @execution_plan.id
+          raise "event #{event.inspect} doesn't belong to plan #{@execution_plan.id}"
+        end
+        @running_steps_manager.event(event)
+      end
+
+      def done?
+        (!@run_manager || @run_manager.done?) && (!@finalize_manager || @finalize_manager.done?)
+      end
+
+      def terminate
+        @running_steps_manager.terminate
+      end
+
+      private
+
+      def compute_next_from_step(step)
+        raise "run manager not set" unless @run_manager
+        raise "run manager already done" if @run_manager.done?
+
+        next_steps = @run_manager.what_is_next(step)
+        if @run_manager.done?
+          start_finalize or finish
+        else
+          next_steps.map { |s| prepare_next_step(s) }
+        end
+      end
+
+      def no_work
+        raise "No work but not done" unless done?
+        []
+      end
+
+      def start_run
+        return if execution_plan.run_flow.empty?
+        raise 'run phase already started' if @run_manager
+        @run_manager = FlowManager.new(execution_plan, execution_plan.run_flow)
+        @run_manager.start.map { |s| prepare_next_step(s) }.tap { |a| raise if a.empty? }
+      end
+
+      def start_finalize
+        return if execution_plan.finalize_flow.empty?
+        raise 'finalize phase already started' if @finalize_manager
+        @finalize_manager = SequentialManager.new(@world, execution_plan)
+        [FinalizeWorkItem.new(execution_plan.id, @finalize_manager)]
+      end
+
+      def finish
+        execution_plan.execution_history.add('finish execution', @world.id)
+        @execution_plan.update_state(execution_plan.error? ? :paused : :stopped)
+        return no_work
+      end
+
+    end
+  end
+end

data/lib/dynflow/director/flow_manager.rb

@@ -0,0 +1,43 @@
+module Dynflow
+  class Director
+    class FlowManager
+      include Algebrick::TypeCheck
+
+      attr_reader :execution_plan, :cursor_index
+
+      def initialize(execution_plan, flow)
+        @execution_plan = Type! execution_plan, ExecutionPlan
+        @flow           = flow
+        @cursor_index   = {}
+        @cursor         = build_root_cursor
+      end
+
+      def done?
+        @cursor.done?
+      end
+
+      # @return [Set] of steps to continue with
+      def what_is_next(flow_step)
+        return [] if flow_step.state == :suspended
+
+        success = flow_step.state != :error
+        return cursor_index[flow_step.id].what_is_next(flow_step, success)
+      end
+
+      # @return [Set] of steps to continue with
+      def start
+        return @cursor.what_is_next.tap do |steps|
+          raise 'invalid state' if steps.empty? && !done?
+        end
+      end
+
+      private
+
+      def build_root_cursor
+        # the root cursor has to always run against sequence
+        sequence = @flow.is_a?(Flows::Sequence) ? @flow : Flows::Sequence.new([@flow])
+        return SequenceCursor.new(self, sequence, nil)
+      end
+    end
+  end
+end

data/lib/dynflow/director/running_steps_manager.rb

@@ -0,0 +1,79 @@
+module Dynflow
+  class Director
+    # Handles the events generated while running actions, makes sure
+    # the events are sent to the action only when in suspended state
+    class RunningStepsManager
+      include Algebrick::TypeCheck
+
+      def initialize(world)
+        @world         = Type! world, World
+        @running_steps = {}
+        @events        = WorkQueue.new(Integer, WorkItem)
+      end
+
+      def terminate
+        pending_work = @events.clear.values.flatten(1)
+        pending_work.each do |w|
+          if EventWorkItem === w
+            w.event.result.fail UnprocessableEvent.new("dropping due to termination")
+          end
+        end
+      end
+
+      def add(step, work)
+        Type! step, ExecutionPlan::Steps::RunStep
+        @running_steps[step.id] = step
+        # we make sure not to run any event when the step is still being executed
+        @events.push(step.id, work)
+        self
+      end
+
+      # @returns [TrueClass|FalseClass, Array<WorkItem>]
+      def done(step)
+        Type! step, ExecutionPlan::Steps::RunStep
+        @events.shift(step.id).tap do |work|
+          work.event.result.success true if EventWorkItem === work
+        end
+
+        if step.state == :suspended
+          return true, [@events.first(step.id)].compact
+        else
+          while (event = @events.shift(step.id))
+            message = "step #{step.execution_plan_id}:#{step.id} dropping event #{event.event}"
+            @world.logger.warn message
+            event.event.result.fail UnprocessableEvent.new(message).
+                tap { |e| e.set_backtrace(caller) }
+          end
+          raise 'assert' unless @events.empty?(step.id)
+          @running_steps.delete(step.id)
+          return false, []
+        end
+      end
+
+      def try_to_terminate
+        @running_steps.delete_if do |_, step|
+          step.state != :running
+        end
+        return @running_steps.empty?
+      end
+
+      # @returns [Array<WorkItem>]
+      def event(event)
+        Type! event, Event
+        next_work_items = []
+
+        step = @running_steps[event.step_id]
+        unless step
+          event.result.fail UnprocessableEvent.new('step is not suspended, it cannot process events')
+          return next_work_items
+        end
+
+        can_run_event = @events.empty?(step.id)
+        work          = EventWorkItem.new(event.execution_plan_id, step, event)
+        @events.push(step.id, work)
+        next_work_items << work if can_run_event
+        next_work_items
+      end
+    end
+  end
+end

data/lib/dynflow/director/sequence_cursor.rb

@@ -0,0 +1,91 @@
+module Dynflow
+  class Director
+    class SequenceCursor
+
+      def initialize(flow_manager, sequence, parent_cursor = nil)
+        @flow_manager    = flow_manager
+        @sequence        = sequence
+        @parent_cursor   = parent_cursor
+        @todo            = []
+        @index           = -1 # starts before first element
+        @no_error_so_far = true
+      end
+
+      # @param [ExecutionPlan::Steps::Abstract, SequenceCursor] work
+      #   step or sequence cursor that was done
+      # @param [true, false] success was the work finished successfully
+      # @return [Array<Integer>] new step_ids that can be done next
+      def what_is_next(work = nil, success = true)
+        unless work.nil? || @todo.delete(work)
+          raise "marking as done work that was not expected: #{work.inspect}"
+        end
+
+        @no_error_so_far &&= success
+
+        if done_here?
+          return next_steps
+        else
+          return []
+        end
+      end
+
+      # return true if we can't move the cursor further, either when
+      # everyting is done in the sequence or there was some failure
+      # that prevents us from moving
+      def done?
+        (!@no_error_so_far && done_here?) || @index == @sequence.size
+      end
+
+      protected
+
+      # steps we can do right now without waiting for anything
+      def steps_todo
+        @todo.map do |item|
+          case item
+          when SequenceCursor
+            item.steps_todo
+          else
+            item
+          end
+        end.flatten
+      end
+
+      def move
+        @index   += 1
+        next_flow = @sequence.sub_flows[@index]
+        add_todo(next_flow)
+      end
+
+      private
+
+      def done_here?
+        @todo.empty?
+      end
+
+      def next_steps
+        move if @no_error_so_far
+        return steps_todo unless done?
+        if @parent_cursor
+          return @parent_cursor.what_is_next(self, @no_error_so_far)
+        else
+          return []
+        end
+      end
+
+      def add_todo(flow)
+        case flow
+        when Flows::Sequence
+          @todo << SequenceCursor.new(@flow_manager, flow, self).tap do |cursor|
+            cursor.move
+          end
+        when Flows::Concurrence
+          flow.sub_flows.each { |sub_flow| add_todo(sub_flow) }
+        when Flows::Atom
+          @flow_manager.cursor_index[flow.step_id] = self
+          @todo << @flow_manager.execution_plan.steps[flow.step_id]
+        end
+      end
+
+    end
+  end
+end

data/lib/dynflow/{executors/parallel → director}/sequential_manager.rb

@@ -1,6 +1,6 @@
 module Dynflow
-  module Executors
-    class Parallel::SequentialManager
+  class Director
+    class SequentialManager
       attr_reader :execution_plan, :world
 
       def initialize(world, execution_plan)

data/lib/dynflow/director/work_queue.rb

@@ -0,0 +1,48 @@
+module Dynflow
+  class Director
+    class WorkQueue
+      include Algebrick::TypeCheck
+
+      def initialize(key_type = Object, work_type = Object)
+        @key_type  = key_type
+        @work_type = work_type
+        @stash     = Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def push(key, work)
+        Type! key, @key_type
+        Type! work, @work_type
+        @stash[key].push work
+      end
+
+      def shift(key)
+        return nil unless present? key
+        @stash[key].shift.tap { |work| @stash.delete(key) if @stash[key].empty? }
+      end
+
+      def present?(key)
+        @stash.key?(key)
+      end
+
+      def empty?(key)
+        !present?(key)
+      end
+
+      def clear
+        ret = @stash.dup
+        @stash.clear
+        ret
+      end
+
+      def size(key)
+        return 0 if empty?(key)
+        @stash[key].size
+      end
+
+      def first(key)
+        return nil if empty?(key)
+        @stash[key].first
+      end
+    end
+  end
+end

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -52,15 +52,15 @@ module Dynflow
 
       def dispatch_request(request, client_world_id, request_id)
         executor_id = match request,
-            (on ~Execution do |execution|
-               AnyExecutor
-             end),
-            (on ~Event do |event|
-               find_executor(event.execution_plan_id)
-             end),
-            (on Ping.(~any) do |receiver_id|
-               receiver_id
-             end)
+                            (on ~Execution do |execution|
+                               AnyExecutor
+                             end),
+                            (on ~Event do |event|
+                               find_executor(event.execution_plan_id)
+                             end),
+                            (on Ping.(~any) do |receiver_id|
+                               receiver_id
+                             end)
         envelope = Envelope[request_id, client_world_id, executor_id, request]
         if Dispatcher::UnknownWorld === envelope.receiver_id
           raise Dynflow::Error, "Could not find an executor for #{envelope}"
@@ -74,15 +74,15 @@ module Dynflow
       def dispatch_response(envelope)
         return unless @tracked_requests.key?(envelope.request_id)
         match envelope.message,
-            (on ~Accepted do
-               @tracked_requests[envelope.request_id].accept!
-             end),
-            (on ~Failed do |msg|
-               resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
-             end),
-            (on Done | Pong do
-               resolve_tracked_request(envelope.request_id)
-             end)
+              (on ~Accepted do
+                 @tracked_requests[envelope.request_id].accept!
+               end),
+              (on ~Failed do |msg|
+                 resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
+               end),
+              (on Done | Pong do
+                 resolve_tracked_request(envelope.request_id)
+               end)
       end
 
       private
@@ -128,12 +128,12 @@ module Dynflow
         else
           tracked_request = @tracked_requests[id]
           resolve_to = match tracked_request.request,
-              (on Execution.(execution_plan_id: ~any) do |uuid|
-                 @world.persistence.load_execution_plan(uuid)
-               end),
-              (on Event | Ping do
-                 true
-               end)
+                             (on Execution.(execution_plan_id: ~any) do |uuid|
+                                @world.persistence.load_execution_plan(uuid)
+                              end),
+                             (on Event | Ping do
+                                true
+                              end)
           @tracked_requests.delete(id).success! resolve_to
         end
       end