dynflow 0.8.16 → 0.8.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +21 -0
  3. data/.rubocop_todo.yml +0 -25
  4. data/doc/pages/plugins/div_tag.rb +1 -1
  5. data/doc/pages/plugins/tags.rb +0 -1
  6. data/examples/orchestrate.rb +0 -1
  7. data/examples/remote_executor.rb +3 -3
  8. data/examples/sub_plan_concurrency_control.rb +0 -1
  9. data/examples/sub_plans.rb +0 -1
  10. data/lib/dynflow.rb +1 -0
  11. data/lib/dynflow/action.rb +6 -6
  12. data/lib/dynflow/config.rb +2 -2
  13. data/lib/dynflow/connectors/database.rb +1 -1
  14. data/lib/dynflow/connectors/direct.rb +1 -1
  15. data/lib/dynflow/coordinator.rb +4 -4
  16. data/lib/dynflow/director.rb +190 -0
  17. data/lib/dynflow/director/execution_plan_manager.rb +107 -0
  18. data/lib/dynflow/director/flow_manager.rb +43 -0
  19. data/lib/dynflow/director/running_steps_manager.rb +79 -0
  20. data/lib/dynflow/director/sequence_cursor.rb +91 -0
  21. data/lib/dynflow/{executors/parallel → director}/sequential_manager.rb +2 -2
  22. data/lib/dynflow/director/work_queue.rb +48 -0
  23. data/lib/dynflow/dispatcher/client_dispatcher.rb +24 -24
  24. data/lib/dynflow/dispatcher/executor_dispatcher.rb +1 -1
  25. data/lib/dynflow/execution_plan.rb +32 -15
  26. data/lib/dynflow/execution_plan/steps/abstract.rb +14 -14
  27. data/lib/dynflow/execution_plan/steps/error.rb +1 -1
  28. data/lib/dynflow/execution_plan/steps/finalize_step.rb +0 -1
  29. data/lib/dynflow/execution_plan/steps/plan_step.rb +11 -12
  30. data/lib/dynflow/execution_plan/steps/run_step.rb +1 -1
  31. data/lib/dynflow/executors/abstract.rb +5 -8
  32. data/lib/dynflow/executors/parallel.rb +4 -34
  33. data/lib/dynflow/executors/parallel/core.rb +18 -118
  34. data/lib/dynflow/executors/parallel/pool.rb +2 -2
  35. data/lib/dynflow/executors/parallel/worker.rb +3 -11
  36. data/lib/dynflow/persistence_adapters/sequel.rb +1 -2
  37. data/lib/dynflow/testing.rb +2 -0
  38. data/lib/dynflow/testing/in_thread_executor.rb +52 -0
  39. data/lib/dynflow/testing/in_thread_world.rb +64 -0
  40. data/lib/dynflow/testing/managed_clock.rb +1 -1
  41. data/lib/dynflow/throttle_limiter.rb +1 -1
  42. data/lib/dynflow/version.rb +1 -1
  43. data/lib/dynflow/world.rb +13 -7
  44. data/test/abnormal_states_recovery_test.rb +10 -0
  45. data/test/action_test.rb +9 -9
  46. data/test/clock_test.rb +0 -2
  47. data/test/concurrency_control_test.rb +1 -1
  48. data/test/execution_plan_test.rb +0 -2
  49. data/test/executor_test.rb +6 -13
  50. data/test/support/code_workflow_example.rb +1 -1
  51. data/test/support/rescue_example.rb +0 -1
  52. data/test/test_helper.rb +9 -12
  53. data/test/testing_test.rb +74 -2
  54. data/web/views/plan_step.erb +2 -0
  55. metadata +11 -8
  56. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +0 -111
  57. data/lib/dynflow/executors/parallel/flow_manager.rb +0 -45
  58. data/lib/dynflow/executors/parallel/running_steps_manager.rb +0 -81
  59. data/lib/dynflow/executors/parallel/sequence_cursor.rb +0 -97
  60. data/lib/dynflow/executors/parallel/work_queue.rb +0 -50
@@ -0,0 +1,107 @@
1
+ module Dynflow
2
+ class Director
3
+ class ExecutionPlanManager
4
+ include Algebrick::TypeCheck
5
+ include Algebrick::Matching
6
+
7
+ attr_reader :execution_plan, :future
8
+
9
+ def initialize(world, execution_plan, future)
10
+ @world = Type! world, World
11
+ @execution_plan = Type! execution_plan, ExecutionPlan
12
+ @future = Type! future, Concurrent::Edge::Future
13
+ @running_steps_manager = RunningStepsManager.new(world)
14
+
15
+ unless [:planned, :paused].include? execution_plan.state
16
+ raise "execution_plan is not in pending or paused state, it's #{execution_plan.state}"
17
+ end
18
+ execution_plan.execution_history.add('start execution', @world.id)
19
+ execution_plan.update_state(:running)
20
+ end
21
+
22
+ def start
23
+ raise "The future was already set" if @future.completed?
24
+ start_run or start_finalize or finish
25
+ end
26
+
27
+ def prepare_next_step(step)
28
+ StepWorkItem.new(execution_plan.id, step).tap do |work|
29
+ @running_steps_manager.add(step, work)
30
+ end
31
+ end
32
+
33
+ # @return [Array<WorkItem>] of Work items to continue with
34
+ def what_is_next(work)
35
+ Type! work, WorkItem
36
+
37
+ case work
38
+ when StepWorkItem
39
+ step = work.step
40
+ execution_plan.steps[step.id] = step
41
+ suspended, work = @running_steps_manager.done(step)
42
+ work = compute_next_from_step(step) unless suspended
43
+ work
44
+ when FinalizeWorkItem
45
+ raise "Finalize work item without @finalize_manager ready" unless @finalize_manager
46
+ finish
47
+ end
48
+ end
49
+
50
+ def event(event)
51
+ Type! event, Event
52
+ unless event.execution_plan_id == @execution_plan.id
53
+ raise "event #{event.inspect} doesn't belong to plan #{@execution_plan.id}"
54
+ end
55
+ @running_steps_manager.event(event)
56
+ end
57
+
58
+ def done?
59
+ (!@run_manager || @run_manager.done?) && (!@finalize_manager || @finalize_manager.done?)
60
+ end
61
+
62
+ def terminate
63
+ @running_steps_manager.terminate
64
+ end
65
+
66
+ private
67
+
68
+ def compute_next_from_step(step)
69
+ raise "run manager not set" unless @run_manager
70
+ raise "run manager already done" if @run_manager.done?
71
+
72
+ next_steps = @run_manager.what_is_next(step)
73
+ if @run_manager.done?
74
+ start_finalize or finish
75
+ else
76
+ next_steps.map { |s| prepare_next_step(s) }
77
+ end
78
+ end
79
+
80
+ def no_work
81
+ raise "No work but not done" unless done?
82
+ []
83
+ end
84
+
85
+ def start_run
86
+ return if execution_plan.run_flow.empty?
87
+ raise 'run phase already started' if @run_manager
88
+ @run_manager = FlowManager.new(execution_plan, execution_plan.run_flow)
89
+ @run_manager.start.map { |s| prepare_next_step(s) }.tap { |a| raise if a.empty? }
90
+ end
91
+
92
+ def start_finalize
93
+ return if execution_plan.finalize_flow.empty?
94
+ raise 'finalize phase already started' if @finalize_manager
95
+ @finalize_manager = SequentialManager.new(@world, execution_plan)
96
+ [FinalizeWorkItem.new(execution_plan.id, @finalize_manager)]
97
+ end
98
+
99
+ def finish
100
+ execution_plan.execution_history.add('finish execution', @world.id)
101
+ @execution_plan.update_state(execution_plan.error? ? :paused : :stopped)
102
+ return no_work
103
+ end
104
+
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,43 @@
1
+ module Dynflow
2
+ class Director
3
+ class FlowManager
4
+ include Algebrick::TypeCheck
5
+
6
+ attr_reader :execution_plan, :cursor_index
7
+
8
+ def initialize(execution_plan, flow)
9
+ @execution_plan = Type! execution_plan, ExecutionPlan
10
+ @flow = flow
11
+ @cursor_index = {}
12
+ @cursor = build_root_cursor
13
+ end
14
+
15
+ def done?
16
+ @cursor.done?
17
+ end
18
+
19
+ # @return [Set] of steps to continue with
20
+ def what_is_next(flow_step)
21
+ return [] if flow_step.state == :suspended
22
+
23
+ success = flow_step.state != :error
24
+ return cursor_index[flow_step.id].what_is_next(flow_step, success)
25
+ end
26
+
27
+ # @return [Set] of steps to continue with
28
+ def start
29
+ return @cursor.what_is_next.tap do |steps|
30
+ raise 'invalid state' if steps.empty? && !done?
31
+ end
32
+ end
33
+
34
+ private
35
+
36
+ def build_root_cursor
37
+ # the root cursor has to always run against sequence
38
+ sequence = @flow.is_a?(Flows::Sequence) ? @flow : Flows::Sequence.new([@flow])
39
+ return SequenceCursor.new(self, sequence, nil)
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,79 @@
1
+ module Dynflow
2
+ class Director
3
+ # Handles the events generated while running actions, makes sure
4
+ # the events are sent to the action only when in suspended state
5
+ class RunningStepsManager
6
+ include Algebrick::TypeCheck
7
+
8
+ def initialize(world)
9
+ @world = Type! world, World
10
+ @running_steps = {}
11
+ @events = WorkQueue.new(Integer, WorkItem)
12
+ end
13
+
14
+ def terminate
15
+ pending_work = @events.clear.values.flatten(1)
16
+ pending_work.each do |w|
17
+ if EventWorkItem === w
18
+ w.event.result.fail UnprocessableEvent.new("dropping due to termination")
19
+ end
20
+ end
21
+ end
22
+
23
+ def add(step, work)
24
+ Type! step, ExecutionPlan::Steps::RunStep
25
+ @running_steps[step.id] = step
26
+ # we make sure not to run any event when the step is still being executed
27
+ @events.push(step.id, work)
28
+ self
29
+ end
30
+
31
+ # @returns [TrueClass|FalseClass, Array<WorkItem>]
32
+ def done(step)
33
+ Type! step, ExecutionPlan::Steps::RunStep
34
+ @events.shift(step.id).tap do |work|
35
+ work.event.result.success true if EventWorkItem === work
36
+ end
37
+
38
+ if step.state == :suspended
39
+ return true, [@events.first(step.id)].compact
40
+ else
41
+ while (event = @events.shift(step.id))
42
+ message = "step #{step.execution_plan_id}:#{step.id} dropping event #{event.event}"
43
+ @world.logger.warn message
44
+ event.event.result.fail UnprocessableEvent.new(message).
45
+ tap { |e| e.set_backtrace(caller) }
46
+ end
47
+ raise 'assert' unless @events.empty?(step.id)
48
+ @running_steps.delete(step.id)
49
+ return false, []
50
+ end
51
+ end
52
+
53
+ def try_to_terminate
54
+ @running_steps.delete_if do |_, step|
55
+ step.state != :running
56
+ end
57
+ return @running_steps.empty?
58
+ end
59
+
60
+ # @returns [Array<WorkItem>]
61
+ def event(event)
62
+ Type! event, Event
63
+ next_work_items = []
64
+
65
+ step = @running_steps[event.step_id]
66
+ unless step
67
+ event.result.fail UnprocessableEvent.new('step is not suspended, it cannot process events')
68
+ return next_work_items
69
+ end
70
+
71
+ can_run_event = @events.empty?(step.id)
72
+ work = EventWorkItem.new(event.execution_plan_id, step, event)
73
+ @events.push(step.id, work)
74
+ next_work_items << work if can_run_event
75
+ next_work_items
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,91 @@
1
+ module Dynflow
2
+ class Director
3
+ class SequenceCursor
4
+
5
+ def initialize(flow_manager, sequence, parent_cursor = nil)
6
+ @flow_manager = flow_manager
7
+ @sequence = sequence
8
+ @parent_cursor = parent_cursor
9
+ @todo = []
10
+ @index = -1 # starts before first element
11
+ @no_error_so_far = true
12
+ end
13
+
14
+ # @param [ExecutionPlan::Steps::Abstract, SequenceCursor] work
15
+ # step or sequence cursor that was done
16
+ # @param [true, false] success was the work finished successfully
17
+ # @return [Array<Integer>] new step_ids that can be done next
18
+ def what_is_next(work = nil, success = true)
19
+ unless work.nil? || @todo.delete(work)
20
+ raise "marking as done work that was not expected: #{work.inspect}"
21
+ end
22
+
23
+ @no_error_so_far &&= success
24
+
25
+ if done_here?
26
+ return next_steps
27
+ else
28
+ return []
29
+ end
30
+ end
31
+
32
+ # return true if we can't move the cursor further, either when
33
+ # everyting is done in the sequence or there was some failure
34
+ # that prevents us from moving
35
+ def done?
36
+ (!@no_error_so_far && done_here?) || @index == @sequence.size
37
+ end
38
+
39
+ protected
40
+
41
+ # steps we can do right now without waiting for anything
42
+ def steps_todo
43
+ @todo.map do |item|
44
+ case item
45
+ when SequenceCursor
46
+ item.steps_todo
47
+ else
48
+ item
49
+ end
50
+ end.flatten
51
+ end
52
+
53
+ def move
54
+ @index += 1
55
+ next_flow = @sequence.sub_flows[@index]
56
+ add_todo(next_flow)
57
+ end
58
+
59
+ private
60
+
61
+ def done_here?
62
+ @todo.empty?
63
+ end
64
+
65
+ def next_steps
66
+ move if @no_error_so_far
67
+ return steps_todo unless done?
68
+ if @parent_cursor
69
+ return @parent_cursor.what_is_next(self, @no_error_so_far)
70
+ else
71
+ return []
72
+ end
73
+ end
74
+
75
+ def add_todo(flow)
76
+ case flow
77
+ when Flows::Sequence
78
+ @todo << SequenceCursor.new(@flow_manager, flow, self).tap do |cursor|
79
+ cursor.move
80
+ end
81
+ when Flows::Concurrence
82
+ flow.sub_flows.each { |sub_flow| add_todo(sub_flow) }
83
+ when Flows::Atom
84
+ @flow_manager.cursor_index[flow.step_id] = self
85
+ @todo << @flow_manager.execution_plan.steps[flow.step_id]
86
+ end
87
+ end
88
+
89
+ end
90
+ end
91
+ end
@@ -1,6 +1,6 @@
1
1
  module Dynflow
2
- module Executors
3
- class Parallel::SequentialManager
2
+ class Director
3
+ class SequentialManager
4
4
  attr_reader :execution_plan, :world
5
5
 
6
6
  def initialize(world, execution_plan)
@@ -0,0 +1,48 @@
1
+ module Dynflow
2
+ class Director
3
+ class WorkQueue
4
+ include Algebrick::TypeCheck
5
+
6
+ def initialize(key_type = Object, work_type = Object)
7
+ @key_type = key_type
8
+ @work_type = work_type
9
+ @stash = Hash.new { |hash, key| hash[key] = [] }
10
+ end
11
+
12
+ def push(key, work)
13
+ Type! key, @key_type
14
+ Type! work, @work_type
15
+ @stash[key].push work
16
+ end
17
+
18
+ def shift(key)
19
+ return nil unless present? key
20
+ @stash[key].shift.tap { |work| @stash.delete(key) if @stash[key].empty? }
21
+ end
22
+
23
+ def present?(key)
24
+ @stash.key?(key)
25
+ end
26
+
27
+ def empty?(key)
28
+ !present?(key)
29
+ end
30
+
31
+ def clear
32
+ ret = @stash.dup
33
+ @stash.clear
34
+ ret
35
+ end
36
+
37
+ def size(key)
38
+ return 0 if empty?(key)
39
+ @stash[key].size
40
+ end
41
+
42
+ def first(key)
43
+ return nil if empty?(key)
44
+ @stash[key].first
45
+ end
46
+ end
47
+ end
48
+ end
@@ -52,15 +52,15 @@ module Dynflow
52
52
 
53
53
  def dispatch_request(request, client_world_id, request_id)
54
54
  executor_id = match request,
55
- (on ~Execution do |execution|
56
- AnyExecutor
57
- end),
58
- (on ~Event do |event|
59
- find_executor(event.execution_plan_id)
60
- end),
61
- (on Ping.(~any) do |receiver_id|
62
- receiver_id
63
- end)
55
+ (on ~Execution do |execution|
56
+ AnyExecutor
57
+ end),
58
+ (on ~Event do |event|
59
+ find_executor(event.execution_plan_id)
60
+ end),
61
+ (on Ping.(~any) do |receiver_id|
62
+ receiver_id
63
+ end)
64
64
  envelope = Envelope[request_id, client_world_id, executor_id, request]
65
65
  if Dispatcher::UnknownWorld === envelope.receiver_id
66
66
  raise Dynflow::Error, "Could not find an executor for #{envelope}"
@@ -74,15 +74,15 @@ module Dynflow
74
74
  def dispatch_response(envelope)
75
75
  return unless @tracked_requests.key?(envelope.request_id)
76
76
  match envelope.message,
77
- (on ~Accepted do
78
- @tracked_requests[envelope.request_id].accept!
79
- end),
80
- (on ~Failed do |msg|
81
- resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
82
- end),
83
- (on Done | Pong do
84
- resolve_tracked_request(envelope.request_id)
85
- end)
77
+ (on ~Accepted do
78
+ @tracked_requests[envelope.request_id].accept!
79
+ end),
80
+ (on ~Failed do |msg|
81
+ resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
82
+ end),
83
+ (on Done | Pong do
84
+ resolve_tracked_request(envelope.request_id)
85
+ end)
86
86
  end
87
87
 
88
88
  private
@@ -128,12 +128,12 @@ module Dynflow
128
128
  else
129
129
  tracked_request = @tracked_requests[id]
130
130
  resolve_to = match tracked_request.request,
131
- (on Execution.(execution_plan_id: ~any) do |uuid|
132
- @world.persistence.load_execution_plan(uuid)
133
- end),
134
- (on Event | Ping do
135
- true
136
- end)
131
+ (on Execution.(execution_plan_id: ~any) do |uuid|
132
+ @world.persistence.load_execution_plan(uuid)
133
+ end),
134
+ (on Event | Ping do
135
+ true
136
+ end)
137
137
  @tracked_requests.delete(id).success! resolve_to
138
138
  end
139
139
  end