dynflow 0.8.16 → 0.8.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +21 -0
  3. data/.rubocop_todo.yml +0 -25
  4. data/doc/pages/plugins/div_tag.rb +1 -1
  5. data/doc/pages/plugins/tags.rb +0 -1
  6. data/examples/orchestrate.rb +0 -1
  7. data/examples/remote_executor.rb +3 -3
  8. data/examples/sub_plan_concurrency_control.rb +0 -1
  9. data/examples/sub_plans.rb +0 -1
  10. data/lib/dynflow.rb +1 -0
  11. data/lib/dynflow/action.rb +6 -6
  12. data/lib/dynflow/config.rb +2 -2
  13. data/lib/dynflow/connectors/database.rb +1 -1
  14. data/lib/dynflow/connectors/direct.rb +1 -1
  15. data/lib/dynflow/coordinator.rb +4 -4
  16. data/lib/dynflow/director.rb +190 -0
  17. data/lib/dynflow/director/execution_plan_manager.rb +107 -0
  18. data/lib/dynflow/director/flow_manager.rb +43 -0
  19. data/lib/dynflow/director/running_steps_manager.rb +79 -0
  20. data/lib/dynflow/director/sequence_cursor.rb +91 -0
  21. data/lib/dynflow/{executors/parallel → director}/sequential_manager.rb +2 -2
  22. data/lib/dynflow/director/work_queue.rb +48 -0
  23. data/lib/dynflow/dispatcher/client_dispatcher.rb +24 -24
  24. data/lib/dynflow/dispatcher/executor_dispatcher.rb +1 -1
  25. data/lib/dynflow/execution_plan.rb +32 -15
  26. data/lib/dynflow/execution_plan/steps/abstract.rb +14 -14
  27. data/lib/dynflow/execution_plan/steps/error.rb +1 -1
  28. data/lib/dynflow/execution_plan/steps/finalize_step.rb +0 -1
  29. data/lib/dynflow/execution_plan/steps/plan_step.rb +11 -12
  30. data/lib/dynflow/execution_plan/steps/run_step.rb +1 -1
  31. data/lib/dynflow/executors/abstract.rb +5 -8
  32. data/lib/dynflow/executors/parallel.rb +4 -34
  33. data/lib/dynflow/executors/parallel/core.rb +18 -118
  34. data/lib/dynflow/executors/parallel/pool.rb +2 -2
  35. data/lib/dynflow/executors/parallel/worker.rb +3 -11
  36. data/lib/dynflow/persistence_adapters/sequel.rb +1 -2
  37. data/lib/dynflow/testing.rb +2 -0
  38. data/lib/dynflow/testing/in_thread_executor.rb +52 -0
  39. data/lib/dynflow/testing/in_thread_world.rb +64 -0
  40. data/lib/dynflow/testing/managed_clock.rb +1 -1
  41. data/lib/dynflow/throttle_limiter.rb +1 -1
  42. data/lib/dynflow/version.rb +1 -1
  43. data/lib/dynflow/world.rb +13 -7
  44. data/test/abnormal_states_recovery_test.rb +10 -0
  45. data/test/action_test.rb +9 -9
  46. data/test/clock_test.rb +0 -2
  47. data/test/concurrency_control_test.rb +1 -1
  48. data/test/execution_plan_test.rb +0 -2
  49. data/test/executor_test.rb +6 -13
  50. data/test/support/code_workflow_example.rb +1 -1
  51. data/test/support/rescue_example.rb +0 -1
  52. data/test/test_helper.rb +9 -12
  53. data/test/testing_test.rb +74 -2
  54. data/web/views/plan_step.erb +2 -0
  55. metadata +11 -8
  56. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +0 -111
  57. data/lib/dynflow/executors/parallel/flow_manager.rb +0 -45
  58. data/lib/dynflow/executors/parallel/running_steps_manager.rb +0 -81
  59. data/lib/dynflow/executors/parallel/sequence_cursor.rb +0 -97
  60. data/lib/dynflow/executors/parallel/work_queue.rb +0 -50
@@ -2,7 +2,7 @@ module Dynflow
2
2
  module Dispatcher
3
3
  class ExecutorDispatcher < Abstract
4
4
  def initialize(world, semaphore)
5
- @world = Type! world, World
5
+ @world = Type! world, World
6
6
  @current_futures = Set.new
7
7
  end
8
8
 
@@ -35,17 +35,17 @@ module Dynflow
35
35
 
36
36
  # all params with default values are part of *private* api
37
37
  def initialize(world,
38
- id = SecureRandom.uuid,
39
- state = :pending,
40
- root_plan_step = nil,
41
- run_flow = Flows::Concurrence.new([]),
42
- finalize_flow = Flows::Sequence.new([]),
43
- steps = {},
44
- started_at = nil,
45
- ended_at = nil,
46
- execution_time = nil,
47
- real_time = 0.0,
48
- execution_history = ExecutionHistory.new)
38
+ id = SecureRandom.uuid,
39
+ state = :pending,
40
+ root_plan_step = nil,
41
+ run_flow = Flows::Concurrence.new([]),
42
+ finalize_flow = Flows::Sequence.new([]),
43
+ steps = {},
44
+ started_at = nil,
45
+ ended_at = nil,
46
+ execution_time = nil,
47
+ real_time = 0.0,
48
+ execution_history = ExecutionHistory.new)
49
49
 
50
50
  @id = Type! id, String
51
51
  @world = Type! world, World
@@ -133,6 +133,18 @@ module Dynflow
133
133
  end
134
134
  end
135
135
 
136
+ def plan_steps
137
+ steps_of_type(Dynflow::ExecutionPlan::Steps::PlanStep)
138
+ end
139
+
140
+ def run_steps
141
+ steps_of_type(Dynflow::ExecutionPlan::Steps::RunStep)
142
+ end
143
+
144
+ def finalize_steps
145
+ steps_of_type(Dynflow::ExecutionPlan::Steps::FinalizeStep)
146
+ end
147
+
136
148
  def failed_steps
137
149
  steps_in_state(:error)
138
150
  end
@@ -165,10 +177,10 @@ module Dynflow
165
177
  execution_history.add("delay", @world.id)
166
178
  serializer = root_plan_step.delay(delay_options, args)
167
179
  delayed_plan = DelayedPlan.new(@world,
168
- id,
169
- delay_options[:start_at],
170
- delay_options.fetch(:start_before, nil),
171
- serializer)
180
+ id,
181
+ delay_options[:start_at],
182
+ delay_options.fetch(:start_before, nil),
183
+ serializer)
172
184
  persistence.save_delayed_plan(delayed_plan)
173
185
  ensure
174
186
  update_state(error? ? :stopped : :scheduled)
@@ -263,6 +275,11 @@ module Dynflow
263
275
  end
264
276
 
265
277
  # @api private
278
+
279
+ def steps_of_type(type)
280
+ steps.values.find_all { |step| step.is_a?(type) }
281
+ end
282
+
266
283
  def current_run_flow
267
284
  @run_flow_stack.last
268
285
  end
@@ -9,18 +9,18 @@ module Dynflow
9
9
  attr_accessor :error
10
10
 
11
11
  def initialize(execution_plan_id,
12
- id,
13
- state,
14
- action_class,
15
- action_id,
16
- error,
17
- world,
18
- started_at = nil,
19
- ended_at = nil,
20
- execution_time = 0.0,
21
- real_time = 0.0,
22
- progress_done = nil,
23
- progress_weight = nil)
12
+ id,
13
+ state,
14
+ action_class,
15
+ action_id,
16
+ error,
17
+ world,
18
+ started_at = nil,
19
+ ended_at = nil,
20
+ execution_time = 0.0,
21
+ real_time = 0.0,
22
+ progress_done = nil,
23
+ progress_weight = nil)
24
24
 
25
25
  @id = id || raise(ArgumentError, 'missing id')
26
26
  @execution_plan_id = Type! execution_plan_id, String
@@ -155,9 +155,9 @@ module Dynflow
155
155
  block.call
156
156
  ensure
157
157
  @progress_done, @progress_weight = action.calculated_progress
158
- @ended_at = Time.now
158
+ @ended_at = Time.now
159
159
  @execution_time += @ended_at - start
160
- @real_time = @ended_at - @started_at
160
+ @real_time = @ended_at - @started_at
161
161
  end
162
162
  end
163
163
  end
@@ -55,7 +55,7 @@ module Dynflow
55
55
 
56
56
  def exception
57
57
  @exception ||
58
- exception_class.exception(message).tap { |e| e.set_backtrace backtrace }
58
+ exception_class.exception(message).tap { |e| e.set_backtrace backtrace }
59
59
  end
60
60
  end
61
61
  end
@@ -13,7 +13,6 @@ module Dynflow
13
13
  }
14
14
  end
15
15
 
16
-
17
16
  def phase
18
17
  Action::Finalize
19
18
  end
@@ -5,17 +5,17 @@ module Dynflow
5
5
 
6
6
  # @param [Array] children is a private API parameter
7
7
  def initialize(execution_plan_id,
8
- id,
9
- state,
10
- action_class,
11
- action_id,
12
- error,
13
- world,
14
- started_at = nil,
15
- ended_at = nil,
16
- execution_time = 0.0,
17
- real_time = 0.0,
18
- children = [])
8
+ id,
9
+ state,
10
+ action_class,
11
+ action_id,
12
+ error,
13
+ world,
14
+ started_at = nil,
15
+ ended_at = nil,
16
+ execution_time = 0.0,
17
+ real_time = 0.0,
18
+ children = [])
19
19
 
20
20
  super execution_plan_id, id, state, action_class, action_id, error, world, started_at,
21
21
  ended_at, execution_time, real_time
@@ -68,7 +68,6 @@ module Dynflow
68
68
  error: [] }
69
69
  end
70
70
 
71
-
72
71
  def self.new_from_hash(hash, execution_plan_id, world)
73
72
  check_class_matching hash
74
73
  new execution_plan_id,
@@ -20,7 +20,7 @@ module Dynflow
20
20
 
21
21
  def cancellable?
22
22
  [:suspended, :running].include?(self.state) &&
23
- self.action_class < Action::Cancellable
23
+ self.action_class < Action::Cancellable
24
24
  end
25
25
 
26
26
  def with_sub_plans?
@@ -1,13 +1,6 @@
1
1
  module Dynflow
2
2
  module Executors
3
3
  class Abstract
4
- Event = Algebrick.type do
5
- fields! execution_plan_id: String,
6
- step_id: Fixnum,
7
- event: Object,
8
- result: Concurrent::Edge::Future
9
- end
10
-
11
4
  include Algebrick::TypeCheck
12
5
  attr_reader :world, :logger
13
6
 
@@ -16,9 +9,13 @@ module Dynflow
16
9
  @logger = world.logger
17
10
  end
18
11
 
12
+ # @param execution_plan_id [String] id of execution plan
13
+ # @param finished [Concurrent::Edge::Future]
14
+ # @param wait_for_acceptance [TrueClass|FalseClass] should the executor confirm receiving
15
+ # the event, disable if calling executor from within executor
19
16
  # @return [Concurrent::Edge::Future]
20
17
  # @raise when execution_plan_id is not accepted
21
- def execute(execution_plan_id)
18
+ def execute(execution_plan_id, finished = Concurrent.future, wait_for_acceptance = true)
22
19
  raise NotImplementedError
23
20
  end
24
21
 
@@ -1,41 +1,10 @@
1
1
  module Dynflow
2
2
  module Executors
3
3
  class Parallel < Abstract
4
-
5
- require 'dynflow/executors/parallel/sequence_cursor'
6
- require 'dynflow/executors/parallel/flow_manager'
7
- require 'dynflow/executors/parallel/work_queue'
8
- require 'dynflow/executors/parallel/execution_plan_manager'
9
- require 'dynflow/executors/parallel/sequential_manager'
10
- require 'dynflow/executors/parallel/running_steps_manager'
11
4
  require 'dynflow/executors/parallel/core'
12
5
  require 'dynflow/executors/parallel/pool'
13
6
  require 'dynflow/executors/parallel/worker'
14
7
 
15
- UnprocessableEvent = Class.new(Dynflow::Error)
16
-
17
- Algebrick.type do |work|
18
- Work = work
19
-
20
- Work::Finalize = type do
21
- fields! sequential_manager: SequentialManager,
22
- execution_plan_id: String
23
- end
24
-
25
- Work::Step = type do
26
- fields! step: ExecutionPlan::Steps::AbstractFlowStep,
27
- execution_plan_id: String
28
- end
29
-
30
- Work::Event = type do
31
- fields! step: ExecutionPlan::Steps::AbstractFlowStep,
32
- execution_plan_id: String,
33
- event: Event
34
- end
35
-
36
- variants Work::Step, Work::Event, Work::Finalize
37
- end
38
-
39
8
  def initialize(world, pool_size = 10)
40
9
  super(world)
41
10
  @core = Core.spawn name: 'parallel-executor-core',
@@ -43,8 +12,9 @@ module Dynflow
43
12
  initialized: @core_initialized = Concurrent.future
44
13
  end
45
14
 
46
- def execute(execution_plan_id, finished = Concurrent.future)
47
- @core.ask([:handle_execution, execution_plan_id, finished]).value!
15
+ def execute(execution_plan_id, finished = Concurrent.future, wait_for_acceptance = true)
16
+ accepted = @core.ask([:handle_execution, execution_plan_id, finished])
17
+ accepted.value! if wait_for_acceptance
48
18
  finished
49
19
  rescue Concurrent::Actor::ActorTerminated => error
50
20
  dynflow_error = Dynflow::Error.new('executor terminated')
@@ -56,7 +26,7 @@ module Dynflow
56
26
  end
57
27
 
58
28
  def event(execution_plan_id, step_id, event, future = Concurrent.future)
59
- @core.ask([:handle_event, Event[execution_plan_id, step_id, event, future]])
29
+ @core.ask([:handle_event, Director::Event[execution_plan_id, step_id, event, future]])
60
30
  future
61
31
  end
62
32
 
@@ -6,38 +6,33 @@ module Dynflow
6
6
  attr_reader :logger
7
7
 
8
8
  def initialize(world, pool_size)
9
- @logger = world.logger
10
- @world = Type! world, World
11
- @pool = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
12
- @execution_plan_managers = {}
13
- @plan_ids_in_rescue = Set.new
14
- @terminated = nil
9
+ @logger = world.logger
10
+ @world = Type! world, World
11
+ @pool = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
12
+ @terminated = nil
13
+ @director = Director.new(@world)
15
14
  end
16
15
 
17
16
  def handle_execution(execution_plan_id, finished)
18
- start_executing track_execution_plan(execution_plan_id, finished)
17
+ if terminating?
18
+ raise Dynflow::Error,
19
+ "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
20
+ end
21
+
22
+ feed_pool(@director.start_execution(execution_plan_id, finished))
19
23
  end
20
24
 
21
25
  def handle_event(event)
22
- Type! event, Parallel::Event
26
+ Type! event, Director::Event
23
27
  if terminating?
24
28
  raise Dynflow::Error,
25
29
  "cannot accept event: #{event} core is terminating"
26
30
  end
27
- execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
28
- if execution_plan_manager
29
- feed_pool execution_plan_manager.event(event)
30
- true
31
- else
32
- raise Dynflow::Error, "no manager for #{event.inspect}"
33
- end
34
- rescue Dynflow::Error => e
35
- event.result.fail e.message
36
- raise e
31
+ feed_pool(@director.handle_event(event))
37
32
  end
38
33
 
39
- def finish_step(step)
40
- update_manager(step)
34
+ def work_finished(work)
35
+ feed_pool(@director.work_finished(work))
41
36
  end
42
37
 
43
38
  def handle_persistence_error(error)
@@ -53,19 +48,7 @@ module Dynflow
53
48
  end
54
49
 
55
50
  def finish_termination
56
- unless @execution_plan_managers.empty?
57
- logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
58
- begin
59
- @execution_plan_managers.values.each do |manager|
60
- manager.terminate
61
- end
62
- rescue Errors::PersistenceError
63
- logger.error "could not to clean the data properly"
64
- end
65
- @execution_plan_managers.values.each do |manager|
66
- finish_plan(manager.execution_plan.id)
67
- end
68
- end
51
+ @director.terminate
69
52
  logger.error '... core terminated.'
70
53
  super
71
54
  end
@@ -78,96 +61,13 @@ module Dynflow
78
61
  self.tell(:handle_persistence_error, e)
79
62
  end
80
63
 
81
- # @return
82
- def track_execution_plan(execution_plan_id, finished)
83
- execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
84
-
85
- if terminating?
86
- raise Dynflow::Error,
87
- "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
88
- end
89
-
90
- if @execution_plan_managers[execution_plan_id]
91
- raise Dynflow::Error,
92
- "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
93
- end
94
-
95
- if execution_plan.state == :stopped
96
- raise Dynflow::Error,
97
- "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
98
- end
99
-
100
- @execution_plan_managers[execution_plan_id] =
101
- ExecutionPlanManager.new(@world, execution_plan, finished)
102
-
103
- rescue Dynflow::Error => e
104
- finished.fail e
105
- nil
106
- end
107
-
108
- def update_manager(finished_work)
109
- manager = @execution_plan_managers[finished_work.execution_plan_id]
110
- next_work = manager.what_is_next(finished_work)
111
- continue_manager manager, next_work
112
- end
113
-
114
- def continue_manager(manager, next_work)
115
- if manager.done?
116
- finish_plan manager.execution_plan.id
117
- else
118
- feed_pool next_work
119
- end
120
- end
121
-
122
- def rescue?(manager)
123
- return false if terminating?
124
- @world.auto_rescue && manager.execution_plan.state == :paused &&
125
- !@plan_ids_in_rescue.include?(manager.execution_plan.id)
126
- end
127
-
128
- def rescue!(manager)
129
- # TODO: after moving to concurrent-ruby actors, there should be better place
130
- # to put this logic of making sure we don't run rescues in endless loop
131
- @plan_ids_in_rescue << manager.execution_plan.id
132
- rescue_plan_id = manager.execution_plan.rescue_plan_id
133
- if rescue_plan_id
134
- reference.tell([:handle_execution, rescue_plan_id, manager.future])
135
- else
136
- set_future(manager)
137
- end
138
- end
139
-
140
64
  def feed_pool(work_items)
141
65
  return if terminating?
142
- Type! work_items, Array, Work, NilClass
143
66
  return if work_items.nil?
144
- work_items = [work_items] if work_items.is_a? Work
145
- work_items.all? { |i| Type! i, Work }
67
+ work_items = [work_items] if work_items.is_a? Director::WorkItem
68
+ work_items.all? { |i| Type! i, Director::WorkItem }
146
69
  work_items.each { |new_work| @pool.tell([:schedule_work, new_work]) }
147
70
  end
148
-
149
- def finish_plan(execution_plan_id)
150
- manager = @execution_plan_managers.delete(execution_plan_id)
151
- if rescue?(manager)
152
- rescue!(manager)
153
- else
154
- set_future(manager)
155
- end
156
- end
157
-
158
- def set_future(manager)
159
- @plan_ids_in_rescue.delete(manager.execution_plan.id)
160
- manager.future.success manager.execution_plan
161
- end
162
-
163
- def start_executing(manager)
164
- return if manager.nil?
165
- Type! manager, ExecutionPlanManager
166
-
167
- next_work = manager.start
168
- continue_manager manager, next_work
169
- end
170
-
171
71
  end
172
72
  end
173
73
  end