dynflow 0.8.16 → 0.8.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +21 -0
  3. data/.rubocop_todo.yml +0 -25
  4. data/doc/pages/plugins/div_tag.rb +1 -1
  5. data/doc/pages/plugins/tags.rb +0 -1
  6. data/examples/orchestrate.rb +0 -1
  7. data/examples/remote_executor.rb +3 -3
  8. data/examples/sub_plan_concurrency_control.rb +0 -1
  9. data/examples/sub_plans.rb +0 -1
  10. data/lib/dynflow.rb +1 -0
  11. data/lib/dynflow/action.rb +6 -6
  12. data/lib/dynflow/config.rb +2 -2
  13. data/lib/dynflow/connectors/database.rb +1 -1
  14. data/lib/dynflow/connectors/direct.rb +1 -1
  15. data/lib/dynflow/coordinator.rb +4 -4
  16. data/lib/dynflow/director.rb +190 -0
  17. data/lib/dynflow/director/execution_plan_manager.rb +107 -0
  18. data/lib/dynflow/director/flow_manager.rb +43 -0
  19. data/lib/dynflow/director/running_steps_manager.rb +79 -0
  20. data/lib/dynflow/director/sequence_cursor.rb +91 -0
  21. data/lib/dynflow/{executors/parallel → director}/sequential_manager.rb +2 -2
  22. data/lib/dynflow/director/work_queue.rb +48 -0
  23. data/lib/dynflow/dispatcher/client_dispatcher.rb +24 -24
  24. data/lib/dynflow/dispatcher/executor_dispatcher.rb +1 -1
  25. data/lib/dynflow/execution_plan.rb +32 -15
  26. data/lib/dynflow/execution_plan/steps/abstract.rb +14 -14
  27. data/lib/dynflow/execution_plan/steps/error.rb +1 -1
  28. data/lib/dynflow/execution_plan/steps/finalize_step.rb +0 -1
  29. data/lib/dynflow/execution_plan/steps/plan_step.rb +11 -12
  30. data/lib/dynflow/execution_plan/steps/run_step.rb +1 -1
  31. data/lib/dynflow/executors/abstract.rb +5 -8
  32. data/lib/dynflow/executors/parallel.rb +4 -34
  33. data/lib/dynflow/executors/parallel/core.rb +18 -118
  34. data/lib/dynflow/executors/parallel/pool.rb +2 -2
  35. data/lib/dynflow/executors/parallel/worker.rb +3 -11
  36. data/lib/dynflow/persistence_adapters/sequel.rb +1 -2
  37. data/lib/dynflow/testing.rb +2 -0
  38. data/lib/dynflow/testing/in_thread_executor.rb +52 -0
  39. data/lib/dynflow/testing/in_thread_world.rb +64 -0
  40. data/lib/dynflow/testing/managed_clock.rb +1 -1
  41. data/lib/dynflow/throttle_limiter.rb +1 -1
  42. data/lib/dynflow/version.rb +1 -1
  43. data/lib/dynflow/world.rb +13 -7
  44. data/test/abnormal_states_recovery_test.rb +10 -0
  45. data/test/action_test.rb +9 -9
  46. data/test/clock_test.rb +0 -2
  47. data/test/concurrency_control_test.rb +1 -1
  48. data/test/execution_plan_test.rb +0 -2
  49. data/test/executor_test.rb +6 -13
  50. data/test/support/code_workflow_example.rb +1 -1
  51. data/test/support/rescue_example.rb +0 -1
  52. data/test/test_helper.rb +9 -12
  53. data/test/testing_test.rb +74 -2
  54. data/web/views/plan_step.erb +2 -0
  55. metadata +11 -8
  56. data/lib/dynflow/executors/parallel/execution_plan_manager.rb +0 -111
  57. data/lib/dynflow/executors/parallel/flow_manager.rb +0 -45
  58. data/lib/dynflow/executors/parallel/running_steps_manager.rb +0 -81
  59. data/lib/dynflow/executors/parallel/sequence_cursor.rb +0 -97
  60. data/lib/dynflow/executors/parallel/work_queue.rb +0 -50
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b6082a4e18537f19b13eaf0409be81baaf334a5c
4
- data.tar.gz: 5c9d56842579de336c97b7f00d4284ef254ba196
3
+ metadata.gz: 15cfcb2e4498b8f1032b1f36e3023aa469cbe313
4
+ data.tar.gz: 62d51d3df0f51ce41ab27692a1af5ca97ca88687
5
5
  SHA512:
6
- metadata.gz: 6d24e6111ffac1b44316a31784afbe1ace05fd130e9c087bff718adfb33335fe2a7c747f176557b15999fd0b7900af1367128606452a7ce94c345894f5707f02
7
- data.tar.gz: d7d3655784db8d993018d5c5e00c3dbebaf4679ae2bacf971eb129ea3e306854b97a803fa6098694ad5d79102aea2f70b456e481ec1af4a90469da71efe0f0a4
6
+ metadata.gz: 7e484ef3b5286395ca76547a08fab9d552ee558b7c3c0825bf262dfc53b7d48169aca85b615a954354c230826853343fb60c44ef63d6aa72d4f787ed811a7d41
7
+ data.tar.gz: 345656ec25bee35465bd8551108072cafd2ddfd0dd8ee10006da92e74ae1564eecd13f0b08e19649b0aa51a3458fb58ef06a690cbf2a2bc95a0eb6050f824956
data/.rubocop.yml CHANGED
@@ -1 +1,22 @@
1
1
  inherit_from: .rubocop_todo.yml
2
+
3
+ # Cop supports --auto-correct.
4
+ # Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
5
+ Style/ExtraSpacing:
6
+ Enabled: true
7
+
8
+ # Cop supports --auto-correct.
9
+ # Configuration parameters: EnforcedStyle, SupportedStyles.
10
+ # SupportedStyles: with_first_parameter, with_fixed_indentation
11
+ Style/AlignParameters:
12
+ Enabled: true
13
+
14
+ # Cop supports --auto-correct.
15
+ # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
16
+ # SupportedStyles: aligned, indented
17
+ Style/MultilineOperationIndentation:
18
+ Enabled: true
19
+
20
+ # Cop supports --auto-correct.
21
+ Style/EmptyLines:
22
+ Enabled: true
data/.rubocop_todo.yml CHANGED
@@ -147,13 +147,6 @@ Style/Alias:
147
147
  Style/AlignHash:
148
148
  Enabled: false
149
149
 
150
- # Offense count: 51
151
- # Cop supports --auto-correct.
152
- # Configuration parameters: EnforcedStyle, SupportedStyles.
153
- # SupportedStyles: with_first_parameter, with_fixed_indentation
154
- Style/AlignParameters:
155
- Enabled: false
156
-
157
150
  # Offense count: 11
158
151
  # Cop supports --auto-correct.
159
152
  # Configuration parameters: EnforcedStyle, SupportedStyles.
@@ -255,11 +248,6 @@ Style/EmptyElse:
255
248
  Style/EmptyLineBetweenDefs:
256
249
  Enabled: false
257
250
 
258
- # Offense count: 14
259
- # Cop supports --auto-correct.
260
- Style/EmptyLines:
261
- Enabled: false
262
-
263
251
  # Offense count: 1
264
252
  # Cop supports --auto-correct.
265
253
  Style/EmptyLinesAroundAccessModifier:
@@ -291,12 +279,6 @@ Style/EmptyLinesAroundMethodBody:
291
279
  Style/EmptyLinesAroundModuleBody:
292
280
  Enabled: false
293
281
 
294
- # Offense count: 25
295
- # Cop supports --auto-correct.
296
- # Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
297
- Style/ExtraSpacing:
298
- Enabled: false
299
-
300
282
  # Offense count: 9
301
283
  # Cop supports --auto-correct.
302
284
  # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
@@ -395,13 +377,6 @@ Style/MultilineIfThen:
395
377
  Style/MultilineMethodCallIndentation:
396
378
  Enabled: false
397
379
 
398
- # Offense count: 9
399
- # Cop supports --auto-correct.
400
- # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
401
- # SupportedStyles: aligned, indented
402
- Style/MultilineOperationIndentation:
403
- Enabled: false
404
-
405
380
  # Offense count: 9
406
381
  # Cop supports --auto-correct.
407
382
  Style/MutableConstant:
@@ -1,7 +1,7 @@
1
1
  module Jekyll
2
2
  class DivTag < Liquid::Block
3
3
  def render(context)
4
- content = super
4
+ content = super
5
5
 
6
6
  <<-HTML.gsub(/^ +\|/, '')
7
7
  |<#{tag} class="#{@markup}">
@@ -65,7 +65,6 @@ module Jekyll
65
65
  [tag.to_s, range < (size = posts.size) ? range = size : size]
66
66
  }
67
67
 
68
-
69
68
  range = 1..range
70
69
 
71
70
  tags.sort!.map! { |tag, size| [tag, range.quantile(size, num)] }
@@ -24,7 +24,6 @@ require_relative 'example_helper'
24
24
 
25
25
  module Orchestrate
26
26
 
27
-
28
27
  class CreateInfrastructure < Dynflow::Action
29
28
 
30
29
  def plan
@@ -21,7 +21,7 @@ class RemoteExecutorExample
21
21
  class << self
22
22
 
23
23
  def run_observer
24
- world = ExampleHelper.create_world do |config|
24
+ world = ExampleHelper.create_world do |config|
25
25
  config.persistence_adapter = persistence_adapter
26
26
  config.connector = connector
27
27
  config.executor = false
@@ -30,7 +30,7 @@ class RemoteExecutorExample
30
30
  end
31
31
 
32
32
  def run_server
33
- world = ExampleHelper.create_world do |config|
33
+ world = ExampleHelper.create_world do |config|
34
34
  config.persistence_adapter = persistence_adapter
35
35
  config.connector = connector
36
36
  end
@@ -65,7 +65,7 @@ class RemoteExecutorExample
65
65
  end
66
66
 
67
67
  def run_client
68
- world = ExampleHelper.create_world do |config|
68
+ world = ExampleHelper.create_world do |config|
69
69
  config.persistence_adapter = persistence_adapter
70
70
  config.executor = false
71
71
  config.connector = connector
@@ -38,7 +38,6 @@ class CostyAction < Dynflow::Action
38
38
  end
39
39
  end
40
40
 
41
-
42
41
  class ConcurrencyControlExample < Dynflow::Action
43
42
  include Dynflow::Action::WithSubPlans
44
43
 
@@ -16,7 +16,6 @@ DESC
16
16
  require_relative 'example_helper'
17
17
  require_relative 'orchestrate_evented'
18
18
 
19
-
20
19
  class SubPlansExample < Dynflow::Action
21
20
  include Dynflow::Action::WithSubPlans
22
21
 
data/lib/dynflow.rb CHANGED
@@ -38,6 +38,7 @@ module Dynflow
38
38
  require 'dynflow/execution_plan'
39
39
  require 'dynflow/delayed_plan'
40
40
  require 'dynflow/action'
41
+ require 'dynflow/director'
41
42
  require 'dynflow/executors'
42
43
  require 'dynflow/logger_adapters'
43
44
  require 'dynflow/world'
@@ -48,9 +48,10 @@ module Dynflow
48
48
  nil
49
49
  end
50
50
 
51
- ERROR = Object.new
52
- SUSPEND = Object.new
53
- Phase = Algebrick.type do
51
+ ERROR = Object.new
52
+ SUSPEND = Object.new
53
+ Skip = Algebrick.atom
54
+ Phase = Algebrick.type do
54
55
  Executable = type do
55
56
  variants Plan = atom,
56
57
  Run = atom,
@@ -58,7 +59,6 @@ module Dynflow
58
59
  end
59
60
  variants Executable, Present = atom
60
61
  end
61
- Skip = Algebrick.atom
62
62
 
63
63
  module Executable
64
64
  def execute_method_name
@@ -119,7 +119,7 @@ module Dynflow
119
119
 
120
120
  def phase!(*phases)
121
121
  phase?(*phases) or
122
- raise TypeError, "Wrong phase #{phase}, required #{phases}"
122
+ raise TypeError, "Wrong phase #{phase}, required #{phases}"
123
123
  end
124
124
 
125
125
  def input=(hash)
@@ -137,7 +137,7 @@ module Dynflow
137
137
  def output
138
138
  if phase? Plan
139
139
  @output_reference or
140
- raise 'plan_self has to be invoked before being able to reference the output'
140
+ raise 'plan_self has to be invoked before being able to reference the output'
141
141
  else
142
142
  @output
143
143
  end
@@ -120,8 +120,8 @@ module Dynflow
120
120
  ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
121
121
  if (config_for_world.pool_size / 2.0) > ar_pool_size
122
122
  config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
123
- "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
124
- 'threads in Dynflow pool.'
123
+ "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
124
+ 'threads in Dynflow pool.'
125
125
  end
126
126
  end
127
127
  end
@@ -151,7 +151,7 @@ module Dynflow
151
151
  1
152
152
  end
153
153
  end
154
- @core = Core.spawn('connector-database-core', self, polling_interval)
154
+ @core = Core.spawn('connector-database-core', self, polling_interval)
155
155
  start_listening(world) if world
156
156
  end
157
157
 
@@ -47,7 +47,7 @@ module Dynflow
47
47
  end
48
48
 
49
49
  def initialize(world = nil)
50
- @core = Core.spawn('connector-direct-core', self)
50
+ @core = Core.spawn('connector-direct-core', self)
51
51
  start_listening(world) if world
52
52
  end
53
53
 
@@ -60,8 +60,8 @@ module Dynflow
60
60
  # @api override
61
61
  # check to be performed before we try to acquire the lock
62
62
  def validate!
63
- Type! id, String
64
- Type! @data, Hash
63
+ Type! id, String
64
+ Type! @data, Hash
65
65
  raise "The record id %{s} too large" % id if id.size > 100
66
66
  raise "The record class name %{s} too large" % self.class.name if self.class.name.size > 100
67
67
  end
@@ -94,7 +94,7 @@ module Dynflow
94
94
  class ExecutorWorld < WorldRecord
95
95
  def initialize(world)
96
96
  super
97
- self.active = !world.terminating?
97
+ self.active = !world.terminating?
98
98
  end
99
99
 
100
100
  def active?
@@ -143,7 +143,7 @@ module Dynflow
143
143
  def initialize(world)
144
144
  super
145
145
  @world = world
146
- @data.merge!(owner_id: "world:#{world.id}", world_id: world.id)
146
+ @data.merge!(owner_id: "world:#{world.id}", world_id: world.id)
147
147
  end
148
148
 
149
149
  def self.lock_id(*args)
@@ -0,0 +1,190 @@
1
+ module Dynflow
2
+ # Director is responsible for telling what to do next when:
3
+ # * new execution starts
4
+ # * an event accurs
5
+ # * some work is finished
6
+ #
7
+ # It's public methods (except terminate) return work items that the
8
+ # executor should understand
9
+ class Director
10
+ include Algebrick::TypeCheck
11
+
12
+ Event = Algebrick.type do
13
+ fields! execution_plan_id: String,
14
+ step_id: Fixnum,
15
+ event: Object,
16
+ result: Concurrent::Edge::Future
17
+ end
18
+
19
+ UnprocessableEvent = Class.new(Dynflow::Error)
20
+
21
+ class WorkItem
22
+ attr_reader :execution_plan_id
23
+
24
+ def initialize(execution_plan_id)
25
+ @execution_plan_id = execution_plan_id
26
+ end
27
+
28
+ def execute
29
+ raise NotImplementedError
30
+ end
31
+ end
32
+
33
+ class StepWorkItem < WorkItem
34
+ attr_reader :step
35
+
36
+ def initialize(execution_plan_id, step)
37
+ super(execution_plan_id)
38
+ @step = step
39
+ end
40
+
41
+ def execute
42
+ @step.execute(nil)
43
+ end
44
+ end
45
+
46
+ class EventWorkItem < StepWorkItem
47
+ attr_reader :event
48
+
49
+ def initialize(execution_plan_id, step, event)
50
+ super(execution_plan_id, step)
51
+ @event = event
52
+ end
53
+
54
+ def execute
55
+ @step.execute(@event.event)
56
+ end
57
+ end
58
+
59
+ class FinalizeWorkItem < WorkItem
60
+ def initialize(execution_plan_id, sequential_manager)
61
+ super(execution_plan_id)
62
+ @sequential_manager = sequential_manager
63
+ end
64
+
65
+ def execute
66
+ @sequential_manager.finalize
67
+ end
68
+ end
69
+
70
+ require 'dynflow/director/work_queue'
71
+ require 'dynflow/director/sequence_cursor'
72
+ require 'dynflow/director/flow_manager'
73
+ require 'dynflow/director/execution_plan_manager'
74
+ require 'dynflow/director/sequential_manager'
75
+ require 'dynflow/director/running_steps_manager'
76
+
77
+ attr_reader :logger
78
+
79
+ def initialize(world)
80
+ @world = world
81
+ @logger = world.logger
82
+ @execution_plan_managers = {}
83
+ @plan_ids_in_rescue = Set.new
84
+ end
85
+
86
+ def start_execution(execution_plan_id, finished)
87
+ manager = track_execution_plan(execution_plan_id, finished)
88
+ return [] unless manager
89
+ unless_done(manager, manager.start)
90
+ end
91
+
92
+ def handle_event(event)
93
+ Type! event, Event
94
+ execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
95
+ if execution_plan_manager
96
+ execution_plan_manager.event(event)
97
+ else
98
+ raise Dynflow::Error, "no manager for #{event.inspect}"
99
+ end
100
+ rescue Dynflow::Error => e
101
+ event.result.fail e.message
102
+ raise e
103
+ end
104
+
105
+ def work_finished(work)
106
+ manager = @execution_plan_managers[work.execution_plan_id]
107
+ unless_done(manager, manager.what_is_next(work))
108
+ end
109
+
110
+ def terminate
111
+ unless @execution_plan_managers.empty?
112
+ logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
113
+ begin
114
+ @execution_plan_managers.values.each do |manager|
115
+ manager.terminate
116
+ end
117
+ rescue Errors::PersistenceError
118
+ logger.error "could not to clean the data properly"
119
+ end
120
+ @execution_plan_managers.values.each do |manager|
121
+ finish_manager(manager)
122
+ end
123
+ end
124
+ end
125
+
126
+ private
127
+
128
+ def unless_done(manager, work_items)
129
+ return [] unless manager
130
+ if manager.done?
131
+ finish_manager(manager)
132
+ return []
133
+ else
134
+ return work_items
135
+ end
136
+ end
137
+
138
+ def finish_manager(manager)
139
+ @execution_plan_managers.delete(manager.execution_plan.id)
140
+ if rescue?(manager)
141
+ rescue!(manager)
142
+ else
143
+ set_future(manager)
144
+ end
145
+ end
146
+
147
+ def rescue?(manager)
148
+ return false if @world.terminating?
149
+ @world.auto_rescue && manager.execution_plan.state == :paused &&
150
+ !@plan_ids_in_rescue.include?(manager.execution_plan.id)
151
+ end
152
+
153
+ def rescue!(manager)
154
+ # TODO: after moving to concurrent-ruby actors, there should be better place
155
+ # to put this logic of making sure we don't run rescues in endless loop
156
+ @plan_ids_in_rescue << manager.execution_plan.id
157
+ rescue_plan_id = manager.execution_plan.rescue_plan_id
158
+ if rescue_plan_id
159
+ @world.executor.execute(rescue_plan_id, manager.future, false)
160
+ else
161
+ set_future(manager)
162
+ end
163
+ end
164
+
165
+ def track_execution_plan(execution_plan_id, finished)
166
+ execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
167
+
168
+ if @execution_plan_managers[execution_plan_id]
169
+ raise Dynflow::Error,
170
+ "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
171
+ end
172
+
173
+ if execution_plan.state == :stopped
174
+ raise Dynflow::Error,
175
+ "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
176
+ end
177
+
178
+ @execution_plan_managers[execution_plan_id] =
179
+ ExecutionPlanManager.new(@world, execution_plan, finished)
180
+ rescue Dynflow::Error => e
181
+ finished.fail e
182
+ nil
183
+ end
184
+
185
+ def set_future(manager)
186
+ @plan_ids_in_rescue.delete(manager.execution_plan.id)
187
+ manager.future.success manager.execution_plan
188
+ end
189
+ end
190
+ end