dynflow 0.8.16 → 0.8.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +21 -0
- data/.rubocop_todo.yml +0 -25
- data/doc/pages/plugins/div_tag.rb +1 -1
- data/doc/pages/plugins/tags.rb +0 -1
- data/examples/orchestrate.rb +0 -1
- data/examples/remote_executor.rb +3 -3
- data/examples/sub_plan_concurrency_control.rb +0 -1
- data/examples/sub_plans.rb +0 -1
- data/lib/dynflow.rb +1 -0
- data/lib/dynflow/action.rb +6 -6
- data/lib/dynflow/config.rb +2 -2
- data/lib/dynflow/connectors/database.rb +1 -1
- data/lib/dynflow/connectors/direct.rb +1 -1
- data/lib/dynflow/coordinator.rb +4 -4
- data/lib/dynflow/director.rb +190 -0
- data/lib/dynflow/director/execution_plan_manager.rb +107 -0
- data/lib/dynflow/director/flow_manager.rb +43 -0
- data/lib/dynflow/director/running_steps_manager.rb +79 -0
- data/lib/dynflow/director/sequence_cursor.rb +91 -0
- data/lib/dynflow/{executors/parallel → director}/sequential_manager.rb +2 -2
- data/lib/dynflow/director/work_queue.rb +48 -0
- data/lib/dynflow/dispatcher/client_dispatcher.rb +24 -24
- data/lib/dynflow/dispatcher/executor_dispatcher.rb +1 -1
- data/lib/dynflow/execution_plan.rb +32 -15
- data/lib/dynflow/execution_plan/steps/abstract.rb +14 -14
- data/lib/dynflow/execution_plan/steps/error.rb +1 -1
- data/lib/dynflow/execution_plan/steps/finalize_step.rb +0 -1
- data/lib/dynflow/execution_plan/steps/plan_step.rb +11 -12
- data/lib/dynflow/execution_plan/steps/run_step.rb +1 -1
- data/lib/dynflow/executors/abstract.rb +5 -8
- data/lib/dynflow/executors/parallel.rb +4 -34
- data/lib/dynflow/executors/parallel/core.rb +18 -118
- data/lib/dynflow/executors/parallel/pool.rb +2 -2
- data/lib/dynflow/executors/parallel/worker.rb +3 -11
- data/lib/dynflow/persistence_adapters/sequel.rb +1 -2
- data/lib/dynflow/testing.rb +2 -0
- data/lib/dynflow/testing/in_thread_executor.rb +52 -0
- data/lib/dynflow/testing/in_thread_world.rb +64 -0
- data/lib/dynflow/testing/managed_clock.rb +1 -1
- data/lib/dynflow/throttle_limiter.rb +1 -1
- data/lib/dynflow/version.rb +1 -1
- data/lib/dynflow/world.rb +13 -7
- data/test/abnormal_states_recovery_test.rb +10 -0
- data/test/action_test.rb +9 -9
- data/test/clock_test.rb +0 -2
- data/test/concurrency_control_test.rb +1 -1
- data/test/execution_plan_test.rb +0 -2
- data/test/executor_test.rb +6 -13
- data/test/support/code_workflow_example.rb +1 -1
- data/test/support/rescue_example.rb +0 -1
- data/test/test_helper.rb +9 -12
- data/test/testing_test.rb +74 -2
- data/web/views/plan_step.erb +2 -0
- metadata +11 -8
- data/lib/dynflow/executors/parallel/execution_plan_manager.rb +0 -111
- data/lib/dynflow/executors/parallel/flow_manager.rb +0 -45
- data/lib/dynflow/executors/parallel/running_steps_manager.rb +0 -81
- data/lib/dynflow/executors/parallel/sequence_cursor.rb +0 -97
- data/lib/dynflow/executors/parallel/work_queue.rb +0 -50
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15cfcb2e4498b8f1032b1f36e3023aa469cbe313
|
4
|
+
data.tar.gz: 62d51d3df0f51ce41ab27692a1af5ca97ca88687
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7e484ef3b5286395ca76547a08fab9d552ee558b7c3c0825bf262dfc53b7d48169aca85b615a954354c230826853343fb60c44ef63d6aa72d4f787ed811a7d41
|
7
|
+
data.tar.gz: 345656ec25bee35465bd8551108072cafd2ddfd0dd8ee10006da92e74ae1564eecd13f0b08e19649b0aa51a3458fb58ef06a690cbf2a2bc95a0eb6050f824956
|
data/.rubocop.yml
CHANGED
@@ -1 +1,22 @@
|
|
1
1
|
inherit_from: .rubocop_todo.yml
|
2
|
+
|
3
|
+
# Cop supports --auto-correct.
|
4
|
+
# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
|
5
|
+
Style/ExtraSpacing:
|
6
|
+
Enabled: true
|
7
|
+
|
8
|
+
# Cop supports --auto-correct.
|
9
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
10
|
+
# SupportedStyles: with_first_parameter, with_fixed_indentation
|
11
|
+
Style/AlignParameters:
|
12
|
+
Enabled: true
|
13
|
+
|
14
|
+
# Cop supports --auto-correct.
|
15
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
|
16
|
+
# SupportedStyles: aligned, indented
|
17
|
+
Style/MultilineOperationIndentation:
|
18
|
+
Enabled: true
|
19
|
+
|
20
|
+
# Cop supports --auto-correct.
|
21
|
+
Style/EmptyLines:
|
22
|
+
Enabled: true
|
data/.rubocop_todo.yml
CHANGED
@@ -147,13 +147,6 @@ Style/Alias:
|
|
147
147
|
Style/AlignHash:
|
148
148
|
Enabled: false
|
149
149
|
|
150
|
-
# Offense count: 51
|
151
|
-
# Cop supports --auto-correct.
|
152
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
153
|
-
# SupportedStyles: with_first_parameter, with_fixed_indentation
|
154
|
-
Style/AlignParameters:
|
155
|
-
Enabled: false
|
156
|
-
|
157
150
|
# Offense count: 11
|
158
151
|
# Cop supports --auto-correct.
|
159
152
|
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
@@ -255,11 +248,6 @@ Style/EmptyElse:
|
|
255
248
|
Style/EmptyLineBetweenDefs:
|
256
249
|
Enabled: false
|
257
250
|
|
258
|
-
# Offense count: 14
|
259
|
-
# Cop supports --auto-correct.
|
260
|
-
Style/EmptyLines:
|
261
|
-
Enabled: false
|
262
|
-
|
263
251
|
# Offense count: 1
|
264
252
|
# Cop supports --auto-correct.
|
265
253
|
Style/EmptyLinesAroundAccessModifier:
|
@@ -291,12 +279,6 @@ Style/EmptyLinesAroundMethodBody:
|
|
291
279
|
Style/EmptyLinesAroundModuleBody:
|
292
280
|
Enabled: false
|
293
281
|
|
294
|
-
# Offense count: 25
|
295
|
-
# Cop supports --auto-correct.
|
296
|
-
# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
|
297
|
-
Style/ExtraSpacing:
|
298
|
-
Enabled: false
|
299
|
-
|
300
282
|
# Offense count: 9
|
301
283
|
# Cop supports --auto-correct.
|
302
284
|
# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
|
@@ -395,13 +377,6 @@ Style/MultilineIfThen:
|
|
395
377
|
Style/MultilineMethodCallIndentation:
|
396
378
|
Enabled: false
|
397
379
|
|
398
|
-
# Offense count: 9
|
399
|
-
# Cop supports --auto-correct.
|
400
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
|
401
|
-
# SupportedStyles: aligned, indented
|
402
|
-
Style/MultilineOperationIndentation:
|
403
|
-
Enabled: false
|
404
|
-
|
405
380
|
# Offense count: 9
|
406
381
|
# Cop supports --auto-correct.
|
407
382
|
Style/MutableConstant:
|
data/doc/pages/plugins/tags.rb
CHANGED
data/examples/orchestrate.rb
CHANGED
data/examples/remote_executor.rb
CHANGED
@@ -21,7 +21,7 @@ class RemoteExecutorExample
|
|
21
21
|
class << self
|
22
22
|
|
23
23
|
def run_observer
|
24
|
-
world
|
24
|
+
world = ExampleHelper.create_world do |config|
|
25
25
|
config.persistence_adapter = persistence_adapter
|
26
26
|
config.connector = connector
|
27
27
|
config.executor = false
|
@@ -30,7 +30,7 @@ class RemoteExecutorExample
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def run_server
|
33
|
-
world
|
33
|
+
world = ExampleHelper.create_world do |config|
|
34
34
|
config.persistence_adapter = persistence_adapter
|
35
35
|
config.connector = connector
|
36
36
|
end
|
@@ -65,7 +65,7 @@ class RemoteExecutorExample
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def run_client
|
68
|
-
world
|
68
|
+
world = ExampleHelper.create_world do |config|
|
69
69
|
config.persistence_adapter = persistence_adapter
|
70
70
|
config.executor = false
|
71
71
|
config.connector = connector
|
data/examples/sub_plans.rb
CHANGED
data/lib/dynflow.rb
CHANGED
data/lib/dynflow/action.rb
CHANGED
@@ -48,9 +48,10 @@ module Dynflow
|
|
48
48
|
nil
|
49
49
|
end
|
50
50
|
|
51
|
-
ERROR
|
52
|
-
SUSPEND
|
53
|
-
|
51
|
+
ERROR = Object.new
|
52
|
+
SUSPEND = Object.new
|
53
|
+
Skip = Algebrick.atom
|
54
|
+
Phase = Algebrick.type do
|
54
55
|
Executable = type do
|
55
56
|
variants Plan = atom,
|
56
57
|
Run = atom,
|
@@ -58,7 +59,6 @@ module Dynflow
|
|
58
59
|
end
|
59
60
|
variants Executable, Present = atom
|
60
61
|
end
|
61
|
-
Skip = Algebrick.atom
|
62
62
|
|
63
63
|
module Executable
|
64
64
|
def execute_method_name
|
@@ -119,7 +119,7 @@ module Dynflow
|
|
119
119
|
|
120
120
|
def phase!(*phases)
|
121
121
|
phase?(*phases) or
|
122
|
-
|
122
|
+
raise TypeError, "Wrong phase #{phase}, required #{phases}"
|
123
123
|
end
|
124
124
|
|
125
125
|
def input=(hash)
|
@@ -137,7 +137,7 @@ module Dynflow
|
|
137
137
|
def output
|
138
138
|
if phase? Plan
|
139
139
|
@output_reference or
|
140
|
-
|
140
|
+
raise 'plan_self has to be invoked before being able to reference the output'
|
141
141
|
else
|
142
142
|
@output
|
143
143
|
end
|
data/lib/dynflow/config.rb
CHANGED
@@ -120,8 +120,8 @@ module Dynflow
|
|
120
120
|
ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
|
121
121
|
if (config_for_world.pool_size / 2.0) > ar_pool_size
|
122
122
|
config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
|
123
|
-
|
124
|
-
|
123
|
+
"it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
|
124
|
+
'threads in Dynflow pool.'
|
125
125
|
end
|
126
126
|
end
|
127
127
|
end
|
data/lib/dynflow/coordinator.rb
CHANGED
@@ -60,8 +60,8 @@ module Dynflow
|
|
60
60
|
# @api override
|
61
61
|
# check to be performed before we try to acquire the lock
|
62
62
|
def validate!
|
63
|
-
Type! id,
|
64
|
-
Type! @data,
|
63
|
+
Type! id, String
|
64
|
+
Type! @data, Hash
|
65
65
|
raise "The record id %{s} too large" % id if id.size > 100
|
66
66
|
raise "The record class name %{s} too large" % self.class.name if self.class.name.size > 100
|
67
67
|
end
|
@@ -94,7 +94,7 @@ module Dynflow
|
|
94
94
|
class ExecutorWorld < WorldRecord
|
95
95
|
def initialize(world)
|
96
96
|
super
|
97
|
-
self.active
|
97
|
+
self.active = !world.terminating?
|
98
98
|
end
|
99
99
|
|
100
100
|
def active?
|
@@ -143,7 +143,7 @@ module Dynflow
|
|
143
143
|
def initialize(world)
|
144
144
|
super
|
145
145
|
@world = world
|
146
|
-
@data.merge!(owner_id: "world:#{world.id}",
|
146
|
+
@data.merge!(owner_id: "world:#{world.id}", world_id: world.id)
|
147
147
|
end
|
148
148
|
|
149
149
|
def self.lock_id(*args)
|
@@ -0,0 +1,190 @@
|
|
1
|
+
module Dynflow
|
2
|
+
# Director is responsible for telling what to do next when:
|
3
|
+
# * new execution starts
|
4
|
+
# * an event accurs
|
5
|
+
# * some work is finished
|
6
|
+
#
|
7
|
+
# It's public methods (except terminate) return work items that the
|
8
|
+
# executor should understand
|
9
|
+
class Director
|
10
|
+
include Algebrick::TypeCheck
|
11
|
+
|
12
|
+
Event = Algebrick.type do
|
13
|
+
fields! execution_plan_id: String,
|
14
|
+
step_id: Fixnum,
|
15
|
+
event: Object,
|
16
|
+
result: Concurrent::Edge::Future
|
17
|
+
end
|
18
|
+
|
19
|
+
UnprocessableEvent = Class.new(Dynflow::Error)
|
20
|
+
|
21
|
+
class WorkItem
|
22
|
+
attr_reader :execution_plan_id
|
23
|
+
|
24
|
+
def initialize(execution_plan_id)
|
25
|
+
@execution_plan_id = execution_plan_id
|
26
|
+
end
|
27
|
+
|
28
|
+
def execute
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class StepWorkItem < WorkItem
|
34
|
+
attr_reader :step
|
35
|
+
|
36
|
+
def initialize(execution_plan_id, step)
|
37
|
+
super(execution_plan_id)
|
38
|
+
@step = step
|
39
|
+
end
|
40
|
+
|
41
|
+
def execute
|
42
|
+
@step.execute(nil)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
class EventWorkItem < StepWorkItem
|
47
|
+
attr_reader :event
|
48
|
+
|
49
|
+
def initialize(execution_plan_id, step, event)
|
50
|
+
super(execution_plan_id, step)
|
51
|
+
@event = event
|
52
|
+
end
|
53
|
+
|
54
|
+
def execute
|
55
|
+
@step.execute(@event.event)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class FinalizeWorkItem < WorkItem
|
60
|
+
def initialize(execution_plan_id, sequential_manager)
|
61
|
+
super(execution_plan_id)
|
62
|
+
@sequential_manager = sequential_manager
|
63
|
+
end
|
64
|
+
|
65
|
+
def execute
|
66
|
+
@sequential_manager.finalize
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
require 'dynflow/director/work_queue'
|
71
|
+
require 'dynflow/director/sequence_cursor'
|
72
|
+
require 'dynflow/director/flow_manager'
|
73
|
+
require 'dynflow/director/execution_plan_manager'
|
74
|
+
require 'dynflow/director/sequential_manager'
|
75
|
+
require 'dynflow/director/running_steps_manager'
|
76
|
+
|
77
|
+
attr_reader :logger
|
78
|
+
|
79
|
+
def initialize(world)
|
80
|
+
@world = world
|
81
|
+
@logger = world.logger
|
82
|
+
@execution_plan_managers = {}
|
83
|
+
@plan_ids_in_rescue = Set.new
|
84
|
+
end
|
85
|
+
|
86
|
+
def start_execution(execution_plan_id, finished)
|
87
|
+
manager = track_execution_plan(execution_plan_id, finished)
|
88
|
+
return [] unless manager
|
89
|
+
unless_done(manager, manager.start)
|
90
|
+
end
|
91
|
+
|
92
|
+
def handle_event(event)
|
93
|
+
Type! event, Event
|
94
|
+
execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
|
95
|
+
if execution_plan_manager
|
96
|
+
execution_plan_manager.event(event)
|
97
|
+
else
|
98
|
+
raise Dynflow::Error, "no manager for #{event.inspect}"
|
99
|
+
end
|
100
|
+
rescue Dynflow::Error => e
|
101
|
+
event.result.fail e.message
|
102
|
+
raise e
|
103
|
+
end
|
104
|
+
|
105
|
+
def work_finished(work)
|
106
|
+
manager = @execution_plan_managers[work.execution_plan_id]
|
107
|
+
unless_done(manager, manager.what_is_next(work))
|
108
|
+
end
|
109
|
+
|
110
|
+
def terminate
|
111
|
+
unless @execution_plan_managers.empty?
|
112
|
+
logger.error "... cleaning #{@execution_plan_managers.size} execution plans ..."
|
113
|
+
begin
|
114
|
+
@execution_plan_managers.values.each do |manager|
|
115
|
+
manager.terminate
|
116
|
+
end
|
117
|
+
rescue Errors::PersistenceError
|
118
|
+
logger.error "could not to clean the data properly"
|
119
|
+
end
|
120
|
+
@execution_plan_managers.values.each do |manager|
|
121
|
+
finish_manager(manager)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
private
|
127
|
+
|
128
|
+
def unless_done(manager, work_items)
|
129
|
+
return [] unless manager
|
130
|
+
if manager.done?
|
131
|
+
finish_manager(manager)
|
132
|
+
return []
|
133
|
+
else
|
134
|
+
return work_items
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def finish_manager(manager)
|
139
|
+
@execution_plan_managers.delete(manager.execution_plan.id)
|
140
|
+
if rescue?(manager)
|
141
|
+
rescue!(manager)
|
142
|
+
else
|
143
|
+
set_future(manager)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def rescue?(manager)
|
148
|
+
return false if @world.terminating?
|
149
|
+
@world.auto_rescue && manager.execution_plan.state == :paused &&
|
150
|
+
!@plan_ids_in_rescue.include?(manager.execution_plan.id)
|
151
|
+
end
|
152
|
+
|
153
|
+
def rescue!(manager)
|
154
|
+
# TODO: after moving to concurrent-ruby actors, there should be better place
|
155
|
+
# to put this logic of making sure we don't run rescues in endless loop
|
156
|
+
@plan_ids_in_rescue << manager.execution_plan.id
|
157
|
+
rescue_plan_id = manager.execution_plan.rescue_plan_id
|
158
|
+
if rescue_plan_id
|
159
|
+
@world.executor.execute(rescue_plan_id, manager.future, false)
|
160
|
+
else
|
161
|
+
set_future(manager)
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
def track_execution_plan(execution_plan_id, finished)
|
166
|
+
execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
|
167
|
+
|
168
|
+
if @execution_plan_managers[execution_plan_id]
|
169
|
+
raise Dynflow::Error,
|
170
|
+
"cannot execute execution_plan_id:#{execution_plan_id} it's already running"
|
171
|
+
end
|
172
|
+
|
173
|
+
if execution_plan.state == :stopped
|
174
|
+
raise Dynflow::Error,
|
175
|
+
"cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
|
176
|
+
end
|
177
|
+
|
178
|
+
@execution_plan_managers[execution_plan_id] =
|
179
|
+
ExecutionPlanManager.new(@world, execution_plan, finished)
|
180
|
+
rescue Dynflow::Error => e
|
181
|
+
finished.fail e
|
182
|
+
nil
|
183
|
+
end
|
184
|
+
|
185
|
+
def set_future(manager)
|
186
|
+
@plan_ids_in_rescue.delete(manager.execution_plan.id)
|
187
|
+
manager.future.success manager.execution_plan
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|