dynflow 1.1.6 → 1.2.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/dynflow.gemspec +2 -2
- data/examples/clock_benchmark.rb +35 -0
- data/examples/memory_limit_watcher.rb +1 -1
- data/lib/dynflow/action.rb +2 -2
- data/lib/dynflow/action/suspended.rb +1 -1
- data/lib/dynflow/action/with_sub_plans.rb +1 -1
- data/lib/dynflow/actor.rb +2 -2
- data/lib/dynflow/actors/execution_plan_cleaner.rb +1 -1
- data/lib/dynflow/clock.rb +11 -8
- data/lib/dynflow/delayed_executors/abstract.rb +1 -1
- data/lib/dynflow/delayed_plan.rb +1 -1
- data/lib/dynflow/director.rb +4 -4
- data/lib/dynflow/director/execution_plan_manager.rb +2 -2
- data/lib/dynflow/director/running_steps_manager.rb +4 -4
- data/lib/dynflow/dispatcher/client_dispatcher.rb +13 -12
- data/lib/dynflow/dispatcher/executor_dispatcher.rb +5 -5
- data/lib/dynflow/execution_plan.rb +1 -1
- data/lib/dynflow/execution_plan/steps/plan_step.rb +4 -2
- data/lib/dynflow/executors/abstract.rb +6 -6
- data/lib/dynflow/executors/parallel.rb +6 -6
- data/lib/dynflow/executors/parallel/core.rb +1 -1
- data/lib/dynflow/rails/daemon.rb +1 -1
- data/lib/dynflow/testing/dummy_executor.rb +2 -2
- data/lib/dynflow/testing/dummy_world.rb +1 -1
- data/lib/dynflow/testing/in_thread_executor.rb +5 -5
- data/lib/dynflow/testing/in_thread_world.rb +6 -6
- data/lib/dynflow/throttle_limiter.rb +5 -5
- data/lib/dynflow/utils.rb +3 -140
- data/lib/dynflow/utils/indifferent_hash.rb +143 -0
- data/lib/dynflow/utils/priority_queue.rb +64 -0
- data/lib/dynflow/version.rb +1 -1
- data/lib/dynflow/world.rb +22 -22
- data/lib/dynflow/world/invalidation.rb +1 -1
- data/test/batch_sub_tasks_test.rb +4 -4
- data/test/concurrency_control_test.rb +6 -6
- data/test/daemon_test.rb +2 -2
- data/test/dispatcher_test.rb +6 -6
- data/test/execution_plan_test.rb +11 -0
- data/test/executor_test.rb +1 -1
- data/test/support/dummy_example.rb +1 -1
- data/test/test_helper.rb +17 -17
- data/test/utils_test.rb +56 -0
- data/test/world_test.rb +2 -2
- metadata +14 -9
@@ -9,28 +9,28 @@ module Dynflow
|
|
9
9
|
super(world)
|
10
10
|
@core = Core.spawn name: 'parallel-executor-core',
|
11
11
|
args: [world, heartbeat_interval, queues_options],
|
12
|
-
initialized: @core_initialized = Concurrent.
|
12
|
+
initialized: @core_initialized = Concurrent::Promises.resolvable_future
|
13
13
|
end
|
14
14
|
|
15
|
-
def execute(execution_plan_id, finished = Concurrent.
|
15
|
+
def execute(execution_plan_id, finished = Concurrent::Promises.resolvable_future, wait_for_acceptance = true)
|
16
16
|
accepted = @core.ask([:handle_execution, execution_plan_id, finished])
|
17
17
|
accepted.value! if wait_for_acceptance
|
18
18
|
finished
|
19
19
|
rescue Concurrent::Actor::ActorTerminated => error
|
20
20
|
dynflow_error = Dynflow::Error.new('executor terminated')
|
21
|
-
finished.
|
21
|
+
finished.reject dynflow_error unless finished.resolved?
|
22
22
|
raise dynflow_error
|
23
23
|
rescue => e
|
24
|
-
finished.
|
24
|
+
finished.reject e unless finished.resolved?
|
25
25
|
raise e
|
26
26
|
end
|
27
27
|
|
28
|
-
def event(execution_plan_id, step_id, event, future = Concurrent.
|
28
|
+
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
|
29
29
|
@core.ask([:handle_event, Director::Event[execution_plan_id, step_id, event, future]])
|
30
30
|
future
|
31
31
|
end
|
32
32
|
|
33
|
-
def terminate(future = Concurrent.
|
33
|
+
def terminate(future = Concurrent::Promises.resolvable_future)
|
34
34
|
@core.tell([:start_termination, future])
|
35
35
|
future
|
36
36
|
end
|
@@ -62,7 +62,7 @@ module Dynflow
|
|
62
62
|
def start_termination(*args)
|
63
63
|
super
|
64
64
|
logger.info 'shutting down Core ...'
|
65
|
-
@pools.values.each { |pool| pool.tell([:start_termination, Concurrent.
|
65
|
+
@pools.values.each { |pool| pool.tell([:start_termination, Concurrent::Promises.resolvable_future]) }
|
66
66
|
end
|
67
67
|
|
68
68
|
def finish_termination(pool_name)
|
data/lib/dynflow/rails/daemon.rb
CHANGED
@@ -36,7 +36,7 @@ module Dynflow
|
|
36
36
|
if options[:memory_limit] && options[:memory_limit].to_i > 0
|
37
37
|
::Rails.application.dynflow.config.on_init do |world|
|
38
38
|
memory_watcher = initialize_memory_watcher(world, options[:memory_limit], options)
|
39
|
-
world.terminated.
|
39
|
+
world.terminated.on_resolution do
|
40
40
|
STDOUT.puts("World has been terminated")
|
41
41
|
memory_watcher = nil # the object can be disposed
|
42
42
|
end
|
@@ -8,7 +8,7 @@ module Dynflow
|
|
8
8
|
@events_to_process = []
|
9
9
|
end
|
10
10
|
|
11
|
-
def event(execution_plan_id, step_id, event, future = Concurrent.
|
11
|
+
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
|
12
12
|
@events_to_process << [execution_plan_id, step_id, event, future]
|
13
13
|
end
|
14
14
|
|
@@ -17,7 +17,7 @@ module Dynflow
|
|
17
17
|
events = @events_to_process.dup
|
18
18
|
clear
|
19
19
|
events.each do |execution_plan_id, step_id, event, future|
|
20
|
-
future.
|
20
|
+
future.fulfill true
|
21
21
|
if event && world.action.state != :suspended
|
22
22
|
return false
|
23
23
|
end
|
@@ -30,7 +30,7 @@ module Dynflow
|
|
30
30
|
[]
|
31
31
|
end
|
32
32
|
|
33
|
-
def event(execution_plan_id, step_id, event, future = Concurrent.
|
33
|
+
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
|
34
34
|
executor.event execution_plan_id, step_id, event, future
|
35
35
|
end
|
36
36
|
|
@@ -7,7 +7,7 @@ module Dynflow
|
|
7
7
|
@work_items = Queue.new
|
8
8
|
end
|
9
9
|
|
10
|
-
def execute(execution_plan_id, finished = Concurrent.
|
10
|
+
def execute(execution_plan_id, finished = Concurrent::Promises.resolvable_future, _wait_for_acceptance = true)
|
11
11
|
feed_queue(@director.start_execution(execution_plan_id, finished))
|
12
12
|
process_work_items
|
13
13
|
finished
|
@@ -25,7 +25,7 @@ module Dynflow
|
|
25
25
|
@director.work_finished(work_item)
|
26
26
|
end
|
27
27
|
|
28
|
-
def event(execution_plan_id, step_id, event, future = Concurrent.
|
28
|
+
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
|
29
29
|
event = (Director::Event[execution_plan_id, step_id, event, future])
|
30
30
|
@director.handle_event(event).each do |work_item|
|
31
31
|
@work_items << work_item
|
@@ -41,11 +41,11 @@ module Dynflow
|
|
41
41
|
work_items.each { |work_item| @work_items.push(work_item) }
|
42
42
|
end
|
43
43
|
|
44
|
-
def terminate(future = Concurrent.
|
44
|
+
def terminate(future = Concurrent::Promises.resolvable_future)
|
45
45
|
@director.terminate
|
46
|
-
future.
|
46
|
+
future.fulfill true
|
47
47
|
rescue => e
|
48
|
-
future.
|
48
|
+
future.reject e
|
49
49
|
end
|
50
50
|
end
|
51
51
|
end
|
@@ -43,21 +43,21 @@ module Dynflow
|
|
43
43
|
@executor = InThreadExecutor.new(self)
|
44
44
|
end
|
45
45
|
|
46
|
-
def execute(execution_plan_id, done = Concurrent.
|
46
|
+
def execute(execution_plan_id, done = Concurrent::Promises.resolvable_future)
|
47
47
|
@executor.execute(execution_plan_id, done)
|
48
48
|
end
|
49
49
|
|
50
|
-
def terminate(future = Concurrent.
|
50
|
+
def terminate(future = Concurrent::Promises.resolvable_future)
|
51
51
|
run_before_termination_hooks
|
52
52
|
@executor.terminate
|
53
53
|
coordinator.delete_world(registered_world)
|
54
|
-
future.
|
55
|
-
@terminated.
|
54
|
+
future.fulfill true
|
55
|
+
@terminated.resolve
|
56
56
|
rescue => e
|
57
|
-
future.
|
57
|
+
future.reject e
|
58
58
|
end
|
59
59
|
|
60
|
-
def event(execution_plan_id, step_id, event, done = Concurrent.
|
60
|
+
def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
|
61
61
|
@executor.event(execution_plan_id, step_id, event, done)
|
62
62
|
end
|
63
63
|
end
|
@@ -39,7 +39,7 @@ module Dynflow
|
|
39
39
|
private
|
40
40
|
|
41
41
|
def spawn
|
42
|
-
Concurrent.
|
42
|
+
Concurrent::Promises.resolvable_future.tap do |initialized|
|
43
43
|
@core = core_class.spawn(:name => 'throttle-limiter',
|
44
44
|
:args => [@world],
|
45
45
|
:initialized => initialized)
|
@@ -59,14 +59,14 @@ module Dynflow
|
|
59
59
|
|
60
60
|
def handle_plans(parent_id, planned_ids, failed_ids)
|
61
61
|
failed = failed_ids.map do |plan_id|
|
62
|
-
::Dynflow::World::Triggered[plan_id, Concurrent.
|
62
|
+
::Dynflow::World::Triggered[plan_id, Concurrent::Promises.resolvable_future].tap do |triggered|
|
63
63
|
execute_triggered(triggered)
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
67
|
planned_ids.map do |child_id|
|
68
|
-
::Dynflow::World::Triggered[child_id, Concurrent.
|
69
|
-
triggered.future.
|
68
|
+
::Dynflow::World::Triggered[child_id, Concurrent::Promises.resolvable_future].tap do |triggered|
|
69
|
+
triggered.future.on_resolution! { self << [:release, parent_id] }
|
70
70
|
execute_triggered(triggered) if @semaphores[parent_id].wait(triggered)
|
71
71
|
end
|
72
72
|
end + failed
|
@@ -99,7 +99,7 @@ module Dynflow
|
|
99
99
|
reason ||= 'The task was cancelled.'
|
100
100
|
@semaphores[parent_id].waiting.each do |triggered|
|
101
101
|
cancel_plan_id(triggered.execution_plan_id, reason)
|
102
|
-
triggered.future.
|
102
|
+
triggered.future.reject(reason)
|
103
103
|
end
|
104
104
|
finish(parent_id)
|
105
105
|
end
|
data/lib/dynflow/utils.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
module Dynflow
|
2
2
|
module Utils
|
3
3
|
|
4
|
+
require 'dynflow/utils/indifferent_hash'
|
5
|
+
require 'dynflow/utils/priority_queue'
|
6
|
+
|
4
7
|
def self.validate_keys!(hash, *valid_keys)
|
5
8
|
valid_keys.flatten!
|
6
9
|
unexpected_options = hash.keys - valid_keys - valid_keys.map(&:to_s)
|
@@ -71,145 +74,5 @@ module Dynflow
|
|
71
74
|
end
|
72
75
|
end
|
73
76
|
end
|
74
|
-
|
75
|
-
# Heaviliy inpired by ActiveSupport::HashWithIndifferentAccess,
|
76
|
-
# reasons we don't want to use the original implementation:
|
77
|
-
# 1. we don't want any core_ext extensions
|
78
|
-
# 2. some users are not happy about seeing the ActiveSupport as
|
79
|
-
# our depednency
|
80
|
-
class IndifferentHash < Hash
|
81
|
-
def initialize(constructor = {})
|
82
|
-
if constructor.respond_to?(:to_hash)
|
83
|
-
super()
|
84
|
-
update(constructor)
|
85
|
-
else
|
86
|
-
super(constructor)
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def default(key = nil)
|
91
|
-
if key.is_a?(Symbol) && include?(key = key.to_s)
|
92
|
-
self[key]
|
93
|
-
else
|
94
|
-
super
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def self.[](*args)
|
99
|
-
new.merge!(Hash[*args])
|
100
|
-
end
|
101
|
-
|
102
|
-
alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
|
103
|
-
alias_method :regular_update, :update unless method_defined?(:regular_update)
|
104
|
-
|
105
|
-
def []=(key, value)
|
106
|
-
regular_writer(convert_key(key), convert_value(value, for: :assignment))
|
107
|
-
end
|
108
|
-
|
109
|
-
alias_method :store, :[]=
|
110
|
-
|
111
|
-
def update(other_hash)
|
112
|
-
if other_hash.is_a? IndifferentHash
|
113
|
-
super(other_hash)
|
114
|
-
else
|
115
|
-
other_hash.to_hash.each_pair do |key, value|
|
116
|
-
if block_given? && key?(key)
|
117
|
-
value = yield(convert_key(key), self[key], value)
|
118
|
-
end
|
119
|
-
regular_writer(convert_key(key), convert_value(value))
|
120
|
-
end
|
121
|
-
self
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
alias_method :merge!, :update
|
126
|
-
|
127
|
-
def key?(key)
|
128
|
-
super(convert_key(key))
|
129
|
-
end
|
130
|
-
|
131
|
-
alias_method :include?, :key?
|
132
|
-
alias_method :has_key?, :key?
|
133
|
-
alias_method :member?, :key?
|
134
|
-
|
135
|
-
def fetch(key, *extras)
|
136
|
-
super(convert_key(key), *extras)
|
137
|
-
end
|
138
|
-
|
139
|
-
def values_at(*indices)
|
140
|
-
indices.collect { |key| self[convert_key(key)] }
|
141
|
-
end
|
142
|
-
|
143
|
-
def dup
|
144
|
-
self.class.new(self).tap do |new_hash|
|
145
|
-
new_hash.default = default
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
def merge(hash, &block)
|
150
|
-
self.dup.update(hash, &block)
|
151
|
-
end
|
152
|
-
|
153
|
-
def reverse_merge(other_hash)
|
154
|
-
super(self.class.new_from_hash_copying_default(other_hash))
|
155
|
-
end
|
156
|
-
|
157
|
-
def reverse_merge!(other_hash)
|
158
|
-
replace(reverse_merge( other_hash ))
|
159
|
-
end
|
160
|
-
|
161
|
-
def replace(other_hash)
|
162
|
-
super(self.class.new_from_hash_copying_default(other_hash))
|
163
|
-
end
|
164
|
-
|
165
|
-
def delete(key)
|
166
|
-
super(convert_key(key))
|
167
|
-
end
|
168
|
-
|
169
|
-
def stringify_keys!; self end
|
170
|
-
def deep_stringify_keys!; self end
|
171
|
-
def stringify_keys; dup end
|
172
|
-
def deep_stringify_keys; dup end
|
173
|
-
def to_options!; self end
|
174
|
-
|
175
|
-
def select(*args, &block)
|
176
|
-
dup.tap { |hash| hash.select!(*args, &block) }
|
177
|
-
end
|
178
|
-
|
179
|
-
def reject(*args, &block)
|
180
|
-
dup.tap { |hash| hash.reject!(*args, &block) }
|
181
|
-
end
|
182
|
-
|
183
|
-
# Convert to a regular hash with string keys.
|
184
|
-
def to_hash
|
185
|
-
_new_hash = Hash.new(default)
|
186
|
-
each do |key, value|
|
187
|
-
_new_hash[key] = convert_value(value, for: :to_hash)
|
188
|
-
end
|
189
|
-
_new_hash
|
190
|
-
end
|
191
|
-
|
192
|
-
protected
|
193
|
-
def convert_key(key)
|
194
|
-
key.kind_of?(Symbol) ? key.to_s : key
|
195
|
-
end
|
196
|
-
|
197
|
-
def convert_value(value, options = {})
|
198
|
-
if value.is_a? Hash
|
199
|
-
if options[:for] == :to_hash
|
200
|
-
value.to_hash
|
201
|
-
else
|
202
|
-
Utils.indifferent_hash(value)
|
203
|
-
end
|
204
|
-
elsif value.is_a?(Array)
|
205
|
-
unless options[:for] == :assignment
|
206
|
-
value = value.dup
|
207
|
-
end
|
208
|
-
value.map! { |e| convert_value(e, options) }
|
209
|
-
else
|
210
|
-
value
|
211
|
-
end
|
212
|
-
end
|
213
|
-
end
|
214
77
|
end
|
215
78
|
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
module Dynflow
|
2
|
+
module Utils
|
3
|
+
# Heavily inpired by ActiveSupport::HashWithIndifferentAccess,
|
4
|
+
# reasons we don't want to use the original implementation:
|
5
|
+
# 1. we don't want any core_ext extensions
|
6
|
+
# 2. some users are not happy about seeing the ActiveSupport as
|
7
|
+
# our depednency
|
8
|
+
class IndifferentHash < Hash
|
9
|
+
def initialize(constructor = {})
|
10
|
+
if constructor.respond_to?(:to_hash)
|
11
|
+
super()
|
12
|
+
update(constructor)
|
13
|
+
else
|
14
|
+
super(constructor)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def default(key = nil)
|
19
|
+
if key.is_a?(Symbol) && include?(key = key.to_s)
|
20
|
+
self[key]
|
21
|
+
else
|
22
|
+
super
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.[](*args)
|
27
|
+
new.merge!(Hash[*args])
|
28
|
+
end
|
29
|
+
|
30
|
+
alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
|
31
|
+
alias_method :regular_update, :update unless method_defined?(:regular_update)
|
32
|
+
|
33
|
+
def []=(key, value)
|
34
|
+
regular_writer(convert_key(key), convert_value(value, for: :assignment))
|
35
|
+
end
|
36
|
+
|
37
|
+
alias_method :store, :[]=
|
38
|
+
|
39
|
+
def update(other_hash)
|
40
|
+
if other_hash.is_a? IndifferentHash
|
41
|
+
super(other_hash)
|
42
|
+
else
|
43
|
+
other_hash.to_hash.each_pair do |key, value|
|
44
|
+
if block_given? && key?(key)
|
45
|
+
value = yield(convert_key(key), self[key], value)
|
46
|
+
end
|
47
|
+
regular_writer(convert_key(key), convert_value(value))
|
48
|
+
end
|
49
|
+
self
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
alias_method :merge!, :update
|
54
|
+
|
55
|
+
def key?(key)
|
56
|
+
super(convert_key(key))
|
57
|
+
end
|
58
|
+
|
59
|
+
alias_method :include?, :key?
|
60
|
+
alias_method :has_key?, :key?
|
61
|
+
alias_method :member?, :key?
|
62
|
+
|
63
|
+
def fetch(key, *extras)
|
64
|
+
super(convert_key(key), *extras)
|
65
|
+
end
|
66
|
+
|
67
|
+
def values_at(*indices)
|
68
|
+
indices.collect { |key| self[convert_key(key)] }
|
69
|
+
end
|
70
|
+
|
71
|
+
def dup
|
72
|
+
self.class.new(self).tap do |new_hash|
|
73
|
+
new_hash.default = default
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def merge(hash, &block)
|
78
|
+
self.dup.update(hash, &block)
|
79
|
+
end
|
80
|
+
|
81
|
+
def reverse_merge(other_hash)
|
82
|
+
super(self.class.new_from_hash_copying_default(other_hash))
|
83
|
+
end
|
84
|
+
|
85
|
+
def reverse_merge!(other_hash)
|
86
|
+
replace(reverse_merge( other_hash ))
|
87
|
+
end
|
88
|
+
|
89
|
+
def replace(other_hash)
|
90
|
+
super(self.class.new_from_hash_copying_default(other_hash))
|
91
|
+
end
|
92
|
+
|
93
|
+
def delete(key)
|
94
|
+
super(convert_key(key))
|
95
|
+
end
|
96
|
+
|
97
|
+
def stringify_keys!; self end
|
98
|
+
def deep_stringify_keys!; self end
|
99
|
+
def stringify_keys; dup end
|
100
|
+
def deep_stringify_keys; dup end
|
101
|
+
def to_options!; self end
|
102
|
+
|
103
|
+
def select(*args, &block)
|
104
|
+
dup.tap { |hash| hash.select!(*args, &block) }
|
105
|
+
end
|
106
|
+
|
107
|
+
def reject(*args, &block)
|
108
|
+
dup.tap { |hash| hash.reject!(*args, &block) }
|
109
|
+
end
|
110
|
+
|
111
|
+
# Convert to a regular hash with string keys.
|
112
|
+
def to_hash
|
113
|
+
_new_hash = Hash.new(default)
|
114
|
+
each do |key, value|
|
115
|
+
_new_hash[key] = convert_value(value, for: :to_hash)
|
116
|
+
end
|
117
|
+
_new_hash
|
118
|
+
end
|
119
|
+
|
120
|
+
protected
|
121
|
+
def convert_key(key)
|
122
|
+
key.kind_of?(Symbol) ? key.to_s : key
|
123
|
+
end
|
124
|
+
|
125
|
+
def convert_value(value, options = {})
|
126
|
+
if value.is_a? Hash
|
127
|
+
if options[:for] == :to_hash
|
128
|
+
value.to_hash
|
129
|
+
else
|
130
|
+
Utils.indifferent_hash(value)
|
131
|
+
end
|
132
|
+
elsif value.is_a?(Array)
|
133
|
+
unless options[:for] == :assignment
|
134
|
+
value = value.dup
|
135
|
+
end
|
136
|
+
value.map! { |e| convert_value(e, options) }
|
137
|
+
else
|
138
|
+
value
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|