dynflow 1.1.6 → 1.2.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -1
  3. data/dynflow.gemspec +2 -2
  4. data/examples/clock_benchmark.rb +35 -0
  5. data/examples/memory_limit_watcher.rb +1 -1
  6. data/lib/dynflow/action.rb +2 -2
  7. data/lib/dynflow/action/suspended.rb +1 -1
  8. data/lib/dynflow/action/with_sub_plans.rb +1 -1
  9. data/lib/dynflow/actor.rb +2 -2
  10. data/lib/dynflow/actors/execution_plan_cleaner.rb +1 -1
  11. data/lib/dynflow/clock.rb +11 -8
  12. data/lib/dynflow/delayed_executors/abstract.rb +1 -1
  13. data/lib/dynflow/delayed_plan.rb +1 -1
  14. data/lib/dynflow/director.rb +4 -4
  15. data/lib/dynflow/director/execution_plan_manager.rb +2 -2
  16. data/lib/dynflow/director/running_steps_manager.rb +4 -4
  17. data/lib/dynflow/dispatcher/client_dispatcher.rb +13 -12
  18. data/lib/dynflow/dispatcher/executor_dispatcher.rb +5 -5
  19. data/lib/dynflow/execution_plan.rb +1 -1
  20. data/lib/dynflow/execution_plan/steps/plan_step.rb +4 -2
  21. data/lib/dynflow/executors/abstract.rb +6 -6
  22. data/lib/dynflow/executors/parallel.rb +6 -6
  23. data/lib/dynflow/executors/parallel/core.rb +1 -1
  24. data/lib/dynflow/rails/daemon.rb +1 -1
  25. data/lib/dynflow/testing/dummy_executor.rb +2 -2
  26. data/lib/dynflow/testing/dummy_world.rb +1 -1
  27. data/lib/dynflow/testing/in_thread_executor.rb +5 -5
  28. data/lib/dynflow/testing/in_thread_world.rb +6 -6
  29. data/lib/dynflow/throttle_limiter.rb +5 -5
  30. data/lib/dynflow/utils.rb +3 -140
  31. data/lib/dynflow/utils/indifferent_hash.rb +143 -0
  32. data/lib/dynflow/utils/priority_queue.rb +64 -0
  33. data/lib/dynflow/version.rb +1 -1
  34. data/lib/dynflow/world.rb +22 -22
  35. data/lib/dynflow/world/invalidation.rb +1 -1
  36. data/test/batch_sub_tasks_test.rb +4 -4
  37. data/test/concurrency_control_test.rb +6 -6
  38. data/test/daemon_test.rb +2 -2
  39. data/test/dispatcher_test.rb +6 -6
  40. data/test/execution_plan_test.rb +11 -0
  41. data/test/executor_test.rb +1 -1
  42. data/test/support/dummy_example.rb +1 -1
  43. data/test/test_helper.rb +17 -17
  44. data/test/utils_test.rb +56 -0
  45. data/test/world_test.rb +2 -2
  46. metadata +14 -9
@@ -9,28 +9,28 @@ module Dynflow
9
9
  super(world)
10
10
  @core = Core.spawn name: 'parallel-executor-core',
11
11
  args: [world, heartbeat_interval, queues_options],
12
- initialized: @core_initialized = Concurrent.future
12
+ initialized: @core_initialized = Concurrent::Promises.resolvable_future
13
13
  end
14
14
 
15
- def execute(execution_plan_id, finished = Concurrent.future, wait_for_acceptance = true)
15
+ def execute(execution_plan_id, finished = Concurrent::Promises.resolvable_future, wait_for_acceptance = true)
16
16
  accepted = @core.ask([:handle_execution, execution_plan_id, finished])
17
17
  accepted.value! if wait_for_acceptance
18
18
  finished
19
19
  rescue Concurrent::Actor::ActorTerminated => error
20
20
  dynflow_error = Dynflow::Error.new('executor terminated')
21
- finished.fail dynflow_error unless finished.completed?
21
+ finished.reject dynflow_error unless finished.resolved?
22
22
  raise dynflow_error
23
23
  rescue => e
24
- finished.fail e unless finished.completed?
24
+ finished.reject e unless finished.resolved?
25
25
  raise e
26
26
  end
27
27
 
28
- def event(execution_plan_id, step_id, event, future = Concurrent.future)
28
+ def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
29
29
  @core.ask([:handle_event, Director::Event[execution_plan_id, step_id, event, future]])
30
30
  future
31
31
  end
32
32
 
33
- def terminate(future = Concurrent.future)
33
+ def terminate(future = Concurrent::Promises.resolvable_future)
34
34
  @core.tell([:start_termination, future])
35
35
  future
36
36
  end
@@ -62,7 +62,7 @@ module Dynflow
62
62
  def start_termination(*args)
63
63
  super
64
64
  logger.info 'shutting down Core ...'
65
- @pools.values.each { |pool| pool.tell([:start_termination, Concurrent.future]) }
65
+ @pools.values.each { |pool| pool.tell([:start_termination, Concurrent::Promises.resolvable_future]) }
66
66
  end
67
67
 
68
68
  def finish_termination(pool_name)
@@ -36,7 +36,7 @@ module Dynflow
36
36
  if options[:memory_limit] && options[:memory_limit].to_i > 0
37
37
  ::Rails.application.dynflow.config.on_init do |world|
38
38
  memory_watcher = initialize_memory_watcher(world, options[:memory_limit], options)
39
- world.terminated.on_completion do
39
+ world.terminated.on_resolution do
40
40
  STDOUT.puts("World has been terminated")
41
41
  memory_watcher = nil # the object can be disposed
42
42
  end
@@ -8,7 +8,7 @@ module Dynflow
8
8
  @events_to_process = []
9
9
  end
10
10
 
11
- def event(execution_plan_id, step_id, event, future = Concurrent.future)
11
+ def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
12
12
  @events_to_process << [execution_plan_id, step_id, event, future]
13
13
  end
14
14
 
@@ -17,7 +17,7 @@ module Dynflow
17
17
  events = @events_to_process.dup
18
18
  clear
19
19
  events.each do |execution_plan_id, step_id, event, future|
20
- future.success true
20
+ future.fulfill true
21
21
  if event && world.action.state != :suspended
22
22
  return false
23
23
  end
@@ -30,7 +30,7 @@ module Dynflow
30
30
  []
31
31
  end
32
32
 
33
- def event(execution_plan_id, step_id, event, future = Concurrent.future)
33
+ def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
34
34
  executor.event execution_plan_id, step_id, event, future
35
35
  end
36
36
 
@@ -7,7 +7,7 @@ module Dynflow
7
7
  @work_items = Queue.new
8
8
  end
9
9
 
10
- def execute(execution_plan_id, finished = Concurrent.future, _wait_for_acceptance = true)
10
+ def execute(execution_plan_id, finished = Concurrent::Promises.resolvable_future, _wait_for_acceptance = true)
11
11
  feed_queue(@director.start_execution(execution_plan_id, finished))
12
12
  process_work_items
13
13
  finished
@@ -25,7 +25,7 @@ module Dynflow
25
25
  @director.work_finished(work_item)
26
26
  end
27
27
 
28
- def event(execution_plan_id, step_id, event, future = Concurrent.future)
28
+ def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
29
29
  event = (Director::Event[execution_plan_id, step_id, event, future])
30
30
  @director.handle_event(event).each do |work_item|
31
31
  @work_items << work_item
@@ -41,11 +41,11 @@ module Dynflow
41
41
  work_items.each { |work_item| @work_items.push(work_item) }
42
42
  end
43
43
 
44
- def terminate(future = Concurrent.future)
44
+ def terminate(future = Concurrent::Promises.resolvable_future)
45
45
  @director.terminate
46
- future.success true
46
+ future.fulfill true
47
47
  rescue => e
48
- future.fail e
48
+ future.reject e
49
49
  end
50
50
  end
51
51
  end
@@ -43,21 +43,21 @@ module Dynflow
43
43
  @executor = InThreadExecutor.new(self)
44
44
  end
45
45
 
46
- def execute(execution_plan_id, done = Concurrent.future)
46
+ def execute(execution_plan_id, done = Concurrent::Promises.resolvable_future)
47
47
  @executor.execute(execution_plan_id, done)
48
48
  end
49
49
 
50
- def terminate(future = Concurrent.future)
50
+ def terminate(future = Concurrent::Promises.resolvable_future)
51
51
  run_before_termination_hooks
52
52
  @executor.terminate
53
53
  coordinator.delete_world(registered_world)
54
- future.success true
55
- @terminated.complete
54
+ future.fulfill true
55
+ @terminated.resolve
56
56
  rescue => e
57
- future.fail e
57
+ future.reject e
58
58
  end
59
59
 
60
- def event(execution_plan_id, step_id, event, done = Concurrent.future)
60
+ def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
61
61
  @executor.event(execution_plan_id, step_id, event, done)
62
62
  end
63
63
  end
@@ -39,7 +39,7 @@ module Dynflow
39
39
  private
40
40
 
41
41
  def spawn
42
- Concurrent.future.tap do |initialized|
42
+ Concurrent::Promises.resolvable_future.tap do |initialized|
43
43
  @core = core_class.spawn(:name => 'throttle-limiter',
44
44
  :args => [@world],
45
45
  :initialized => initialized)
@@ -59,14 +59,14 @@ module Dynflow
59
59
 
60
60
  def handle_plans(parent_id, planned_ids, failed_ids)
61
61
  failed = failed_ids.map do |plan_id|
62
- ::Dynflow::World::Triggered[plan_id, Concurrent.future].tap do |triggered|
62
+ ::Dynflow::World::Triggered[plan_id, Concurrent::Promises.resolvable_future].tap do |triggered|
63
63
  execute_triggered(triggered)
64
64
  end
65
65
  end
66
66
 
67
67
  planned_ids.map do |child_id|
68
- ::Dynflow::World::Triggered[child_id, Concurrent.future].tap do |triggered|
69
- triggered.future.on_completion! { self << [:release, parent_id] }
68
+ ::Dynflow::World::Triggered[child_id, Concurrent::Promises.resolvable_future].tap do |triggered|
69
+ triggered.future.on_resolution! { self << [:release, parent_id] }
70
70
  execute_triggered(triggered) if @semaphores[parent_id].wait(triggered)
71
71
  end
72
72
  end + failed
@@ -99,7 +99,7 @@ module Dynflow
99
99
  reason ||= 'The task was cancelled.'
100
100
  @semaphores[parent_id].waiting.each do |triggered|
101
101
  cancel_plan_id(triggered.execution_plan_id, reason)
102
- triggered.future.fail(reason)
102
+ triggered.future.reject(reason)
103
103
  end
104
104
  finish(parent_id)
105
105
  end
@@ -1,6 +1,9 @@
1
1
  module Dynflow
2
2
  module Utils
3
3
 
4
+ require 'dynflow/utils/indifferent_hash'
5
+ require 'dynflow/utils/priority_queue'
6
+
4
7
  def self.validate_keys!(hash, *valid_keys)
5
8
  valid_keys.flatten!
6
9
  unexpected_options = hash.keys - valid_keys - valid_keys.map(&:to_s)
@@ -71,145 +74,5 @@ module Dynflow
71
74
  end
72
75
  end
73
76
  end
74
-
75
- # Heaviliy inpired by ActiveSupport::HashWithIndifferentAccess,
76
- # reasons we don't want to use the original implementation:
77
- # 1. we don't want any core_ext extensions
78
- # 2. some users are not happy about seeing the ActiveSupport as
79
- # our depednency
80
- class IndifferentHash < Hash
81
- def initialize(constructor = {})
82
- if constructor.respond_to?(:to_hash)
83
- super()
84
- update(constructor)
85
- else
86
- super(constructor)
87
- end
88
- end
89
-
90
- def default(key = nil)
91
- if key.is_a?(Symbol) && include?(key = key.to_s)
92
- self[key]
93
- else
94
- super
95
- end
96
- end
97
-
98
- def self.[](*args)
99
- new.merge!(Hash[*args])
100
- end
101
-
102
- alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
103
- alias_method :regular_update, :update unless method_defined?(:regular_update)
104
-
105
- def []=(key, value)
106
- regular_writer(convert_key(key), convert_value(value, for: :assignment))
107
- end
108
-
109
- alias_method :store, :[]=
110
-
111
- def update(other_hash)
112
- if other_hash.is_a? IndifferentHash
113
- super(other_hash)
114
- else
115
- other_hash.to_hash.each_pair do |key, value|
116
- if block_given? && key?(key)
117
- value = yield(convert_key(key), self[key], value)
118
- end
119
- regular_writer(convert_key(key), convert_value(value))
120
- end
121
- self
122
- end
123
- end
124
-
125
- alias_method :merge!, :update
126
-
127
- def key?(key)
128
- super(convert_key(key))
129
- end
130
-
131
- alias_method :include?, :key?
132
- alias_method :has_key?, :key?
133
- alias_method :member?, :key?
134
-
135
- def fetch(key, *extras)
136
- super(convert_key(key), *extras)
137
- end
138
-
139
- def values_at(*indices)
140
- indices.collect { |key| self[convert_key(key)] }
141
- end
142
-
143
- def dup
144
- self.class.new(self).tap do |new_hash|
145
- new_hash.default = default
146
- end
147
- end
148
-
149
- def merge(hash, &block)
150
- self.dup.update(hash, &block)
151
- end
152
-
153
- def reverse_merge(other_hash)
154
- super(self.class.new_from_hash_copying_default(other_hash))
155
- end
156
-
157
- def reverse_merge!(other_hash)
158
- replace(reverse_merge( other_hash ))
159
- end
160
-
161
- def replace(other_hash)
162
- super(self.class.new_from_hash_copying_default(other_hash))
163
- end
164
-
165
- def delete(key)
166
- super(convert_key(key))
167
- end
168
-
169
- def stringify_keys!; self end
170
- def deep_stringify_keys!; self end
171
- def stringify_keys; dup end
172
- def deep_stringify_keys; dup end
173
- def to_options!; self end
174
-
175
- def select(*args, &block)
176
- dup.tap { |hash| hash.select!(*args, &block) }
177
- end
178
-
179
- def reject(*args, &block)
180
- dup.tap { |hash| hash.reject!(*args, &block) }
181
- end
182
-
183
- # Convert to a regular hash with string keys.
184
- def to_hash
185
- _new_hash = Hash.new(default)
186
- each do |key, value|
187
- _new_hash[key] = convert_value(value, for: :to_hash)
188
- end
189
- _new_hash
190
- end
191
-
192
- protected
193
- def convert_key(key)
194
- key.kind_of?(Symbol) ? key.to_s : key
195
- end
196
-
197
- def convert_value(value, options = {})
198
- if value.is_a? Hash
199
- if options[:for] == :to_hash
200
- value.to_hash
201
- else
202
- Utils.indifferent_hash(value)
203
- end
204
- elsif value.is_a?(Array)
205
- unless options[:for] == :assignment
206
- value = value.dup
207
- end
208
- value.map! { |e| convert_value(e, options) }
209
- else
210
- value
211
- end
212
- end
213
- end
214
77
  end
215
78
  end
@@ -0,0 +1,143 @@
1
+ module Dynflow
2
+ module Utils
3
+ # Heavily inpired by ActiveSupport::HashWithIndifferentAccess,
4
+ # reasons we don't want to use the original implementation:
5
+ # 1. we don't want any core_ext extensions
6
+ # 2. some users are not happy about seeing the ActiveSupport as
7
+ # our depednency
8
+ class IndifferentHash < Hash
9
+ def initialize(constructor = {})
10
+ if constructor.respond_to?(:to_hash)
11
+ super()
12
+ update(constructor)
13
+ else
14
+ super(constructor)
15
+ end
16
+ end
17
+
18
+ def default(key = nil)
19
+ if key.is_a?(Symbol) && include?(key = key.to_s)
20
+ self[key]
21
+ else
22
+ super
23
+ end
24
+ end
25
+
26
+ def self.[](*args)
27
+ new.merge!(Hash[*args])
28
+ end
29
+
30
+ alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
31
+ alias_method :regular_update, :update unless method_defined?(:regular_update)
32
+
33
+ def []=(key, value)
34
+ regular_writer(convert_key(key), convert_value(value, for: :assignment))
35
+ end
36
+
37
+ alias_method :store, :[]=
38
+
39
+ def update(other_hash)
40
+ if other_hash.is_a? IndifferentHash
41
+ super(other_hash)
42
+ else
43
+ other_hash.to_hash.each_pair do |key, value|
44
+ if block_given? && key?(key)
45
+ value = yield(convert_key(key), self[key], value)
46
+ end
47
+ regular_writer(convert_key(key), convert_value(value))
48
+ end
49
+ self
50
+ end
51
+ end
52
+
53
+ alias_method :merge!, :update
54
+
55
+ def key?(key)
56
+ super(convert_key(key))
57
+ end
58
+
59
+ alias_method :include?, :key?
60
+ alias_method :has_key?, :key?
61
+ alias_method :member?, :key?
62
+
63
+ def fetch(key, *extras)
64
+ super(convert_key(key), *extras)
65
+ end
66
+
67
+ def values_at(*indices)
68
+ indices.collect { |key| self[convert_key(key)] }
69
+ end
70
+
71
+ def dup
72
+ self.class.new(self).tap do |new_hash|
73
+ new_hash.default = default
74
+ end
75
+ end
76
+
77
+ def merge(hash, &block)
78
+ self.dup.update(hash, &block)
79
+ end
80
+
81
+ def reverse_merge(other_hash)
82
+ super(self.class.new_from_hash_copying_default(other_hash))
83
+ end
84
+
85
+ def reverse_merge!(other_hash)
86
+ replace(reverse_merge( other_hash ))
87
+ end
88
+
89
+ def replace(other_hash)
90
+ super(self.class.new_from_hash_copying_default(other_hash))
91
+ end
92
+
93
+ def delete(key)
94
+ super(convert_key(key))
95
+ end
96
+
97
+ def stringify_keys!; self end
98
+ def deep_stringify_keys!; self end
99
+ def stringify_keys; dup end
100
+ def deep_stringify_keys; dup end
101
+ def to_options!; self end
102
+
103
+ def select(*args, &block)
104
+ dup.tap { |hash| hash.select!(*args, &block) }
105
+ end
106
+
107
+ def reject(*args, &block)
108
+ dup.tap { |hash| hash.reject!(*args, &block) }
109
+ end
110
+
111
+ # Convert to a regular hash with string keys.
112
+ def to_hash
113
+ _new_hash = Hash.new(default)
114
+ each do |key, value|
115
+ _new_hash[key] = convert_value(value, for: :to_hash)
116
+ end
117
+ _new_hash
118
+ end
119
+
120
+ protected
121
+ def convert_key(key)
122
+ key.kind_of?(Symbol) ? key.to_s : key
123
+ end
124
+
125
+ def convert_value(value, options = {})
126
+ if value.is_a? Hash
127
+ if options[:for] == :to_hash
128
+ value.to_hash
129
+ else
130
+ Utils.indifferent_hash(value)
131
+ end
132
+ elsif value.is_a?(Array)
133
+ unless options[:for] == :assignment
134
+ value = value.dup
135
+ end
136
+ value.map! { |e| convert_value(e, options) }
137
+ else
138
+ value
139
+ end
140
+ end
141
+ end
142
+ end
143
+ end