dynflow 0.8.21 → 0.8.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 25cceba5d864e15547744e5a0038c3aa39da72c1
4
- data.tar.gz: 6743618a3dec29ce3221c79840983343b9e4371d
3
+ metadata.gz: 2dca125fdba138da52e0ffaff7b17a979945be2b
4
+ data.tar.gz: de40a2fcd90aa602ba2db2cc12e28bfc435c750c
5
5
  SHA512:
6
- metadata.gz: aa59cc73158b14f8dee9dd6a8ac5388559973dd53396608b4f57687eb92263890016ebc0cfd1cc0d3227cc9879de6bbb594fc458f5ae7988eb3cefe8b4ba4fdb
7
- data.tar.gz: d2c95326b522da13d3107e517bf7ff276f83dd2904da94175d621b68e9300b976998c3ef8c1a2841cccfa3aa8d064aa20351def789516af4cc0c5fb48ec57e26
6
+ metadata.gz: 760ace645251b0b91e0454c2794b4546a93504f542fb2f07452fe76eaed1e75326d1aeabcc9394391b12a410064a08e35413a813f88f9acf0ffcef3227309369
7
+ data.tar.gz: 4b69a6c1590bfff3c96b04dc5da8c76119680482375f8341331984b3a8deb1bdbc08a06b194f56bacc28fc323a6a55a7e8ff6f0d60b0edc7121ae29f101fa4ab
@@ -7,6 +7,8 @@ rvm:
7
7
  - "2.0.0"
8
8
  - "2.1.5"
9
9
  - "2.2.0"
10
+ - "2.3.1"
11
+ - "2.4.0"
10
12
 
11
13
  env:
12
14
  global:
data/Gemfile CHANGED
@@ -8,6 +8,7 @@ end
8
8
 
9
9
  group :pry do
10
10
  gem 'pry'
11
+ gem 'pry-byebug'
11
12
  end
12
13
 
13
14
  group :postgresql do
@@ -29,3 +30,7 @@ end
29
30
  group :lint do
30
31
  gem 'rubocop', '0.39.0'
31
32
  end
33
+
34
+ group :memory_watcher do
35
+ gem 'get_process_mem'
36
+ end
@@ -0,0 +1,59 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative 'example_helper'
4
+
5
+ example_description = <<DESC
6
+ Memory limit watcher Example
7
+ ===========================
8
+
9
+ In this example we are setting a watcher that will terminate our world object
10
+ when process memory consumption exceeds a limit that will be set.
11
+
12
+
13
+ DESC
14
+
15
+ module MemorylimiterExample
16
+ class SampleAction < Dynflow::Action
17
+ def plan(memory_to_use)
18
+ plan_self(number: memory_to_use)
19
+ end
20
+
21
+ def run
22
+ array = Array.new(input[:number].to_i)
23
+ puts "[action] allocated #{input[:number]} cells"
24
+ end
25
+ end
26
+ end
27
+
28
+ if $0 == __FILE__
29
+ puts example_description
30
+
31
+ world = ExampleHelper.create_world do |config|
32
+ config.exit_on_terminate = false
33
+ end
34
+
35
+ world.terminated.on_completion do
36
+ puts '[world] The world has been terminated'
37
+ end
38
+
39
+ require 'get_process_mem'
40
+ memory_info_provider = GetProcessMem.new
41
+ puts '[info] Preparing memory watcher: '
42
+ require 'dynflow/watchers/memory_consumption_watcher'
43
+ puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
44
+ limit = memory_info_provider.bytes + 500_000
45
+ puts "[info] Setting memory limit to #{limit} bytes"
46
+ watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(world, limit, polling_interval: 1)
47
+ puts '[info] Small action: '
48
+ world.trigger(MemorylimiterExample::SampleAction, 10)
49
+ sleep 2
50
+ puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
51
+ puts '[info] Big action: '
52
+ world.trigger(MemorylimiterExample::SampleAction, 500_000)
53
+ sleep 2
54
+ puts "[info] now the process consumes #{memory_info_provider.bytes} bytes."
55
+ puts '[info] Small action again - will not execute, the world is not accepting requests'
56
+ world.trigger(MemorylimiterExample::SampleAction, 500_000)
57
+ sleep 2
58
+ puts 'Done'
59
+ end
@@ -1,4 +1,5 @@
1
1
  module Dynflow
2
+ # rubocop:disable Metrics/ClassLength
2
3
  class Action < Serializable
3
4
 
4
5
  OutputReference = ExecutionPlan::OutputReference
@@ -21,6 +22,7 @@ module Dynflow
21
22
  require 'dynflow/action/polling'
22
23
  require 'dynflow/action/cancellable'
23
24
  require 'dynflow/action/with_sub_plans'
25
+ require 'dynflow/action/with_bulk_sub_plans'
24
26
 
25
27
  def self.all_children
26
28
  children.values.inject(children.values) do |children, child|
@@ -122,6 +124,10 @@ module Dynflow
122
124
  raise TypeError, "Wrong phase #{phase}, required #{phases}"
123
125
  end
124
126
 
127
+ def label
128
+ self.class.name
129
+ end
130
+
125
131
  def input=(hash)
126
132
  Type! hash, Hash
127
133
  phase! Plan
@@ -543,4 +549,5 @@ module Dynflow
543
549
  @trigger.nil?
544
550
  end
545
551
  end
552
+ # rubocop:enable Metrics/ClassLength
546
553
  end
@@ -0,0 +1,88 @@
1
+ module Dynflow
2
+ module Action::WithBulkSubPlans
3
+ include Dynflow::Action::Cancellable
4
+
5
+ DEFAULT_BATCH_SIZE = 100
6
+
7
+ # Should return a slice of size items starting from item with index from
8
+ def batch(from, size)
9
+ raise NotImplementedError
10
+ end
11
+
12
+ PlanNextBatch = Algebrick.atom
13
+
14
+ def run(event = nil)
15
+ if event === PlanNextBatch
16
+ spawn_plans if can_spawn_next_batch?
17
+ suspend
18
+ else
19
+ super
20
+ end
21
+ end
22
+
23
+ def initiate
24
+ output[:planned_count] = 0
25
+ output[:total_count] = total_count
26
+ super
27
+ end
28
+
29
+ def increase_counts(planned, failed)
30
+ super(planned, failed, false)
31
+ output[:planned_count] += planned
32
+ end
33
+
34
+ # Should return the expected total count of tasks
35
+ def total_count
36
+ raise NotImplementedError
37
+ end
38
+
39
+ # Returns the items in the current batch
40
+ def current_batch
41
+ start_position = output[:planned_count]
42
+ size = start_position + batch_size > total_count ? total_count - start_position : batch_size
43
+ batch(start_position, size)
44
+ end
45
+
46
+ def batch_size
47
+ DEFAULT_BATCH_SIZE
48
+ end
49
+
50
+ # The same logic as in Action::WithSubPlans, but calculated using the expected total count
51
+ def run_progress
52
+ if counts_set?
53
+ (output[:success_count] + output[:failed_count]).to_f / total_count
54
+ else
55
+ 0.1
56
+ end
57
+ end
58
+
59
+ def spawn_plans
60
+ super
61
+ ensure
62
+ suspended_action << PlanNextBatch
63
+ end
64
+
65
+ def cancel!
66
+ # Count the not-yet-planned tasks as failed
67
+ output[:failed_count] += total_count - output[:planned_count]
68
+ if uses_concurrency_control
69
+ # Tell the throttle limiter to cancel the tasks its managing
70
+ world.throttle_limiter.cancel!(execution_plan_id)
71
+ else
72
+ # Just stop the tasks which were not started yet
73
+ sub_plans(:state => 'planned').each { |sub_plan| sub_plan.update_state(:stopped) }
74
+ end
75
+ running = sub_plans(:state => 'running')
76
+ # Pass the cancel event to running sub plans if they can be cancelled
77
+ running.each { |sub_plan| sub_plan.cancel! if sub_plan.cancellable? }
78
+ suspend
79
+ end
80
+
81
+ private
82
+
83
+ def can_spawn_next_batch?
84
+ total_count - output[:success_count] - output[:pending_count] - output[:failed_count] > 0
85
+ end
86
+
87
+ end
88
+ end
@@ -26,16 +26,16 @@ module Dynflow
26
26
  end
27
27
 
28
28
  def initiate
29
- sub_plans = create_sub_plans
30
- sub_plans = Array[sub_plans] unless sub_plans.is_a? Array
31
29
  if uses_concurrency_control
32
- planned, failed = sub_plans.partition { |plan| plan.state == :planned }
33
- calculate_time_distribution sub_plans.count
34
- sub_plans = world.throttle_limiter.handle_plans!(execution_plan_id,
35
- planned.map(&:id),
36
- failed.map(&:id),
37
- input[:concurrency_control])
30
+ calculate_time_distribution
31
+ world.throttle_limiter.initialize_plan(execution_plan_id, input[:concurrency_control])
38
32
  end
33
+ spawn_plans
34
+ end
35
+
36
+ def spawn_plans
37
+ sub_plans = create_sub_plans
38
+ sub_plans = Array[sub_plans] unless sub_plans.is_a? Array
39
39
  wait_for_sub_plans sub_plans
40
40
  end
41
41
 
@@ -71,19 +71,26 @@ module Dynflow
71
71
  # Helper for creating sub plans
72
72
  def trigger(*args)
73
73
  if uses_concurrency_control
74
- world.plan_with_caller(self, *args)
74
+ trigger_with_concurrency_control(*args)
75
75
  else
76
76
  world.trigger { world.plan_with_caller(self, *args) }
77
77
  end
78
78
  end
79
79
 
80
+ def trigger_with_concurrency_control(*args)
81
+ record = world.plan_with_caller(self, *args)
82
+ records = [[record.id], []]
83
+ records.reverse! unless record.state == :planned
84
+ @world.throttle_limiter.handle_plans!(execution_plan_id, *records).first
85
+ end
86
+
80
87
  def limit_concurrency_level(level)
81
88
  input[:concurrency_control] ||= {}
82
89
  input[:concurrency_control][:level] = ::Dynflow::Semaphores::Stateful.new(level).to_hash
83
90
  end
84
91
 
85
- def calculate_time_distribution(count)
86
- time = input[:concurrency_control][:time]
92
+ def calculate_time_distribution
93
+ time, count = input[:concurrency_control][:time]
87
94
  unless time.nil? || time.is_a?(Hash)
88
95
  # Assume concurrency level 1 unless stated otherwise
89
96
  level = input[:concurrency_control].fetch(:level, {}).fetch(:free, 1)
@@ -94,25 +101,14 @@ module Dynflow
94
101
  end
95
102
  end
96
103
 
97
- def distribute_over_time(time_span)
104
+ def distribute_over_time(time_span, count)
98
105
  input[:concurrency_control] ||= {}
99
- input[:concurrency_control][:time] = time_span
106
+ input[:concurrency_control][:time] = [time_span, count]
100
107
  end
101
108
 
102
109
  def wait_for_sub_plans(sub_plans)
103
- output.update(total_count: 0,
104
- failed_count: 0,
105
- success_count: 0,
106
- pending_count: 0)
107
-
108
110
  planned, failed = sub_plans.partition(&:planned?)
109
-
110
- sub_plan_ids = (planned + failed).map(&:execution_plan_id)
111
-
112
- output[:total_count] = sub_plan_ids.size
113
- output[:failed_count] = failed.size
114
- output[:pending_count] = planned.size
115
-
111
+ increase_counts(planned.count, failed.count)
116
112
  if planned.any?
117
113
  notify_on_finish(planned)
118
114
  else
@@ -120,8 +116,16 @@ module Dynflow
120
116
  end
121
117
  end
122
118
 
119
+ def increase_counts(planned, failed, track_total = true)
120
+ output[:total_count] = output.fetch(:total_count, 0) + planned + failed if track_total
121
+ output[:failed_count] = output.fetch(:failed_count, 0) + failed
122
+ output[:pending_count] = output.fetch(:pending_count, 0) + planned
123
+ output[:success_count] ||= 0
124
+ end
125
+
123
126
  def try_to_finish
124
127
  if done?
128
+ world.throttle_limiter.finish(execution_plan_id)
125
129
  check_for_errors!
126
130
  on_finish
127
131
  return true
@@ -132,6 +136,8 @@ module Dynflow
132
136
 
133
137
  def resume
134
138
  if sub_plans.all? { |sub_plan| sub_plan.error_in_plan? }
139
+ # We're starting over and need to reset the counts
140
+ %w(total failed pending success).each { |key| output.delete("#{key}_count".to_sym) }
135
141
  initiate
136
142
  else
137
143
  recalculate_counts
@@ -28,6 +28,10 @@ module Dynflow
28
28
  def run
29
29
  input[:job_class].constantize.perform_now(*input[:job_arguments])
30
30
  end
31
+
32
+ def label
33
+ input[:job_class]
34
+ end
31
35
  end
32
36
  end
33
37
  end
@@ -12,7 +12,8 @@ module Dynflow
12
12
  require 'dynflow/execution_plan/output_reference'
13
13
  require 'dynflow/execution_plan/dependency_graph'
14
14
 
15
- attr_reader :id, :world, :root_plan_step, :steps, :run_flow, :finalize_flow,
15
+ attr_reader :id, :world, :label,
16
+ :root_plan_step, :steps, :run_flow, :finalize_flow,
16
17
  :started_at, :ended_at, :execution_time, :real_time, :execution_history
17
18
 
18
19
  def self.states
@@ -36,6 +37,7 @@ module Dynflow
36
37
  # all params with default values are part of *private* api
37
38
  def initialize(world,
38
39
  id = SecureRandom.uuid,
40
+ label = nil,
39
41
  state = :pending,
40
42
  root_plan_step = nil,
41
43
  run_flow = Flows::Concurrence.new([]),
@@ -49,6 +51,7 @@ module Dynflow
49
51
 
50
52
  @id = Type! id, String
51
53
  @world = Type! world, World
54
+ @label = Type! label, String, NilClass
52
55
  self.state = state
53
56
  @run_flow = Type! run_flow, Flows::Abstract
54
57
  @finalize_flow = Type! finalize_flow, Flows::Abstract
@@ -203,7 +206,8 @@ module Dynflow
203
206
  update_state(:planning)
204
207
  world.middleware.execute(:plan_phase, root_plan_step.action_class, self) do
205
208
  with_planning_scope do
206
- root_plan_step.execute(self, nil, false, *args)
209
+ root_action = root_plan_step.execute(self, nil, false, *args)
210
+ @label = root_action.label
207
211
 
208
212
  if @dependency_graph.unresolved?
209
213
  raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
@@ -336,9 +340,10 @@ module Dynflow
336
340
  end
337
341
 
338
342
  def to_hash
339
- recursive_to_hash id: self.id,
343
+ recursive_to_hash id: id,
340
344
  class: self.class.to_s,
341
- state: self.state,
345
+ label: label,
346
+ state: state,
342
347
  result: result,
343
348
  root_plan_step_id: root_plan_step && root_plan_step.id,
344
349
  run_flow: run_flow,
@@ -361,6 +366,7 @@ module Dynflow
361
366
  steps = steps_from_hash(hash[:step_ids], execution_plan_id, world)
362
367
  self.new(world,
363
368
  execution_plan_id,
369
+ hash[:label],
364
370
  hash[:state],
365
371
  steps[hash[:root_plan_step_id]],
366
372
  Flows::Abstract.from_hash(hash[:run_flow]),
@@ -27,7 +27,7 @@ module Dynflow
27
27
  META_DATA.fetch :execution_plan
28
28
  end
29
29
 
30
- META_DATA = { execution_plan: %w(state result started_at ended_at real_time execution_time),
30
+ META_DATA = { execution_plan: %w(label state result started_at ended_at real_time execution_time),
31
31
  action: %w(caller_execution_plan_id caller_action_id),
32
32
  step: %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight),
33
33
  envelope: %w(receiver_id),
@@ -309,10 +309,11 @@ module Dynflow
309
309
  def filter(what, data_set, filters)
310
310
  Type! filters, NilClass, Hash
311
311
  return data_set if filters.nil?
312
+ filters = filters.each.with_object({}) { |(k, v), hash| hash[k.to_s] = v }
312
313
 
313
- unknown = filters.keys.map(&:to_s) - META_DATA.fetch(what)
314
+ unknown = filters.keys - META_DATA.fetch(what)
314
315
  if what == :execution_plan
315
- unknown -= %w[uuid caller_execution_plan_id caller_action_id]
316
+ unknown -= %w[uuid caller_execution_plan_id caller_action_id delayed]
316
317
 
317
318
  if filters.key?('caller_action_id') && !filters.key?('caller_execution_plan_id')
318
319
  raise ArgumentError, "caller_action_id given but caller_execution_plan_id missing"
@@ -322,6 +323,11 @@ module Dynflow
322
323
  data_set = data_set.join_table(:inner, TABLES[:action], :execution_plan_uuid => :uuid).
323
324
  select_all(TABLES[:execution_plan]).distinct
324
325
  end
326
+ if filters.key?('delayed')
327
+ filters.delete('delayed')
328
+ data_set = data_set.join_table(:inner, TABLES[:delayed], :execution_plan_uuid => :uuid).
329
+ select_all(TABLES[:execution_plan]).distinct
330
+ end
325
331
  end
326
332
 
327
333
  unless unknown.empty?
@@ -0,0 +1,7 @@
1
+ Sequel.migration do
2
+ change do
3
+ alter_table(:dynflow_execution_plans) do
4
+ add_column :label, String
5
+ end
6
+ end
7
+ end
@@ -52,6 +52,7 @@ module Dynflow
52
52
  @executor.terminate
53
53
  coordinator.delete_world(registered_world)
54
54
  future.success true
55
+ @terminated.complete
55
56
  rescue => e
56
57
  future.fail e
57
58
  end
@@ -8,6 +8,14 @@ module Dynflow
8
8
  spawn
9
9
  end
10
10
 
11
+ def initialize_plan(plan_id, semaphores_hash)
12
+ core.tell([:initialize_plan, plan_id, semaphores_hash])
13
+ end
14
+
15
+ def finish(plan_id)
16
+ core.tell([:finish, plan_id])
17
+ end
18
+
11
19
  def handle_plans!(*args)
12
20
  core.ask!([:handle_plans, *args])
13
21
  end
@@ -44,10 +52,12 @@ module Dynflow
44
52
  @semaphores = {}
45
53
  end
46
54
 
47
- def handle_plans(parent_id, planned_ids, failed_ids, semaphores_hash)
48
- @semaphores[parent_id] = create_semaphores(semaphores_hash)
49
- set_up_clock_for(parent_id, true)
55
+ def initialize_plan(plan_id, semaphores_hash)
56
+ @semaphores[plan_id] = create_semaphores(semaphores_hash)
57
+ set_up_clock_for(plan_id, true)
58
+ end
50
59
 
60
+ def handle_plans(parent_id, planned_ids, failed_ids)
51
61
  failed = failed_ids.map do |plan_id|
52
62
  ::Dynflow::World::Triggered[plan_id, Concurrent.future].tap do |triggered|
53
63
  execute_triggered(triggered)
@@ -82,7 +92,6 @@ module Dynflow
82
92
  if semaphore.has_waiting? && semaphore.get == 1
83
93
  execute_triggered(semaphore.get_waiting)
84
94
  end
85
- @semaphores.delete(plan_id) unless semaphore.has_waiting?
86
95
  end
87
96
 
88
97
  def cancel(parent_id, reason = nil)
@@ -92,10 +101,14 @@ module Dynflow
92
101
  cancel_plan_id(triggered.execution_plan_id, reason)
93
102
  triggered.future.fail(reason)
94
103
  end
95
- @semaphores.delete(parent_id)
104
+ finish(parent_id)
96
105
  end
97
106
  end
98
107
 
108
+ def finish(parent_id)
109
+ @semaphores.delete(parent_id)
110
+ end
111
+
99
112
  private
100
113
 
101
114
  def cancel_plan_id(plan_id, reason)
@@ -1,3 +1,3 @@
1
1
  module Dynflow
2
- VERSION = '0.8.21'
2
+ VERSION = '0.8.22'
3
3
  end
@@ -0,0 +1,32 @@
1
+ require 'get_process_mem'
2
+
3
+ module Dynflow
4
+ module Watchers
5
+ class MemoryConsumptionWatcher
6
+
7
+ attr_reader :memory_limit, :world
8
+
9
+ def initialize(world, memory_limit, options)
10
+ @memory_limit = memory_limit
11
+ @world = world
12
+ @polling_interval = options[:polling_interval] || 60
13
+ @memory_info_provider = options[:memory_info_provider] || GetProcessMem.new
14
+ set_timer options[:initial_wait] || @polling_interval
15
+ end
16
+
17
+ def check_memory_state
18
+ if @memory_info_provider.bytes > @memory_limit
19
+ # terminate the world and stop polling
20
+ world.terminate
21
+ else
22
+ # memory is under the limit - keep waiting
23
+ set_timer
24
+ end
25
+ end
26
+
27
+ def set_timer(interval = @polling_interval)
28
+ @world.clock.ping(self, interval, nil, :check_memory_state)
29
+ end
30
+ end
31
+ end
32
+ end
@@ -7,7 +7,8 @@ module Dynflow
7
7
  attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
8
8
  :transaction_adapter, :logger_adapter, :coordinator,
9
9
  :persistence, :action_classes, :subscription_index,
10
- :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter
10
+ :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter,
11
+ :terminated
11
12
 
12
13
  def initialize(config)
13
14
  @id = SecureRandom.uuid
@@ -30,6 +31,7 @@ module Dynflow
30
31
  @auto_validity_check = config_for_world.auto_validity_check
31
32
  @validity_check_timeout = config_for_world.validity_check_timeout
32
33
  @throttle_limiter = config_for_world.throttle_limiter
34
+ @terminated = Concurrent.event
33
35
  calculate_subscription_index
34
36
 
35
37
  if executor
@@ -204,7 +206,7 @@ module Dynflow
204
206
 
205
207
  def terminate(future = Concurrent.future)
206
208
  @termination_barrier.synchronize do
207
- @terminated ||= Concurrent.future do
209
+ @terminating ||= Concurrent.future do
208
210
  begin
209
211
  run_before_termination_hooks
210
212
 
@@ -242,6 +244,7 @@ module Dynflow
242
244
  end
243
245
 
244
246
  coordinator.delete_world(registered_world)
247
+ @terminated.complete
245
248
  true
246
249
  rescue => e
247
250
  logger.fatal(e)
@@ -251,12 +254,12 @@ module Dynflow
251
254
  end
252
255
  end
253
256
 
254
- @terminated.tangle(future)
257
+ @terminating.tangle(future)
255
258
  future
256
259
  end
257
260
 
258
261
  def terminating?
259
- defined?(@terminated)
262
+ defined?(@terminating)
260
263
  end
261
264
 
262
265
  # Invalidate another world, that left some data in the runtime,
@@ -0,0 +1,98 @@
1
+ require_relative 'test_helper'
2
+
3
+ module Dynflow
4
+ module BatchSubTaskTest
5
+ describe 'Batch sub-tasks' do
6
+ include PlanAssertions
7
+ include Dynflow::Testing::Assertions
8
+ include Dynflow::Testing::Factories
9
+ include TestHelpers
10
+
11
+ class FailureSimulator
12
+ def self.should_fail?
13
+ @should_fail
14
+ end
15
+
16
+ def self.should_fail!
17
+ @should_fail = true
18
+ end
19
+
20
+ def self.wont_fail!
21
+ @should_fail = false
22
+ end
23
+ end
24
+
25
+ let(:world) { WorldFactory.create_world }
26
+
27
+ class ChildAction < ::Dynflow::Action
28
+ def plan(should_fail = false)
29
+ raise "Simulated failure" if FailureSimulator.should_fail?
30
+ plan_self
31
+ end
32
+
33
+ def run
34
+ output[:run] = true
35
+ end
36
+ end
37
+
38
+ class ParentAction < ::Dynflow::Action
39
+ include Dynflow::Action::WithSubPlans
40
+ include Dynflow::Action::WithBulkSubPlans
41
+
42
+ def plan(count, concurrency_level = nil, time_span = nil)
43
+ limit_concurrency_level(concurrency_level) unless concurrency_level.nil?
44
+ distribute_over_time(time_span, count) unless time_span.nil?
45
+ plan_self :count => count
46
+ end
47
+
48
+ def create_sub_plans
49
+ output[:batch_count] ||= 0
50
+ output[:batch_count] += 1
51
+ current_batch.map { |i| trigger(ChildAction) }
52
+ end
53
+
54
+ def batch(from, size)
55
+ (1..total_count).to_a.slice(from, size)
56
+ end
57
+
58
+ def batch_size
59
+ 5
60
+ end
61
+
62
+ def total_count
63
+ input[:count]
64
+ end
65
+ end
66
+
67
+ it 'starts tasks in batches' do
68
+ FailureSimulator.wont_fail!
69
+ plan = world.plan(ParentAction, 20)
70
+ future = world.execute plan.id
71
+ wait_for { future.completed? }
72
+ action = plan.entry_action
73
+
74
+ action.output[:batch_count].must_equal action.total_count / action.batch_size
75
+ end
76
+
77
+ it 'can resume tasks' do
78
+ FailureSimulator.should_fail!
79
+ plan = world.plan(ParentAction, 20)
80
+ future = world.execute plan.id
81
+ wait_for { future.completed? }
82
+ action = plan.entry_action
83
+ action.output[:batch_count].must_equal 1
84
+ future.value.state.must_equal :paused
85
+
86
+ FailureSimulator.wont_fail!
87
+ future = world.execute plan.id
88
+ wait_for { future.completed? }
89
+ action = future.value.entry_action
90
+ future.value.state.must_equal :stopped
91
+ action.output[:batch_count].must_equal (action.total_count / action.batch_size) + 1
92
+ action.output[:total_count].must_equal action.total_count
93
+ action.output[:success_count].must_equal action.total_count
94
+ end
95
+
96
+ end
97
+ end
98
+ end
@@ -63,7 +63,7 @@ module Dynflow
63
63
 
64
64
  def plan(count, concurrency_level = nil, time_span = nil, should_sleep = nil)
65
65
  limit_concurrency_level(concurrency_level) unless concurrency_level.nil?
66
- distribute_over_time(time_span) unless time_span.nil?
66
+ distribute_over_time(time_span, count) unless time_span.nil?
67
67
  plan_self :count => count, :should_sleep => should_sleep
68
68
  end
69
69
 
@@ -29,6 +29,7 @@ module Dynflow
29
29
 
30
30
  it 'restores the plan properly' do
31
31
  deserialized_execution_plan.id.must_equal execution_plan.id
32
+ deserialized_execution_plan.label.must_equal execution_plan.label
32
33
 
33
34
  assert_steps_equal execution_plan.root_plan_step, deserialized_execution_plan.root_plan_step
34
35
  assert_equal execution_plan.steps.keys, deserialized_execution_plan.steps.keys
@@ -44,6 +45,20 @@ module Dynflow
44
45
 
45
46
  end
46
47
 
48
+ describe '#label' do
49
+ let :execution_plan do
50
+ world.plan(Support::CodeWorkflowExample::FastCommit, 'sha' => 'abc123')
51
+ end
52
+
53
+ let :dummy_execution_plan do
54
+ world.plan(Support::CodeWorkflowExample::Dummy)
55
+ end
56
+
57
+ it 'is determined by the action#label method of entry action' do
58
+ execution_plan.label.must_equal 'Support::CodeWorkflowExample::FastCommit'
59
+ dummy_execution_plan.label.must_equal 'dummy_action'
60
+ end
61
+ end
47
62
  describe '#result' do
48
63
 
49
64
  let :execution_plan do
@@ -0,0 +1,87 @@
1
+ require_relative 'test_helper'
2
+ require 'fileutils'
3
+ require 'dynflow/watchers/memory_consumption_watcher'
4
+
5
+ module Dynflow
6
+ module MemoryConsumptionWatcherTest
7
+ describe ::Dynflow::Watchers::MemoryConsumptionWatcher do
8
+ let(:world) { Minitest::Mock.new('world') }
9
+ describe 'initialization' do
10
+ it 'starts a timer on the world' do
11
+ clock = Minitest::Mock.new('clock')
12
+ world.expect(:clock, clock)
13
+ init_interval = 1000
14
+ clock.expect(:ping, true) do |clock_who, clock_when, _|
15
+ clock_when.must_equal init_interval
16
+ end
17
+
18
+ Dynflow::Watchers::MemoryConsumptionWatcher.new world, 1, initial_wait: init_interval
19
+
20
+ clock.verify
21
+ end
22
+ end
23
+
24
+ describe 'polling' do
25
+ let(:memory_info_provider) { Minitest::Mock.new('memory_info_provider') }
26
+ it 'continues to poll, if memory limit is not exceeded' do
27
+ clock = Minitest::Mock.new('clock')
28
+ # define method clock
29
+ world.expect(:clock, clock)
30
+ init_interval = 1000
31
+ polling_interval = 2000
32
+ clock.expect(:ping, true) do |clock_who, clock_when, _|
33
+ clock_when.must_equal init_interval
34
+ true
35
+ end
36
+ clock.expect(:ping, true) do |clock_who, clock_when, _|
37
+ clock_when.must_equal polling_interval
38
+ true
39
+ end
40
+ memory_info_provider.expect(:bytes, 0)
41
+
42
+ # stub the clock method to always return our mock clock
43
+ world.stub(:clock, clock) do
44
+ watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(
45
+ world,
46
+ 1,
47
+ initial_wait: init_interval,
48
+ memory_info_provider: memory_info_provider,
49
+ polling_interval: polling_interval
50
+ )
51
+ watcher.check_memory_state
52
+ end
53
+
54
+ clock.verify
55
+ memory_info_provider.verify
56
+ end
57
+
58
+ it 'terminates the world, if memory limit reached' do
59
+ clock = Minitest::Mock.new('clock')
60
+ # define method clock
61
+ world.expect(:clock, clock)
62
+ world.expect(:terminate, true)
63
+
64
+ init_interval = 1000
65
+ clock.expect(:ping, true) do |clock_who, clock_when, _|
66
+ clock_when.must_equal init_interval
67
+ true
68
+ end
69
+ memory_info_provider.expect(:bytes, 10)
70
+
71
+ # stub the clock method to always return our mock clock
72
+ watcher = Dynflow::Watchers::MemoryConsumptionWatcher.new(
73
+ world,
74
+ 1,
75
+ initial_wait: init_interval,
76
+ memory_info_provider: memory_info_provider
77
+ )
78
+ watcher.check_memory_state
79
+
80
+ clock.verify
81
+ memory_info_provider.verify
82
+ world.verify
83
+ end
84
+ end
85
+ end
86
+ end
87
+ end
@@ -1,15 +1,14 @@
1
1
  require_relative 'test_helper'
2
- require 'fileutils'
3
2
 
4
3
  module Dynflow
5
4
  module PersistenceTest
6
5
  describe 'persistence adapters' do
7
6
 
8
7
  let :execution_plans_data do
9
- [{ id: 'plan1', state: 'paused' },
10
- { id: 'plan2', state: 'stopped' },
11
- { id: 'plan3', state: 'paused' },
12
- { id: 'plan4', state: 'paused' }]
8
+ [{ id: 'plan1', :label => 'test1', state: 'paused' },
9
+ { id: 'plan2', :label => 'test2', state: 'stopped' },
10
+ { id: 'plan3', :label => 'test3', state: 'paused' },
11
+ { id: 'plan4', :label => 'test4', state: 'paused' }]
13
12
  end
14
13
 
15
14
  let :action_data do
@@ -37,6 +36,10 @@ module Dynflow
37
36
  end
38
37
  end
39
38
 
39
+ def format_time(time)
40
+ time.strftime('%Y-%m-%d %H:%M:%S')
41
+ end
42
+
40
43
  def prepare_action(plan)
41
44
  adapter.save_action(plan, action_data[:id], action_data)
42
45
  end
@@ -57,6 +60,7 @@ module Dynflow
57
60
  end
58
61
  end
59
62
 
63
+ # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
60
64
  def self.it_acts_as_persistence_adapter
61
65
  before do
62
66
  # the tests expect clean field
@@ -76,20 +80,22 @@ module Dynflow
76
80
 
77
81
  it 'supports ordering' do
78
82
  prepare_plans
79
- if adapter.ordering_by.include?(:state)
83
+ if adapter.ordering_by.include?('state')
80
84
  loaded_plans = adapter.find_execution_plans(order_by: 'state')
81
- loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3', 'plan2']
85
+ loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan3 plan4 plan2)
82
86
 
83
87
  loaded_plans = adapter.find_execution_plans(order_by: 'state', desc: true)
84
- loaded_plans.map { |h| h[:id] }.must_equal ['plan2', 'plan3', 'plan1']
88
+ loaded_plans.map { |h| h[:id] }.must_equal %w(plan2 plan1 plan3 plan4)
85
89
  end
86
90
  end
87
91
 
88
92
  it 'supports filtering' do
89
93
  prepare_plans
90
- if adapter.ordering_by.include?(:state)
94
+ if adapter.ordering_by.include?('state')
95
+ loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'] })
96
+ loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
91
97
  loaded_plans = adapter.find_execution_plans(filters: { state: ['paused'] })
92
- loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3']
98
+ loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan3', 'plan4']
93
99
 
94
100
  loaded_plans = adapter.find_execution_plans(filters: { state: ['stopped'] })
95
101
  loaded_plans.map { |h| h[:id] }.must_equal ['plan2']
@@ -98,10 +104,20 @@ module Dynflow
98
104
  loaded_plans.map { |h| h[:id] }.must_equal []
99
105
 
100
106
  loaded_plans = adapter.find_execution_plans(filters: { state: ['stopped', 'paused'] })
101
- loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan2', 'plan3']
107
+ loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan2 plan3 plan4)
102
108
 
103
109
  loaded_plans = adapter.find_execution_plans(filters: { 'state' => ['stopped', 'paused'] })
104
- loaded_plans.map { |h| h[:id] }.must_equal ['plan1', 'plan2', 'plan3']
110
+ loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan2 plan3 plan4)
111
+
112
+ loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'], :delayed => true })
113
+ loaded_plans.must_be_empty
114
+
115
+ adapter.save_delayed_plan('plan1',
116
+ :execution_plan_uuid => 'plan1',
117
+ :start_at => format_time(Time.now + 60),
118
+ :start_before => format_time(Time.now - 60))
119
+ loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'], :delayed => true })
120
+ loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
105
121
  end
106
122
  end
107
123
  end
@@ -112,7 +128,7 @@ module Dynflow
112
128
  prepare_plans
113
129
  adapter.load_execution_plan('plan1')[:id].must_equal 'plan1'
114
130
  adapter.load_execution_plan('plan1')['id'].must_equal 'plan1'
115
- adapter.load_execution_plan('plan1').keys.size.must_equal 7
131
+ adapter.load_execution_plan('plan1').keys.size.must_equal 8
116
132
 
117
133
  adapter.save_execution_plan('plan1', nil)
118
134
  -> { adapter.load_execution_plan('plan1') }.must_raise KeyError
@@ -173,11 +189,12 @@ module Dynflow
173
189
  it 'finds plans with start_before in past' do
174
190
  start_time = Time.now
175
191
  prepare_plans
176
- fmt =->(time) { time.strftime('%Y-%m-%d %H:%M:%S') }
177
- adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => fmt.call(start_time + 60), :start_before => fmt.call(start_time - 60))
178
- adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => fmt.call(start_time - 60))
179
- adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :start_at => fmt.call(start_time + 60))
180
- adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :start_at => fmt.call(start_time - 60), :start_before => fmt.call(start_time - 60))
192
+ adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => format_time(start_time + 60),
193
+ :start_before => format_time(start_time - 60))
194
+ adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => format_time(start_time - 60))
195
+ adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :start_at => format_time(start_time + 60))
196
+ adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :start_at => format_time(start_time - 60),
197
+ :start_before => format_time(start_time - 60))
181
198
  plans = adapter.find_past_delayed_plans(start_time)
182
199
  plans.length.must_equal 3
183
200
  plans.map { |plan| plan[:execution_plan_uuid] }.must_equal %w(plan2 plan4 plan1)
@@ -214,6 +214,9 @@ module Support
214
214
  end
215
215
 
216
216
  class Dummy < Dynflow::Action
217
+ def label
218
+ 'dummy_action'
219
+ end
217
220
  end
218
221
 
219
222
  class DummyWithFinalize < Dynflow::Action
@@ -168,7 +168,7 @@ module Dynflow
168
168
 
169
169
  describe "in thread executor" do
170
170
  let :world do
171
- Dynflow::Testing::InThreadWorld.instance
171
+ WorldFactory.create_world(Dynflow::Testing::InThreadWorld)
172
172
  end
173
173
 
174
174
  let :issues_data do
@@ -18,6 +18,18 @@ module Dynflow
18
18
  registered_world.meta.must_equal('fast' => true)
19
19
  end
20
20
  end
21
+
22
+ describe '#terminate' do
23
+ it 'fires an event after termination' do
24
+ terminated_event = world.terminated
25
+ terminated_event.completed?.must_equal false
26
+ world.terminate
27
+ # wait for termination process to finish, but don't block
28
+ # the test from running.
29
+ terminated_event.wait(10)
30
+ terminated_event.completed?.must_equal true
31
+ end
32
+ end
21
33
  end
22
34
  end
23
35
  end
@@ -9,7 +9,7 @@
9
9
  <thead>
10
10
  <tr>
11
11
  <th><%= order_link(:id, "Id") %></th>
12
- <th><%= order_link(:action, "Action") %></th>
12
+ <th><%= order_link(:label, "Label") %></th>
13
13
  <th><%= order_link(:state, "State") %></th>
14
14
  <th><%= order_link(:result, "Result") %></th>
15
15
  <th><%= order_link(:started_at, "Started at") %></th>
@@ -20,7 +20,7 @@
20
20
  <% @plans.each do |plan| %>
21
21
  <tr>
22
22
  <td><%= h(plan.id) %></td>
23
- <td><%= h(plan.root_plan_step.action_class.name) %></td>
23
+ <td><%= h(plan.label || plan.root_plan_step.action_class.name) %></td>
24
24
  <th><%= h(plan.state) %></th>
25
25
  <th><%= h(plan.result) %></th>
26
26
  <th><%= h(plan.started_at) %></th>
@@ -1,3 +1,11 @@
1
+ <p>
2
+ <b>Id:</b>
3
+ <%= h(@plan.id) %>
4
+ </p>
5
+ <p>
6
+ <b>Label:</b>
7
+ <%= h(@plan.label) %>
8
+ </p>
1
9
  <p>
2
10
  <b>Status:</b>
3
11
  <%= h(@plan.state) %>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dynflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.21
4
+ version: 0.8.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ivan Necas
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-03-05 00:00:00.000000000 Z
12
+ date: 2017-03-19 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: multi_json
@@ -365,6 +365,7 @@ files:
365
365
  - dynflow.gemspec
366
366
  - examples/example_helper.rb
367
367
  - examples/future_execution.rb
368
+ - examples/memory_limit_watcher.rb
368
369
  - examples/orchestrate.rb
369
370
  - examples/orchestrate_evented.rb
370
371
  - examples/remote_executor.rb
@@ -380,6 +381,7 @@ files:
380
381
  - lib/dynflow/action/rescue.rb
381
382
  - lib/dynflow/action/suspended.rb
382
383
  - lib/dynflow/action/timeouts.rb
384
+ - lib/dynflow/action/with_bulk_sub_plans.rb
383
385
  - lib/dynflow/action/with_sub_plans.rb
384
386
  - lib/dynflow/active_job/queue_adapter.rb
385
387
  - lib/dynflow/actor.rb
@@ -459,6 +461,7 @@ files:
459
461
  - lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb
460
462
  - lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb
461
463
  - lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb
464
+ - lib/dynflow/persistence_adapters/sequel_migrations/010_add_execution_plans_label.rb
462
465
  - lib/dynflow/rails.rb
463
466
  - lib/dynflow/rails/configuration.rb
464
467
  - lib/dynflow/rails/daemon.rb
@@ -493,6 +496,7 @@ files:
493
496
  - lib/dynflow/transaction_adapters/none.rb
494
497
  - lib/dynflow/utils.rb
495
498
  - lib/dynflow/version.rb
499
+ - lib/dynflow/watchers/memory_consumption_watcher.rb
496
500
  - lib/dynflow/web.rb
497
501
  - lib/dynflow/web/console.rb
498
502
  - lib/dynflow/web/console_helpers.rb
@@ -503,6 +507,7 @@ files:
503
507
  - test/abnormal_states_recovery_test.rb
504
508
  - test/action_test.rb
505
509
  - test/activejob_adapter.rb
510
+ - test/batch_sub_tasks_test.rb
506
511
  - test/clock_test.rb
507
512
  - test/concurrency_control_test.rb
508
513
  - test/coordinator_test.rb
@@ -510,6 +515,7 @@ files:
510
515
  - test/execution_plan_test.rb
511
516
  - test/executor_test.rb
512
517
  - test/future_execution_test.rb
518
+ - test/memory_cosumption_watcher_test.rb
513
519
  - test/middleware_test.rb
514
520
  - test/persistence_test.rb
515
521
  - test/prepare_travis_env.sh
@@ -581,6 +587,7 @@ test_files:
581
587
  - test/abnormal_states_recovery_test.rb
582
588
  - test/action_test.rb
583
589
  - test/activejob_adapter.rb
590
+ - test/batch_sub_tasks_test.rb
584
591
  - test/clock_test.rb
585
592
  - test/concurrency_control_test.rb
586
593
  - test/coordinator_test.rb
@@ -588,6 +595,7 @@ test_files:
588
595
  - test/execution_plan_test.rb
589
596
  - test/executor_test.rb
590
597
  - test/future_execution_test.rb
598
+ - test/memory_cosumption_watcher_test.rb
591
599
  - test/middleware_test.rb
592
600
  - test/persistence_test.rb
593
601
  - test/prepare_travis_env.sh