dynflow 0.8.37 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/doc/pages/source/documentation/index.md +39 -0
  3. data/examples/orchestrate.rb +8 -0
  4. data/lib/dynflow/action.rb +7 -0
  5. data/lib/dynflow/active_job/queue_adapter.rb +5 -0
  6. data/lib/dynflow/config.rb +49 -12
  7. data/lib/dynflow/coordinator.rb +11 -0
  8. data/lib/dynflow/director.rb +9 -8
  9. data/lib/dynflow/director/execution_plan_manager.rb +2 -2
  10. data/lib/dynflow/director/running_steps_manager.rb +1 -1
  11. data/lib/dynflow/execution_plan.rb +2 -2
  12. data/lib/dynflow/execution_plan/steps/abstract.rb +12 -5
  13. data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb +8 -0
  14. data/lib/dynflow/execution_plan/steps/finalize_step.rb +5 -0
  15. data/lib/dynflow/execution_plan/steps/run_step.rb +5 -0
  16. data/lib/dynflow/executors/parallel.rb +2 -2
  17. data/lib/dynflow/executors/parallel/core.rb +42 -14
  18. data/lib/dynflow/executors/parallel/pool.rb +3 -2
  19. data/lib/dynflow/persistence_adapters/sequel.rb +1 -1
  20. data/lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb +7 -0
  21. data/lib/dynflow/rails.rb +1 -0
  22. data/lib/dynflow/rails/configuration.rb +31 -11
  23. data/lib/dynflow/utils.rb +10 -0
  24. data/lib/dynflow/version.rb +1 -1
  25. data/lib/dynflow/web/console.rb +9 -7
  26. data/lib/dynflow/web/console_helpers.rb +16 -0
  27. data/lib/dynflow/web/filtering_helpers.rb +5 -0
  28. data/lib/dynflow/world.rb +28 -28
  29. data/test/activejob_adapter_test.rb +6 -2
  30. data/test/support/dummy_example.rb +4 -0
  31. data/test/test_helper.rb +1 -0
  32. data/test/world_test.rb +9 -4
  33. data/web/views/world_validation_result.erb +15 -0
  34. data/web/views/worlds.erb +30 -23
  35. metadata +4 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 07d302583997706186e5c5a55185a32b35757971615e5e42b73d9121cab2d23d
4
- data.tar.gz: 9b5f20a8788784b95f4f52d66f3af1b7b1425f077a370b51a2e2ca2aed2b1c10
3
+ metadata.gz: 111e155822b2ec8d00a8f0e249ce1333f6d4e826bb10247e172a523e7212a9d6
4
+ data.tar.gz: 4969844384cd6ea8add967e10495cc534bee36dbbe4fe5a9b978ecd8c09d802c
5
5
  SHA512:
6
- metadata.gz: 3cd1c9ca74ab31c3afead86db86af7565c2f34e71c48d608d0dcac3ba105f0113fed48f9560865a3acced8545dd0daa55f7149311b91d81387b8ab00e2132aa8
7
- data.tar.gz: 6fbf67d06c58b64076178aed4cd271d07d1ac89b5a6d962360dd140a63324c738552e74586d9d350e08216963ffa925cacffa53be45fb932ad9a982ada1b6bae
6
+ metadata.gz: 65b612808157a76872102e4070bbb0b0ca8e23d9eba4f032a7881d67b884f33be74dc0fc7fbbcd458d15be675e33f10ba00c0fb24e4c3e237a69e7ae27558104
7
+ data.tar.gz: 66c6e35276566b7d239a25c5f95e8b52331e70aa32f36a36027c3fbf64d64069f611b8ffaab3caac5a89bf370a4c6fadc3bb5736de8bde9d3b7f50bf7a962149
@@ -940,6 +940,45 @@ Solutions are:
940
940
  - **Offloading computation** - CPU heavy parts can be offloaded to different services
941
941
  notifying the suspended actions when the computation is done.
942
942
 
943
+ ### Multiple queues support
944
+
945
+ By default, a single queue and a pool of workers is used to process
946
+ all the actions in the system. This can cause some actions to block
947
+ execution of some higher-priority ones in the system.
948
+
949
+ To address this case, it's possible to define additional queues tied
950
+ to additional pool of workers dedicated for it. This way, they can be
951
+ processed more independently from the default queue.
952
+
953
+ To use the queue, one needs to register additional queues when defining
954
+ the executor world:
955
+
956
+ ```ruby
957
+ config = Dynflow::Config.new
958
+ config.queues.add(:slow, :pool_size => 5)
959
+ world = Dynflow::World.new(config)
960
+ ```
961
+
962
+ The action to use the queue just needs to override the `queue` method like this:
963
+
964
+
965
+ ```ruby
966
+ class MyAction < Dynflow::Action
967
+ def queue
968
+ :slow
969
+ end
970
+
971
+ def run
972
+ sleep 60
973
+ end
974
+ end
975
+ ```
976
+
977
+ In the current implementation, it's expected all the executors would
978
+ have the same set of queues defined. In the future implementation it
979
+ should be possible to have dedicated executors with just a subset of
980
+ queues
981
+
943
982
  ### Middleware
944
983
 
945
984
  Each action class has chain of middlewares which wrap phases of the action execution.
@@ -71,6 +71,10 @@ module Orchestrate
71
71
 
72
72
  class PrepareDisk < Base
73
73
 
74
+ def queue
75
+ :slow
76
+ end
77
+
74
78
  input_format do
75
79
  param :name
76
80
  end
@@ -144,6 +148,10 @@ module Orchestrate
144
148
  end
145
149
 
146
150
  if $0 == __FILE__
151
+ world = ExampleHelper.create_world do |config|
152
+ config.queues.add(:slow, :pool_size => 3)
153
+ end
154
+ ExampleHelper.set_world(world)
147
155
  ExampleHelper.world.action_logger.level = Logger::INFO
148
156
  ExampleHelper.something_should_fail!
149
157
  ExampleHelper.world.trigger(Orchestrate::CreateInfrastructure)
@@ -180,6 +180,7 @@ module Dynflow
180
180
  @from_subscription = Type! from_subscription, TrueClass, FalseClass
181
181
  end
182
182
 
183
+ # action that caused this action to be planned. Available only in planning phase
183
184
  def triggering_action
184
185
  phase! Plan
185
186
  @triggering_action
@@ -316,6 +317,12 @@ module Dynflow
316
317
  false
317
318
  end
318
319
 
320
+ # @override define what pool should the action be run in. The
321
+ # queue defined here will also be used as the default queue for
322
+ # all the steps planned under this action, unless overrided by sub-action
323
+ def queue
324
+ end
325
+
319
326
  protected
320
327
 
321
328
  def state=(state)
@@ -23,9 +23,14 @@ module Dynflow
23
23
  end
24
24
 
25
25
  class JobWrapper < Dynflow::Action
26
+ def queue
27
+ input[:queue].to_sym
28
+ end
29
+
26
30
  def plan(attributes)
27
31
  input[:job_class] = attributes['job_class']
28
32
  input[:job_arguments] = attributes['arguments']
33
+ input[:queue] = attributes['queue_name']
29
34
  plan_self
30
35
  end
31
36
 
@@ -32,15 +32,51 @@ module Dynflow
32
32
  @config.validate(self)
33
33
  end
34
34
 
35
+ def queues
36
+ @queues ||= @config.queues.finalized_config(self)
37
+ end
38
+
35
39
  def method_missing(name)
36
40
  return @cache[name] if @cache.key?(name)
37
41
  value = @config.send(name)
38
42
  value = value.call(@world, self) if value.is_a? Proc
39
- @config.send("validate_#{ name }!", value)
43
+ validation_method = "validate_#{ name }!"
44
+ @config.send(validation_method, value) if @config.respond_to?(validation_method)
40
45
  @cache[name] = value
41
46
  end
42
47
  end
43
48
 
49
+ class QueuesConfig
50
+ attr_reader :queues
51
+
52
+ def initialize
53
+ @queues = {:default => {}}
54
+ end
55
+
56
+ # Add a new queue to the configuration
57
+ #
58
+ # @param [Hash] queue_options
59
+ # @option queue_options :pool_size The amount of workers available for the queue.
60
+ # By default, it uses global pool_size config option.
61
+ def add(name, queue_options = {})
62
+ Utils.validate_keys!(queue_options, :pool_size)
63
+ name = name.to_sym
64
+ raise ArgumentError, "Queue #{name} is already defined" if @queues.key?(name)
65
+ @queues[name] = queue_options
66
+ end
67
+
68
+ def finalized_config(config_for_world)
69
+ @queues.values.each do |queue_options|
70
+ queue_options[:pool_size] ||= config_for_world.pool_size
71
+ end
72
+ @queues
73
+ end
74
+ end
75
+
76
+ def queues
77
+ @queues ||= QueuesConfig.new
78
+ end
79
+
44
80
  config_attr :logger_adapter, LoggerAdapters::Abstract do
45
81
  LoggerAdapters::Simple.new
46
82
  end
@@ -62,7 +98,7 @@ module Dynflow
62
98
  end
63
99
 
64
100
  config_attr :executor, Executors::Abstract, FalseClass do |world, config|
65
- Executors::Parallel.new(world, config.pool_size)
101
+ Executors::Parallel.new(world, config.queues)
66
102
  end
67
103
 
68
104
  config_attr :executor_semaphore, Semaphores::Abstract, FalseClass do |world, config|
@@ -126,7 +162,7 @@ module Dynflow
126
162
  Action.all_children
127
163
  end
128
164
 
129
- config_attr :meta do
165
+ config_attr :meta do |world, config|
130
166
  { 'hostname' => Socket.gethostname, 'pid' => Process.pid }
131
167
  end
132
168
 
@@ -140,17 +176,18 @@ module Dynflow
140
176
 
141
177
  def validate(config_for_world)
142
178
  if defined? ::ActiveRecord::Base
143
- ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
144
- if (config_for_world.pool_size / 2.0) > ar_pool_size
145
- config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
146
- "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
147
- 'threads in Dynflow pool.'
179
+ begin
180
+ ar_pool_size = ::ActiveRecord::Base.connection_pool.instance_variable_get(:@size)
181
+ if (config_for_world.pool_size / 2.0) > ar_pool_size
182
+ config_for_world.world.logger.warn 'Consider increasing ActiveRecord::Base.connection_pool size, ' +
183
+ "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
184
+ 'threads in Dynflow pool.'
185
+ end
186
+ rescue ActiveRecord::ConnectionNotEstablished # rubocop:disable Lint/HandleExceptions
187
+ # If in tests or in an environment where ActiveRecord doesn't have a
188
+ # real DB connection, we want to skip AR configuration altogether
148
189
  end
149
190
  end
150
-
151
- rescue ActiveRecord::ConnectionNotEstablished # rubocop:disable Lint/HandleExceptions
152
- # If in tests or in an environment where ActiveRecord doesn't have a
153
- # real DB connection, we want to skip AR configuration altogether
154
191
  end
155
192
  end
156
193
  end
@@ -89,6 +89,10 @@ module Dynflow
89
89
  def meta
90
90
  @data[:meta]
91
91
  end
92
+
93
+ def executor?
94
+ raise NotImplementedError
95
+ end
92
96
  end
93
97
 
94
98
  class ExecutorWorld < WorldRecord
@@ -105,9 +109,16 @@ module Dynflow
105
109
  Type! value, Algebrick::Types::Boolean
106
110
  @data[:active] = value
107
111
  end
112
+
113
+ def executor?
114
+ true
115
+ end
108
116
  end
109
117
 
110
118
  class ClientWorld < WorldRecord
119
+ def executor?
120
+ false
121
+ end
111
122
  end
112
123
 
113
124
  class Lock < Record
@@ -19,10 +19,11 @@ module Dynflow
19
19
  UnprocessableEvent = Class.new(Dynflow::Error)
20
20
 
21
21
  class WorkItem
22
- attr_reader :execution_plan_id
22
+ attr_reader :execution_plan_id, :queue
23
23
 
24
- def initialize(execution_plan_id)
24
+ def initialize(execution_plan_id, queue)
25
25
  @execution_plan_id = execution_plan_id
26
+ @queue = queue
26
27
  end
27
28
 
28
29
  def execute
@@ -33,8 +34,8 @@ module Dynflow
33
34
  class StepWorkItem < WorkItem
34
35
  attr_reader :step
35
36
 
36
- def initialize(execution_plan_id, step)
37
- super(execution_plan_id)
37
+ def initialize(execution_plan_id, step, queue)
38
+ super(execution_plan_id, queue)
38
39
  @step = step
39
40
  end
40
41
 
@@ -46,8 +47,8 @@ module Dynflow
46
47
  class EventWorkItem < StepWorkItem
47
48
  attr_reader :event
48
49
 
49
- def initialize(execution_plan_id, step, event)
50
- super(execution_plan_id, step)
50
+ def initialize(execution_plan_id, step, event, queue)
51
+ super(execution_plan_id, step, queue)
51
52
  @event = event
52
53
  end
53
54
 
@@ -57,8 +58,8 @@ module Dynflow
57
58
  end
58
59
 
59
60
  class FinalizeWorkItem < WorkItem
60
- def initialize(execution_plan_id, sequential_manager)
61
- super(execution_plan_id)
61
+ def initialize(execution_plan_id, sequential_manager, queue)
62
+ super(execution_plan_id, queue)
62
63
  @sequential_manager = sequential_manager
63
64
  end
64
65
 
@@ -25,7 +25,7 @@ module Dynflow
25
25
  end
26
26
 
27
27
  def prepare_next_step(step)
28
- StepWorkItem.new(execution_plan.id, step).tap do |work|
28
+ StepWorkItem.new(execution_plan.id, step, step.queue).tap do |work|
29
29
  @running_steps_manager.add(step, work)
30
30
  end
31
31
  end
@@ -93,7 +93,7 @@ module Dynflow
93
93
  return if execution_plan.finalize_flow.empty?
94
94
  raise 'finalize phase already started' if @finalize_manager
95
95
  @finalize_manager = SequentialManager.new(@world, execution_plan)
96
- [FinalizeWorkItem.new(execution_plan.id, @finalize_manager)]
96
+ [FinalizeWorkItem.new(execution_plan.id, @finalize_manager, execution_plan.finalize_steps.first.queue)]
97
97
  end
98
98
 
99
99
  def finish
@@ -69,7 +69,7 @@ module Dynflow
69
69
  end
70
70
 
71
71
  can_run_event = @events.empty?(step.id)
72
- work = EventWorkItem.new(event.execution_plan_id, step, event)
72
+ work = EventWorkItem.new(event.execution_plan_id, step, event, step.queue)
73
73
  @events.push(step.id, work)
74
74
  next_work_items << work if can_run_event
75
75
  next_work_items
@@ -389,7 +389,7 @@ module Dynflow
389
389
 
390
390
  def add_run_step(action)
391
391
  add_step(Steps::RunStep, action.class, action.id).tap do |step|
392
- step.progress_weight = action.run_progress_weight
392
+ step.update_from_action(action)
393
393
  @dependency_graph.add_dependencies(step, action)
394
394
  current_run_flow.add_and_resolve(@dependency_graph, Flows::Atom.new(step.id))
395
395
  end
@@ -397,7 +397,7 @@ module Dynflow
397
397
 
398
398
  def add_finalize_step(action)
399
399
  add_step(Steps::FinalizeStep, action.class, action.id).tap do |step|
400
- step.progress_weight = action.finalize_progress_weight
400
+ step.update_from_action(action)
401
401
  finalize_flow << Flows::Atom.new(step.id)
402
402
  end
403
403
  end
@@ -5,9 +5,10 @@ module Dynflow
5
5
  include Stateful
6
6
 
7
7
  attr_reader :execution_plan_id, :id, :state, :action_class, :action_id, :world, :started_at,
8
- :ended_at, :execution_time, :real_time
8
+ :ended_at, :execution_time, :real_time, :queue
9
9
  attr_accessor :error
10
10
 
11
+ # rubocop:disable Metrics/ParameterLists
11
12
  def initialize(execution_plan_id,
12
13
  id,
13
14
  state,
@@ -20,7 +21,8 @@ module Dynflow
20
21
  execution_time = 0.0,
21
22
  real_time = 0.0,
22
23
  progress_done = nil,
23
- progress_weight = nil)
24
+ progress_weight = nil,
25
+ queue = nil)
24
26
 
25
27
  @id = id || raise(ArgumentError, 'missing id')
26
28
  @execution_plan_id = Type! execution_plan_id, String
@@ -34,6 +36,8 @@ module Dynflow
34
36
  @progress_done = Type! progress_done, Numeric, NilClass
35
37
  @progress_weight = Type! progress_weight, Numeric, NilClass
36
38
 
39
+ @queue = Type! queue, Symbol, NilClass
40
+
37
41
  self.state = state.to_sym
38
42
 
39
43
  Child! action_class, Action
@@ -41,6 +45,7 @@ module Dynflow
41
45
 
42
46
  @action_id = action_id || raise(ArgumentError, 'missing action_id')
43
47
  end
48
+ # rubocop:enable Metrics/ParameterLists
44
49
 
45
50
  def action_logger
46
51
  @world.action_logger
@@ -87,7 +92,8 @@ module Dynflow
87
92
  execution_time: execution_time,
88
93
  real_time: real_time,
89
94
  progress_done: progress_done,
90
- progress_weight: progress_weight
95
+ progress_weight: progress_weight,
96
+ queue: queue
91
97
  end
92
98
 
93
99
  def progress_done
@@ -132,7 +138,7 @@ module Dynflow
132
138
 
133
139
  def self.new_from_hash(hash, execution_plan_id, world)
134
140
  check_class_matching hash
135
- new execution_plan_id,
141
+ new(execution_plan_id,
136
142
  hash[:id],
137
143
  hash[:state],
138
144
  Action.constantize(hash[:action_class]),
@@ -144,7 +150,8 @@ module Dynflow
144
150
  hash[:execution_time].to_f,
145
151
  hash[:real_time].to_f,
146
152
  hash[:progress_done].to_f,
147
- hash[:progress_weight].to_f
153
+ hash[:progress_weight].to_f,
154
+ (hash[:queue] && hash[:queue].to_sym))
148
155
  end
149
156
 
150
157
  private
@@ -2,6 +2,14 @@ module Dynflow
2
2
  module ExecutionPlan::Steps
3
3
  class AbstractFlowStep < Abstract
4
4
 
5
+ # Method called when initializing the step to customize the behavior based on the
6
+ # action definition during the planning phase
7
+ def update_from_action(action)
8
+ @queue = action.queue
9
+ @queue ||= action.triggering_action.queue if action.triggering_action
10
+ @queue ||= :default
11
+ end
12
+
5
13
  def execute(*args)
6
14
  return self if [:skipped, :success].include? self.state
7
15
  open_action do |action|
@@ -13,6 +13,11 @@ module Dynflow
13
13
  }
14
14
  end
15
15
 
16
+ def update_from_action(action)
17
+ super
18
+ self.progress_weight = action.finalize_progress_weight
19
+ end
20
+
16
21
  def phase
17
22
  Action::Finalize
18
23
  end
@@ -14,6 +14,11 @@ module Dynflow
14
14
  }
15
15
  end
16
16
 
17
+ def update_from_action(action)
18
+ super
19
+ self.progress_weight = action.run_progress_weight
20
+ end
21
+
17
22
  def phase
18
23
  Action::Run
19
24
  end
@@ -5,10 +5,10 @@ module Dynflow
5
5
  require 'dynflow/executors/parallel/pool'
6
6
  require 'dynflow/executors/parallel/worker'
7
7
 
8
- def initialize(world, pool_size = 10)
8
+ def initialize(world, queues_options = { :default => { :pool_size => 5 }})
9
9
  super(world)
10
10
  @core = Core.spawn name: 'parallel-executor-core',
11
- args: [world, pool_size],
11
+ args: [world, queues_options],
12
12
  initialized: @core_initialized = Concurrent.future
13
13
  end
14
14
 
@@ -1,16 +1,28 @@
1
1
  module Dynflow
2
2
  module Executors
3
3
  class Parallel < Abstract
4
-
5
4
  class Core < Actor
6
5
  attr_reader :logger
7
6
 
8
- def initialize(world, pool_size)
9
- @logger = world.logger
10
- @world = Type! world, World
11
- @pool = Pool.spawn('pool', reference, pool_size, world.transaction_adapter)
12
- @terminated = nil
13
- @director = Director.new(@world)
7
+ def initialize(world, queues_options)
8
+ @logger = world.logger
9
+ @world = Type! world, World
10
+ @queues_options = queues_options
11
+ @pools = {}
12
+ @terminated = nil
13
+ @director = Director.new(@world)
14
+
15
+ initialize_queues
16
+ end
17
+
18
+ def initialize_queues
19
+ default_pool_size = @queues_options[:default][:pool_size]
20
+ @queues_options.each do |(queue_name, queue_options)|
21
+ queue_pool_size = queue_options.fetch(:pool_size, default_pool_size)
22
+ @pools[queue_name] = Pool.spawn("pool #{queue_name}", reference,
23
+ queue_name, queue_pool_size,
24
+ @world.transaction_adapter)
25
+ end
14
26
  end
15
27
 
16
28
  def handle_execution(execution_plan_id, finished)
@@ -43,14 +55,17 @@ module Dynflow
43
55
 
44
56
  def start_termination(*args)
45
57
  super
46
- logger.info 'shutting down Dynflow Core ...'
47
- @pool.tell([:start_termination, Concurrent.future])
58
+ logger.info 'shutting down Core ...'
59
+ @pools.values.each { |pool| pool.tell([:start_termination, Concurrent.future]) }
48
60
  end
49
61
 
50
- def finish_termination
62
+ def finish_termination(pool_name)
63
+ @pools.delete(pool_name)
64
+ # we expect this message from all worker pools
65
+ return unless @pools.empty?
51
66
  @director.terminate
52
- logger.error '... Dynflow Core has shut down.'
53
- super
67
+ logger.error '... core terminated.'
68
+ super()
54
69
  end
55
70
 
56
71
  def dead_letter_routing
@@ -58,7 +73,9 @@ module Dynflow
58
73
  end
59
74
 
60
75
  def execution_status(execution_plan_id = nil)
61
- @pool.ask!([:execution_status, execution_plan_id])
76
+ @pools.each_with_object({}) do |(pool_name, pool), hash|
77
+ hash[pool_name] = pool.ask!([:execution_status, execution_plan_id])
78
+ end
62
79
  end
63
80
 
64
81
  private
@@ -74,7 +91,18 @@ module Dynflow
74
91
  return if work_items.nil?
75
92
  work_items = [work_items] if work_items.is_a? Director::WorkItem
76
93
  work_items.all? { |i| Type! i, Director::WorkItem }
77
- work_items.each { |new_work| @pool.tell([:schedule_work, new_work]) }
94
+ work_items.each do |new_work|
95
+ pool = @pools[new_work.queue]
96
+ unless pool
97
+ logger.error("Pool is not available for queue #{new_work.queue}, falling back to #{fallback_queue}")
98
+ pool = @pools[fallback_queue]
99
+ end
100
+ pool.tell([:schedule_work, new_work])
101
+ end
102
+ end
103
+
104
+ def fallback_queue
105
+ :default
78
106
  end
79
107
  end
80
108
  end
@@ -46,7 +46,8 @@ module Dynflow
46
46
  end
47
47
  end
48
48
 
49
- def initialize(core, pool_size, transaction_adapter)
49
+ def initialize(core, name, pool_size, transaction_adapter)
50
+ @name = name
50
51
  @executor_core = core
51
52
  @pool_size = pool_size
52
53
  @free_workers = Array.new(pool_size) { |i| Worker.spawn("worker-#{i}", reference, transaction_adapter) }
@@ -84,7 +85,7 @@ module Dynflow
84
85
  def try_to_terminate
85
86
  if terminating? && @free_workers.size == @pool_size
86
87
  @free_workers.map { |worker| worker.ask(:terminate!) }.map(&:wait)
87
- @executor_core.tell(:finish_termination)
88
+ @executor_core.tell([:finish_termination, @name])
88
89
  finish_termination
89
90
  end
90
91
  end
@@ -33,7 +33,7 @@ module Dynflow
33
33
  META_DATA = { execution_plan: %w(label state result started_at ended_at real_time execution_time root_plan_step_id class),
34
34
  action: %w(caller_execution_plan_id caller_action_id class plan_step_id run_step_id finalize_step_id),
35
35
  step: %w(state started_at ended_at real_time execution_time action_id progress_done progress_weight
36
- class action_class execution_plan_uuid),
36
+ class action_class execution_plan_uuid queue),
37
37
  envelope: %w(receiver_id),
38
38
  coordinator_record: %w(id owner_id class),
39
39
  delayed: %w(execution_plan_uuid start_at start_before args_serializer)}
@@ -0,0 +1,7 @@
1
+ Sequel.migration do
2
+ change do
3
+ alter_table(:dynflow_steps) do
4
+ add_column :queue, String
5
+ end
6
+ end
7
+ end
data/lib/dynflow/rails.rb CHANGED
@@ -38,6 +38,7 @@ module Dynflow
38
38
  @world = world
39
39
 
40
40
  unless config.remote?
41
+ config.increase_db_pool_size(world)
41
42
  config.run_on_init_hooks(world)
42
43
  # leave this just for long-running executors
43
44
  unless config.rake_task_with_executor?
@@ -7,7 +7,8 @@ module Dynflow
7
7
  # the number of threads in the pool handling the execution
8
8
  attr_accessor :pool_size
9
9
 
10
- # the size of db connection pool
10
+ # the size of db connection pool, if not set, it's calculated
11
+ # from the amount of workers in the pool
11
12
  attr_accessor :db_pool_size
12
13
 
13
14
  # set true if the executor runs externally (by default true in procution, othewise false)
@@ -32,7 +33,6 @@ module Dynflow
32
33
 
33
34
  def initialize
34
35
  self.pool_size = 5
35
- self.db_pool_size = pool_size + 5
36
36
  self.remote = ::Rails.env.production?
37
37
  self.transaction_adapter = ::Dynflow::TransactionAdapters::ActiveRecord.new
38
38
  self.eager_load_paths = []
@@ -83,13 +83,24 @@ module Dynflow
83
83
  end
84
84
 
85
85
  def increase_db_pool_size?
86
- !::Rails.env.test?
86
+ !::Rails.env.test? && !remote?
87
+ end
88
+
89
+ def calculate_db_pool_size(world)
90
+ self.db_pool_size || world.config.queues.values.inject(5) do |pool_size, pool_options|
91
+ pool_size += pool_options[:pool_size]
92
+ end
87
93
  end
88
94
 
89
95
  # To avoid pottential timeouts on db connection pool, make sure
90
96
  # we have the pool bigger than the thread pool
91
- def increase_db_pool_size
97
+ def increase_db_pool_size(world = nil)
98
+ if world.nil?
99
+ warn 'Deprecated: using `increase_db_pool_size` outside of Dynflow code is not needed anymore'
100
+ return
101
+ end
92
102
  if increase_db_pool_size?
103
+ db_pool_size = calculate_db_pool_size(world)
93
104
  ::ActiveRecord::Base.connection_pool.disconnect!
94
105
 
95
106
  config = ::ActiveRecord::Base.configurations[::Rails.env]
@@ -100,11 +111,11 @@ module Dynflow
100
111
 
101
112
  # generates the options hash consumable by the Dynflow's world
102
113
  def world_config
103
- ::Dynflow::Config.new.tap do |config|
114
+ @world_config ||= ::Dynflow::Config.new.tap do |config|
104
115
  config.auto_rescue = true
105
116
  config.logger_adapter = ::Dynflow::LoggerAdapters::Delegator.new(action_logger, dynflow_logger)
106
117
  config.pool_size = 5
107
- config.persistence_adapter = initialize_persistence
118
+ config.persistence_adapter = ->(world, _) { initialize_persistence(world) }
108
119
  config.transaction_adapter = transaction_adapter
109
120
  config.executor = ->(world, _) { initialize_executor(world) }
110
121
  config.connector = ->(world, _) { initialize_connector(world) }
@@ -115,13 +126,18 @@ module Dynflow
115
126
  end
116
127
  end
117
128
 
129
+ # expose the queues definition to Rails developers
130
+ def queues
131
+ world_config.queues
132
+ end
133
+
118
134
  protected
119
135
 
120
- def default_sequel_adapter_options
136
+ def default_sequel_adapter_options(world)
121
137
  db_config = ::ActiveRecord::Base.configurations[::Rails.env].dup
122
138
  db_config['adapter'] = db_config['adapter'].gsub(/_?makara_?/, '')
123
139
  db_config['adapter'] = 'postgres' if db_config['adapter'] == 'postgresql'
124
- db_config['max_connections'] = db_pool_size if increase_db_pool_size?
140
+ db_config['max_connections'] = calculate_db_pool_size(world) if increase_db_pool_size?
125
141
 
126
142
  if db_config['adapter'] == 'sqlite3'
127
143
  db_config['adapter'] = 'sqlite'
@@ -139,7 +155,7 @@ module Dynflow
139
155
  if remote?
140
156
  false
141
157
  else
142
- ::Dynflow::Executors::Parallel.new(world, pool_size)
158
+ ::Dynflow::Executors::Parallel.new(world, world.config.queues)
143
159
  end
144
160
  end
145
161
 
@@ -147,9 +163,13 @@ module Dynflow
147
163
  ::Dynflow::Connectors::Database.new(world)
148
164
  end
149
165
 
166
+ def persistence_class
167
+ ::Dynflow::PersistenceAdapters::Sequel
168
+ end
169
+
150
170
  # Sequel adapter based on Rails app database.yml configuration
151
- def initialize_persistence
152
- ::Dynflow::PersistenceAdapters::Sequel.new(default_sequel_adapter_options)
171
+ def initialize_persistence(world)
172
+ persistence_class.new(default_sequel_adapter_options(world))
153
173
  end
154
174
  end
155
175
  end
data/lib/dynflow/utils.rb CHANGED
@@ -1,6 +1,16 @@
1
1
  module Dynflow
2
2
  module Utils
3
3
 
4
+ def self.validate_keys!(hash, *valid_keys)
5
+ valid_keys.flatten!
6
+ unexpected_options = hash.keys - valid_keys - valid_keys.map(&:to_s)
7
+ unless unexpected_options.empty?
8
+ raise ArgumentError, "Unexpected options #{unexpected_options.inspect}. "\
9
+ "Valid keys are: #{valid_keys.map(&:inspect).join(', ')}"
10
+ end
11
+ hash
12
+ end
13
+
4
14
  def self.symbolize_keys(hash)
5
15
  return hash.symbolize_keys if hash.respond_to?(:symbolize_keys)
6
16
  hash.reduce({}) do |new_hash, (key, value)|
@@ -1,3 +1,3 @@
1
1
  module Dynflow
2
- VERSION = '0.8.37'.freeze
2
+ VERSION = '1.0.0'.freeze
3
3
  end
@@ -33,28 +33,30 @@ module Dynflow
33
33
  end
34
34
 
35
35
  get('/worlds') do
36
- @worlds = world.coordinator.find_worlds
36
+ load_worlds
37
37
  erb :worlds
38
38
  end
39
39
 
40
40
  post('/worlds/execution_status') do
41
- @worlds = world.coordinator.find_worlds(true)
42
- @worlds.each do |w|
41
+ load_worlds
42
+ @executors.each do |w|
43
43
  hash = world.get_execution_status(w.data['id'], nil, 5).value!
44
- hash[:execution_status] = hash[:execution_status].values.reduce(:+) || 0
45
- w.data.update(hash)
44
+ hash.each do |_queue_name, info|
45
+ info[:queue_size] = info[:execution_status].values.reduce(:+) || 0
46
+ end
47
+ w.data.update(:status => hash)
46
48
  end
47
49
  erb :worlds
48
50
  end
49
51
 
50
52
  post('/worlds/check') do
51
- @worlds = world.coordinator.find_worlds
53
+ load_worlds
52
54
  @validation_results = world.worlds_validity_check(params[:invalidate])
53
55
  erb :worlds
54
56
  end
55
57
 
56
58
  post('/worlds/:id/check') do |id|
57
- @worlds = world.coordinator.find_worlds
59
+ load_worlds
58
60
  @validation_results = world.worlds_validity_check(params[:invalidate], id: params[:id])
59
61
  erb :worlds
60
62
  end
@@ -9,6 +9,22 @@ module Dynflow
9
9
  end
10
10
  end
11
11
 
12
+ def value_field(key, value)
13
+ strip_heredoc(<<-HTML)
14
+ <p>
15
+ <b>#{key}:</b>
16
+ #{h(value)}
17
+ </p>
18
+ HTML
19
+ end
20
+
21
+ def strip_heredoc(str)
22
+ strip_size = str.lines.reject { |l| l =~ /^\s*$/ }.map { |l| l[/^\s*/].length }.min
23
+ str.lines.map do |line|
24
+ line[strip_size..-1] || ''
25
+ end.join
26
+ end
27
+
12
28
  def prettify_value(value)
13
29
  YAML.dump(value)
14
30
  end
@@ -28,6 +28,11 @@ module Dynflow
28
28
  return @filtering_options
29
29
  end
30
30
 
31
+ def load_worlds(executors_only = false)
32
+ @worlds = world.coordinator.find_worlds(executors_only)
33
+ @executors, @clients = @worlds.partition(&:executor?)
34
+ end
35
+
31
36
  def find_execution_plans_options(show_all = false)
32
37
  options = Utils.indifferent_hash({})
33
38
  options.merge!(filtering_options(show_all))
data/lib/dynflow/world.rb CHANGED
@@ -4,7 +4,7 @@ module Dynflow
4
4
  include Algebrick::TypeCheck
5
5
  include Algebrick::Matching
6
6
 
7
- attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
7
+ attr_reader :id, :config, :client_dispatcher, :executor_dispatcher, :executor, :connector,
8
8
  :transaction_adapter, :logger_adapter, :coordinator,
9
9
  :persistence, :action_classes, :subscription_index,
10
10
  :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter,
@@ -13,53 +13,53 @@ module Dynflow
13
13
  def initialize(config)
14
14
  @id = SecureRandom.uuid
15
15
  @clock = spawn_and_wait(Clock, 'clock')
16
- config_for_world = Config::ForWorld.new(config, self)
17
- @logger_adapter = config_for_world.logger_adapter
18
- config_for_world.validate
19
- @transaction_adapter = config_for_world.transaction_adapter
20
- @persistence = Persistence.new(self, config_for_world.persistence_adapter,
21
- :backup_deleted_plans => config_for_world.backup_deleted_plans,
22
- :backup_dir => config_for_world.backup_dir)
23
- @coordinator = Coordinator.new(config_for_world.coordinator_adapter)
24
- @executor = config_for_world.executor
25
- @action_classes = config_for_world.action_classes
26
- @auto_rescue = config_for_world.auto_rescue
27
- @exit_on_terminate = Concurrent::AtomicBoolean.new(config_for_world.exit_on_terminate)
28
- @connector = config_for_world.connector
16
+ @config = Config::ForWorld.new(config, self)
17
+ @logger_adapter = @config.logger_adapter
18
+ @config.validate
19
+ @transaction_adapter = @config.transaction_adapter
20
+ @persistence = Persistence.new(self, @config.persistence_adapter,
21
+ :backup_deleted_plans => @config.backup_deleted_plans,
22
+ :backup_dir => @config.backup_dir)
23
+ @coordinator = Coordinator.new(@config.coordinator_adapter)
24
+ @executor = @config.executor
25
+ @action_classes = @config.action_classes
26
+ @auto_rescue = @config.auto_rescue
27
+ @exit_on_terminate = Concurrent::AtomicBoolean.new(@config.exit_on_terminate)
28
+ @connector = @config.connector
29
29
  @middleware = Middleware::World.new
30
30
  @middleware.use Middleware::Common::Transaction if @transaction_adapter
31
31
  @client_dispatcher = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self)
32
- @dead_letter_handler = spawn_and_wait(DeadLetterSilencer, 'default_dead_letter_handler', config_for_world.silent_dead_letter_matchers)
33
- @meta = config_for_world.meta
34
- @auto_validity_check = config_for_world.auto_validity_check
35
- @validity_check_timeout = config_for_world.validity_check_timeout
36
- @throttle_limiter = config_for_world.throttle_limiter
32
+ @dead_letter_handler = spawn_and_wait(DeadLetterSilencer, 'default_dead_letter_handler', @config.silent_dead_letter_matchers)
33
+ @auto_validity_check = @config.auto_validity_check
34
+ @validity_check_timeout = @config.validity_check_timeout
35
+ @throttle_limiter = @config.throttle_limiter
37
36
  @terminated = Concurrent.event
38
- @termination_timeout = config_for_world.termination_timeout
37
+ @termination_timeout = @config.termination_timeout
39
38
  calculate_subscription_index
40
39
 
41
40
  if executor
42
- @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self, config_for_world.executor_semaphore)
41
+ @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self, @config.executor_semaphore)
43
42
  executor.initialized.wait
44
43
  end
45
44
  perform_validity_checks if auto_validity_check
46
45
 
47
- @delayed_executor = try_spawn(config_for_world, :delayed_executor, Coordinator::DelayedExecutorLock)
48
- @execution_plan_cleaner = try_spawn(config_for_world, :execution_plan_cleaner, Coordinator::ExecutionPlanCleanerLock)
49
- @meta = config_for_world.meta
46
+ @delayed_executor = try_spawn(:delayed_executor, Coordinator::DelayedExecutorLock)
47
+ @execution_plan_cleaner = try_spawn(:execution_plan_cleaner, Coordinator::ExecutionPlanCleanerLock)
48
+ @meta = @config.meta
49
+ @meta['queues'] = @config.queues if @executor
50
50
  @meta['delayed_executor'] = true if @delayed_executor
51
51
  @meta['execution_plan_cleaner'] = true if @execution_plan_cleaner
52
52
  coordinator.register_world(registered_world)
53
53
  @termination_barrier = Mutex.new
54
54
  @before_termination_hooks = Queue.new
55
55
 
56
- if config_for_world.auto_terminate
56
+ if @config.auto_terminate
57
57
  at_exit do
58
58
  @exit_on_terminate.make_false # make sure we don't terminate twice
59
59
  self.terminate.wait
60
60
  end
61
61
  end
62
- self.auto_execute if config_for_world.auto_execute
62
+ self.auto_execute if @config.auto_execute
63
63
  @delayed_executor.start if @delayed_executor
64
64
  end
65
65
 
@@ -402,9 +402,9 @@ module Dynflow
402
402
  []
403
403
  end
404
404
 
405
- def try_spawn(config_for_world, what, lock_class = nil)
405
+ def try_spawn(what, lock_class = nil)
406
406
  object = nil
407
- return nil if !executor || (object = config_for_world.public_send(what)).nil?
407
+ return nil if !executor || (object = @config.public_send(what)).nil?
408
408
 
409
409
  coordinator.acquire(lock_class.new(self)) if lock_class
410
410
  object.spawn.wait
@@ -4,6 +4,8 @@ require 'dynflow/active_job/queue_adapter'
4
4
 
5
5
  module Dynflow
6
6
  class SampleJob < ::ActiveJob::Base
7
+ queue_as :slow
8
+
7
9
  def perform(msg)
8
10
  puts "This job says #{msg}"
9
11
  end
@@ -20,8 +22,10 @@ module Dynflow
20
22
  rails_app_mock .expect(:dynflow, dynflow_mock)
21
23
  rails_mock = Minitest::Mock.new
22
24
  rails_mock.expect(:application, rails_app_mock)
23
- @original_rails = ::Rails
24
- Object.send(:remove_const, 'Rails')
25
+ if defined? ::Rails
26
+ @original_rails = ::Rails
27
+ Object.send(:remove_const, 'Rails')
28
+ end
25
29
  Object.const_set('Rails', rails_mock)
26
30
  end
27
31
 
@@ -38,6 +38,10 @@ module Support
38
38
  $slow_actions_done ||= 0
39
39
  $slow_actions_done +=1
40
40
  end
41
+
42
+ def queue
43
+ :slow
44
+ end
41
45
  end
42
46
 
43
47
  class Polling < Dynflow::Action
data/test/test_helper.rb CHANGED
@@ -95,6 +95,7 @@ module WorldFactory
95
95
  config.auto_terminate = false
96
96
  config.backup_deleted_plans = false
97
97
  config.backup_dir = nil
98
+ config.queues.add(:slow, :pool_size => 1)
98
99
  yield config if block_given?
99
100
  return config
100
101
  end
data/test/world_test.rb CHANGED
@@ -10,24 +10,29 @@ module Dynflow
10
10
  describe '#meta' do
11
11
  it 'by default informs about the hostname and the pid running the world' do
12
12
  registered_world = world.coordinator.find_worlds(false, id: world.id).first
13
- registered_world.meta.must_equal('hostname' => Socket.gethostname, 'pid' => Process.pid)
13
+ registered_world.meta.must_equal('hostname' => Socket.gethostname, 'pid' => Process.pid,
14
+ 'queues' => { 'default' => { 'pool_size' => 5 },
15
+ 'slow' => { 'pool_size' => 1 }})
14
16
  end
15
17
 
16
18
  it 'is configurable' do
17
19
  registered_world = world.coordinator.find_worlds(false, id: world_with_custom_meta.id).first
18
- registered_world.meta.must_equal('fast' => true)
20
+ registered_world.meta['fast'].must_equal true
19
21
  end
20
22
  end
21
23
 
22
24
  describe '#get_execution_status' do
23
25
  let(:base) do
24
- { :pool_size => 5, :free_workers => 5, :execution_status => {} }
26
+ { :default => { :pool_size => 5, :free_workers => 5, :execution_status => {} },
27
+ :slow => { :pool_size=> 1, :free_workers=> 1, :execution_status=> {}} }
25
28
  end
26
29
 
27
30
  it 'retrieves correct execution items count' do
28
31
  world.get_execution_status(world.id, nil, 5).value!.must_equal(base)
29
32
  id = 'something like uuid'
30
- expected = base.merge(:execution_status => { id => 0 })
33
+ expected = base.dup
34
+ expected[:default][:execution_status] = { id => 0 }
35
+ expected[:slow][:execution_status] = { id => 0 }
31
36
  world.get_execution_status(world.id, id, 5).value!.must_equal(expected)
32
37
  end
33
38
  end
@@ -0,0 +1,15 @@
1
+ <% validation_result = @validation_results[world.id] if @validation_results %>
2
+
3
+ <% unless validation_result == :invalidated %>
4
+ <a href="<%= url("/worlds/#{world.id}/check") %>" class="postlink">check</a>
5
+ <% end %>
6
+
7
+ <% if validation_result %>
8
+ <% if validation_result == :invalid %>
9
+ <a href="<%= url("/worlds/#{world.id}/check?invalidate=true") %>" class="postlink">invalidate</a>
10
+ <% end %>
11
+
12
+ <span class="label label-<%= validation_result_css_class(validation_result) %>">
13
+ <%= h(validation_result) %>
14
+ </span>
15
+ <% end %>
data/web/views/worlds.erb CHANGED
@@ -6,41 +6,48 @@
6
6
  <li><a href="<%= url("/worlds/execution_status") %>" class="postlink">load execution items counts</a>: see counts of execution items per world</li>
7
7
  </ul>
8
8
 
9
+ <h3>Executors</h3>
10
+ <% @executors.each do |world| %>
11
+ <%= value_field('Id', world.id) %>
12
+ <%= value_field('Metadata', world.meta) %>
13
+ <p>
14
+ <b>Status:</b>
15
+ <%= erb :world_validation_result, locals: { world: world } %>
16
+ </p>
17
+ <% if world.data[:status] %>
18
+ <table class="table">
19
+ <thead>
20
+ <tr>
21
+ <th>Queue name</th>
22
+ <th>Queue size</th>
23
+ <th>Free/Total workers</th>
24
+ </tr>
25
+ </thead>
26
+ <% world.data[:status].each do |queue_name, info| %>
27
+ <tr>
28
+ <td><%= h(queue_name) %></td>
29
+ <td><%= h(info[:queue_size]) %></td>
30
+ <td><%= h(info[:free_workers]) %>/<%= h(info[:pool_size]) %></td>
31
+ </tr>
32
+ <% end %>
33
+ </table>
34
+ <% end %>
35
+ <% end %>
36
+ <h3>Clients</h3>
9
37
  <table class="table">
10
38
  <thead>
11
39
  <tr>
12
40
  <th>Id</th>
13
41
  <th>Meta</th>
14
- <th>Executor?</th>
15
- <th>Execution items</th>
16
- <th>Free/Total workers</th>
17
42
  <th></th>
18
43
  </tr>
19
44
  </thead>
20
-
21
- <% @worlds.each do |world| %>
45
+ <% @clients.each do |world| %>
22
46
  <tr>
23
47
  <td><%= h(world.id) %></td>
24
48
  <td><%= h(world.meta) %></td>
25
- <td><%= "true" if world.is_a? Dynflow::Coordinator::ExecutorWorld %></td>
26
- <td><%= h(world.data['execution_status'] || 'N/A') %></td>
27
- <td><%= world.data.key?('free_workers') ? "#{world.data['free_workers']}/#{world.data[:pool_size]}" : 'N/A' %></td>
28
49
  <td>
29
- <% validation_result = @validation_results[world.id] if @validation_results %>
30
-
31
- <% unless validation_result == :invalidated %>
32
- <a href="<%= url("/worlds/#{world.id}/check") %>" class="postlink">check</a>
33
- <% end %>
34
-
35
- <% if validation_result %>
36
- <% if validation_result == :invalid %>
37
- <a href="<%= url("/worlds/#{world.id}/check?invalidate=true") %>" class="postlink">invalidate</a>
38
- <% end %>
39
-
40
- <span class="label label-<%= validation_result_css_class(validation_result) %>">
41
- <%= h(validation_result) %>
42
- </span>
43
- <% end %>
50
+ <%= erb :world_validation_result, locals: { world: world } %>
44
51
  </td>
45
52
  </tr>
46
53
  <% end %>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dynflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.37
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ivan Necas
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-03-12 00:00:00.000000000 Z
12
+ date: 2018-03-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: multi_json
@@ -482,6 +482,7 @@ files:
482
482
  - lib/dynflow/persistence_adapters/sequel_migrations/013_add_action_columns.rb
483
483
  - lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb
484
484
  - lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb
485
+ - lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb
485
486
  - lib/dynflow/rails.rb
486
487
  - lib/dynflow/rails/configuration.rb
487
488
  - lib/dynflow/rails/daemon.rb
@@ -579,6 +580,7 @@ files:
579
580
  - web/views/layout.erb
580
581
  - web/views/plan_step.erb
581
582
  - web/views/show.erb
583
+ - web/views/world_validation_result.erb
582
584
  - web/views/worlds.erb
583
585
  homepage: http://github.com/Dynflow/dynflow
584
586
  licenses: