dynflow 0.8.1 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +8 -8
  2. data/doc/pages/source/documentation/index.md +47 -12
  3. data/dynflow.gemspec +2 -2
  4. data/examples/future_execution.rb +73 -0
  5. data/lib/dynflow.rb +4 -1
  6. data/lib/dynflow/action.rb +15 -0
  7. data/lib/dynflow/config.rb +15 -1
  8. data/lib/dynflow/coordinator.rb +7 -0
  9. data/lib/dynflow/execution_plan.rb +15 -3
  10. data/lib/dynflow/execution_plan/steps/plan_step.rb +5 -1
  11. data/lib/dynflow/middleware.rb +4 -0
  12. data/lib/dynflow/middleware/stack.rb +1 -1
  13. data/lib/dynflow/middleware/world.rb +1 -1
  14. data/lib/dynflow/persistence.rb +19 -0
  15. data/lib/dynflow/persistence_adapters/abstract.rb +16 -0
  16. data/lib/dynflow/persistence_adapters/sequel.rb +31 -3
  17. data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb +17 -0
  18. data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb +13 -0
  19. data/lib/dynflow/scheduled_plan.rb +65 -0
  20. data/lib/dynflow/schedulers.rb +9 -0
  21. data/lib/dynflow/schedulers/abstract.rb +37 -0
  22. data/lib/dynflow/schedulers/abstract_core.rb +65 -0
  23. data/lib/dynflow/schedulers/polling.rb +32 -0
  24. data/lib/dynflow/serializers.rb +8 -0
  25. data/lib/dynflow/serializers/abstract.rb +15 -0
  26. data/lib/dynflow/serializers/noop.rb +15 -0
  27. data/lib/dynflow/version.rb +1 -1
  28. data/lib/dynflow/web/console.rb +8 -23
  29. data/lib/dynflow/web/console_helpers.rb +10 -0
  30. data/lib/dynflow/world.rb +99 -24
  31. data/test/abnormal_states_recovery_test.rb +64 -0
  32. data/test/future_execution_test.rb +114 -0
  33. data/test/middleware_test.rb +8 -2
  34. data/test/support/middleware_example.rb +11 -0
  35. data/test/test_helper.rb +1 -0
  36. data/web/views/show.erb +11 -0
  37. data/web/views/worlds.erb +19 -3
  38. metadata +19 -6
data/lib/dynflow/world.rb CHANGED
@@ -7,31 +7,37 @@ module Dynflow
7
7
  attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
8
8
  :transaction_adapter, :logger_adapter, :coordinator,
9
9
  :persistence, :action_classes, :subscription_index,
10
- :middleware, :auto_rescue, :clock, :meta
10
+ :middleware, :auto_rescue, :clock, :meta, :scheduler, :auto_validity_check, :validity_check_timeout
11
11
 
12
12
  def initialize(config)
13
- @id = SecureRandom.uuid
14
- @clock = spawn_and_wait(Clock, 'clock')
15
- config_for_world = Config::ForWorld.new(config, self)
13
+ @id = SecureRandom.uuid
14
+ @clock = spawn_and_wait(Clock, 'clock')
15
+ config_for_world = Config::ForWorld.new(config, self)
16
16
  config_for_world.validate
17
- @logger_adapter = config_for_world.logger_adapter
18
- @transaction_adapter = config_for_world.transaction_adapter
19
- @persistence = Persistence.new(self, config_for_world.persistence_adapter)
20
- @coordinator = Coordinator.new(config_for_world.coordinator_adapter)
21
- @executor = config_for_world.executor
22
- @action_classes = config_for_world.action_classes
23
- @auto_rescue = config_for_world.auto_rescue
24
- @exit_on_terminate = config_for_world.exit_on_terminate
25
- @connector = config_for_world.connector
26
- @middleware = Middleware::World.new
27
- @client_dispatcher = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self)
28
- @meta = config_for_world.meta
17
+ @logger_adapter = config_for_world.logger_adapter
18
+ @transaction_adapter = config_for_world.transaction_adapter
19
+ @persistence = Persistence.new(self, config_for_world.persistence_adapter)
20
+ @coordinator = Coordinator.new(config_for_world.coordinator_adapter)
21
+ @executor = config_for_world.executor
22
+ @action_classes = config_for_world.action_classes
23
+ @auto_rescue = config_for_world.auto_rescue
24
+ @exit_on_terminate = config_for_world.exit_on_terminate
25
+ @connector = config_for_world.connector
26
+ @middleware = Middleware::World.new
27
+ @client_dispatcher = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self)
28
+ @meta = config_for_world.meta
29
+ @auto_validity_check = config_for_world.auto_validity_check
30
+ @validity_check_timeout = config_for_world.validity_check_timeout
29
31
  calculate_subscription_index
30
32
 
31
33
  if executor
32
34
  @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self)
33
35
  executor.initialized.wait
34
36
  end
37
+ self.worlds_validity_check if auto_validity_check
38
+ @scheduler = try_spawn_scheduler(config_for_world)
39
+ @meta = config_for_world.meta
40
+ @meta['scheduler'] = true if @scheduler
35
41
  coordinator.register_world(registered_world)
36
42
  @termination_barrier = Mutex.new
37
43
  @before_termination_hooks = Queue.new
@@ -43,6 +49,7 @@ module Dynflow
43
49
  end
44
50
  end
45
51
  self.auto_execute if config_for_world.auto_execute
52
+ @scheduler.start if @scheduler
46
53
  end
47
54
 
48
55
  def before_termination(&block)
@@ -90,16 +97,22 @@ module Dynflow
90
97
  # ExecutionPlan is executed.
91
98
  Triggered = type { fields! execution_plan_id: String, future: Concurrent::Edge::Future }
92
99
 
93
- variants PlaningFailed, Triggered
100
+ Scheduled = type { fields! execution_plan_id: String }
101
+
102
+ variants PlaningFailed, Triggered, Scheduled
94
103
  end
95
104
 
96
105
  module TriggerResult
97
106
  def planned?
98
- match self, PlaningFailed => false, Triggered => true
107
+ match self, PlaningFailed => false, Triggered => true, Scheduled => false
99
108
  end
100
109
 
101
110
  def triggered?
102
- match self, PlaningFailed => false, Triggered => true
111
+ match self, PlaningFailed => false, Triggered => true, Scheduled => false
112
+ end
113
+
114
+ def scheduled?
115
+ match self, PlaningFailed => false, Triggered => false, Scheduled => true
103
116
  end
104
117
 
105
118
  def id
@@ -133,6 +146,20 @@ module Dynflow
133
146
  end
134
147
  end
135
148
 
149
+ def schedule(action_class, schedule_options, *args)
150
+ raise 'No action_class given' if action_class.nil?
151
+ execution_plan = ExecutionPlan.new self
152
+ execution_plan.schedule(action_class, {}, schedule_options, *args)
153
+ scheduled_plan = ScheduledPlan.new(self,
154
+ execution_plan.id,
155
+ schedule_options[:start_at],
156
+ schedule_options.fetch(:start_before, nil),
157
+ args,
158
+ execution_plan.entry_action.serializer)
159
+ persistence.save_scheduled_plan(scheduled_plan)
160
+ Scheduled[execution_plan.id]
161
+ end
162
+
136
163
  def plan(action_class, *args)
137
164
  ExecutionPlan.new(self).tap do |execution_plan|
138
165
  execution_plan.prepare(action_class)
@@ -179,6 +206,10 @@ module Dynflow
179
206
  begin
180
207
  run_before_termination_hooks
181
208
 
209
+ if scheduler
210
+ logger.info "start terminating scheduler..."
211
+ scheduler.terminate.wait
212
+ end
182
213
 
183
214
  if executor
184
215
  connector.stop_receiving_new_work(self)
@@ -229,13 +260,15 @@ module Dynflow
229
260
  def invalidate(world)
230
261
  Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
231
262
  coordinator.acquire(Coordinator::WorldInvalidationLock.new(self, world)) do
232
- old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
233
- owner_id: "world:#{world.id}")
263
+ if world.is_a? Coordinator::ExecutorWorld
264
+ old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
265
+ owner_id: "world:#{world.id}")
234
266
 
235
- coordinator.deactivate_world(world)
267
+ coordinator.deactivate_world(world)
236
268
 
237
- old_execution_locks.each do |execution_lock|
238
- invalidate_execution_lock(execution_lock)
269
+ old_execution_locks.each do |execution_lock|
270
+ invalidate_execution_lock(execution_lock)
271
+ end
239
272
  end
240
273
 
241
274
  coordinator.delete_world(world)
@@ -267,6 +300,40 @@ module Dynflow
267
300
  logger.error "failed to write data while invalidating execution lock #{execution_lock}"
268
301
  end
269
302
 
303
+ def worlds_validity_check(auto_invalidate = true, worlds_filter = {})
304
+ worlds = coordinator.find_worlds(false, worlds_filter)
305
+
306
+ world_checks = worlds.reduce({}) do |hash, world|
307
+ hash.update(world => ping(world.id, self.validity_check_timeout))
308
+ end
309
+ world_checks.values.each(&:wait)
310
+
311
+ results = {}
312
+ world_checks.each do |world, check|
313
+ if check.success?
314
+ result = :valid
315
+ else
316
+ if auto_invalidate
317
+ begin
318
+ invalidate(world)
319
+ result = :invalidated
320
+ rescue => e
321
+ result = e.message
322
+ end
323
+ else
324
+ result = :invalid
325
+ end
326
+ end
327
+ results[world.id] = result
328
+ end
329
+
330
+ unless results.values.all? { |result| result == :valid }
331
+ logger.error "invalid worlds found #{results.inspect}"
332
+ end
333
+
334
+ return results
335
+ end
336
+
270
337
  # executes plans that are planned/paused and haven't reported any error yet (usually when no executor
271
338
  # was available by the time of planning or terminating)
272
339
  def auto_execute
@@ -277,6 +344,14 @@ module Dynflow
277
344
  end
278
345
  end
279
346
 
347
+ def try_spawn_scheduler(config_for_world)
348
+ return nil if !executor || config_for_world.scheduler.nil?
349
+ coordinator.acquire(Coordinator::SchedulerLock.new(self))
350
+ config_for_world.scheduler
351
+ rescue Coordinator::LockError => e
352
+ nil
353
+ end
354
+
280
355
  private
281
356
 
282
357
  def calculate_subscription_index
@@ -146,6 +146,70 @@ module Dynflow
146
146
  plan.execution_history.map { |h| [h.name, h.world_id] }.must_equal(expected_history)
147
147
  end
148
148
  end
149
+
150
+ describe '#worlds_validity_check' do
151
+ describe 'the auto_validity_check is enabled' do
152
+ let :invalid_world do
153
+ Coordinator::ClientWorld.new(OpenStruct.new(id: '123', meta: {}))
154
+ end
155
+
156
+ let :invalid_world_2 do
157
+ Coordinator::ClientWorld.new(OpenStruct.new(id: '456', meta: {}))
158
+ end
159
+
160
+ let :client_world do
161
+ create_world(false)
162
+ end
163
+
164
+ let :world_with_auto_validity_check do
165
+ create_world do |config|
166
+ config.auto_validity_check = true
167
+ config.validity_check_timeout = 0.2
168
+ end
169
+ end
170
+
171
+ it 'performs the validity check on world creation if auto_validity_check enabled' do
172
+ client_world.coordinator.register_world(invalid_world)
173
+ client_world.coordinator.find_worlds(false, id: invalid_world.id).wont_be_empty
174
+ world_with_auto_validity_check
175
+ client_world.coordinator.find_worlds(false, id: invalid_world.id).must_be_empty
176
+ end
177
+
178
+ it 'by default, the auto_validity_check is enabled only for executor words' do
179
+ create_world(false).auto_validity_check.must_equal false
180
+ create_world(true).auto_validity_check.must_equal true
181
+ end
182
+
183
+ it 'reports the validation status' do
184
+ client_world.coordinator.register_world(invalid_world)
185
+ results = client_world.worlds_validity_check
186
+ client_world.coordinator.find_worlds(false, id: invalid_world.id).must_be_empty
187
+
188
+ results[invalid_world.id].must_equal :invalidated
189
+
190
+ results[client_world.id].must_equal :valid
191
+ end
192
+
193
+ it 'allows checking only, without actual invalidation' do
194
+ client_world.coordinator.register_world(invalid_world)
195
+ results = client_world.worlds_validity_check(false)
196
+ client_world.coordinator.find_worlds(false, id: invalid_world.id).wont_be_empty
197
+
198
+ results[invalid_world.id].must_equal :invalid
199
+ end
200
+
201
+ it 'allows to filter the worlds to run the check on' do
202
+ client_world.coordinator.register_world(invalid_world)
203
+ client_world.coordinator.register_world(invalid_world_2)
204
+ client_world.coordinator.find_worlds(false, id: [invalid_world.id, invalid_world_2.id]).size.must_equal 2
205
+
206
+ results = client_world.worlds_validity_check(true, :id => invalid_world.id)
207
+ results.must_equal(invalid_world.id => :invalidated)
208
+ client_world.coordinator.find_worlds(false, id: [invalid_world.id, invalid_world_2.id]).size.must_equal 1
209
+ end
210
+ end
211
+ end
212
+
149
213
  end
150
214
  end
151
215
  end
@@ -0,0 +1,114 @@
1
+ require_relative 'test_helper'
2
+
3
+ module Dynflow
4
+ module FutureExecutionTest
5
+ describe 'Future Execution' do
6
+ include PlanAssertions
7
+ include Dynflow::Testing::Assertions
8
+ include Dynflow::Testing::Factories
9
+
10
+ describe 'action scheduling' do
11
+
12
+ before do
13
+ world.persistence.delete_scheduled_plans(:execution_plan_uuid => [])
14
+ end
15
+
16
+ let(:world) { WorldFactory.create_world }
17
+ let(:plan) do
18
+ scheduled = world.schedule(::Support::DummyExample::Dummy, { :start_at => @start_at })
19
+ scheduled.must_be :scheduled?
20
+ world.persistence.load_scheduled_plan(scheduled.execution_plan_id)
21
+ end
22
+ let(:history_names) do
23
+ ->(execution_plan) { execution_plan.execution_history.map(&:name) }
24
+ end
25
+ let(:execution_plan) { plan.execution_plan }
26
+
27
+ it 'schedules the action' do
28
+ @start_at = Time.now.utc + 180
29
+ execution_plan.steps.count.must_equal 1
30
+ plan.start_at.inspect.must_equal (@start_at).inspect
31
+ history_names.call(execution_plan).must_equal ['schedule']
32
+ end
33
+
34
+ it 'finds scheduled plans' do
35
+ @start_at = Time.now.utc - 100
36
+ plan
37
+ past_scheduled_plans = world.persistence.find_past_scheduled_plans(@start_at + 10)
38
+ past_scheduled_plans.length.must_equal 1
39
+ past_scheduled_plans.first.execution_plan_uuid.must_equal execution_plan.id
40
+ end
41
+
42
+ it 'scheduled plans can be planned and executed' do
43
+ @start_at = Time.now.utc + 180
44
+ execution_plan.state.must_equal :scheduled
45
+ plan.plan
46
+ execution_plan.state.must_equal :planned
47
+ execution_plan.result.must_equal :pending
48
+ assert_planning_success execution_plan
49
+ history_names.call(execution_plan).must_equal ['schedule']
50
+ executed = plan.execute
51
+ executed.wait
52
+ executed.value.state.must_equal :stopped
53
+ executed.value.result.must_equal :success
54
+ executed.value.execution_history.count.must_equal 3
55
+ end
56
+
57
+ it 'expired plans can be failed' do
58
+ @start_at = Time.now.utc + 180
59
+ plan.timeout
60
+ execution_plan.state.must_equal :stopped
61
+ execution_plan.result.must_equal :error
62
+ execution_plan.errors.first.message.must_match /could not be started before set time/
63
+ history_names.call(execution_plan).must_equal %W(schedule timeout)
64
+ end
65
+
66
+ end
67
+
68
+ describe 'polling scheduler' do
69
+ let(:dummy_world) { Dynflow::Testing::DummyWorld.new }
70
+ let(:persistence) { MiniTest::Mock.new }
71
+ let(:options) { { :poll_interval => 15, :time_source => -> { dummy_world.clock.current_time } } }
72
+ let(:scheduler) { Schedulers::Polling.new(dummy_world, options) }
73
+ let(:klok) { dummy_world.clock }
74
+
75
+ it 'checks for scheduled plans in regular intervals' do
76
+ start_time = klok.current_time
77
+ persistence.expect(:find_past_scheduled_plans, [], [start_time])
78
+ persistence.expect(:find_past_scheduled_plans, [], [start_time + options[:poll_interval]])
79
+ dummy_world.stub :persistence, persistence do
80
+ klok.pending_pings.length.must_equal 0
81
+ scheduler.start.wait
82
+ klok.pending_pings.length.must_equal 1
83
+ klok.pending_pings.first.who.ref.must_be_same_as scheduler.core
84
+ klok.pending_pings.first.when.must_equal start_time + options[:poll_interval]
85
+ klok.progress
86
+ scheduler.terminate.wait
87
+ klok.pending_pings.length.must_equal 1
88
+ klok.pending_pings.first.who.ref.must_be_same_as scheduler.core
89
+ klok.pending_pings.first.when.must_equal start_time + 2 * options[:poll_interval]
90
+ klok.progress
91
+ klok.pending_pings.length.must_equal 0
92
+ end
93
+ end
94
+ end
95
+
96
+ describe 'serializers' do
97
+ let(:save_and_load) do
98
+ ->(thing) { MultiJson.load(MultiJson.dump(thing)) }
99
+ end
100
+ let(:simulated_use) do
101
+ lambda do |serializer_class, input|
102
+ serializer = serializer_class.new
103
+ serializer.deserialize(save_and_load.call(serializer.serialize *input))
104
+ end
105
+ end
106
+
107
+ it 'noop serializer [de]serializes correctly for simple types' do
108
+ input = [1, 2.0, 'three', ['four-1', 'four-2'], { 'five' => 5 }]
109
+ simulated_use.call(Dynflow::Serializers::Noop, input).must_equal input
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
@@ -12,8 +12,14 @@ module Dynflow
12
12
  end
13
13
 
14
14
  it "wraps the action method calls" do
15
- world.trigger(Support::MiddlewareExample::LoggingAction, {}).finished.wait
16
- log.must_equal %w[LogMiddleware::before_plan_phase
15
+ schedule = world.schedule(Support::MiddlewareExample::LoggingAction, { :start_at => Time.now.utc - 60 }, {})
16
+ plan = world.persistence.load_scheduled_plan schedule.execution_plan_id
17
+ plan.plan
18
+ plan.execute.wait
19
+ log.must_equal %w[LogMiddleware::before_schedule
20
+ schedule
21
+ LogMiddleware::after_schedule
22
+ LogMiddleware::before_plan_phase
17
23
  LogMiddleware::before_plan
18
24
  plan
19
25
  LogMiddleware::after_plan
@@ -14,6 +14,12 @@ module Support
14
14
  LogMiddleware.log << "#{self.class.name[/\w+$/]}::#{message}"
15
15
  end
16
16
 
17
+ def schedule(*args)
18
+ log 'before_schedule'
19
+ pass *args
20
+ log 'after_schedule'
21
+ end
22
+
17
23
  def plan(args)
18
24
  log 'before_plan'
19
25
  pass(args)
@@ -72,6 +78,11 @@ module Support
72
78
  LogMiddleware.log << message
73
79
  end
74
80
 
81
+ def schedule(schedule_options, *args)
82
+ log 'schedule'
83
+ Dynflow::Serializers::Noop.new
84
+ end
85
+
75
86
  def plan(input)
76
87
  log 'plan'
77
88
  plan_self(input)
data/test/test_helper.rb CHANGED
@@ -88,6 +88,7 @@ module WorldFactory
88
88
  config.persistence_adapter = persistence_adapter
89
89
  config.logger_adapter = logger_adapter
90
90
  config.coordinator_adapter = coordinator_adapter
91
+ config.scheduler = nil
91
92
  config.auto_rescue = false
92
93
  config.exit_on_terminate = false
93
94
  config.auto_execute = false
data/web/views/show.erb CHANGED
@@ -11,6 +11,17 @@
11
11
  <%= h(@plan.result) %>
12
12
  </p>
13
13
 
14
+ <% if @plan.state == :scheduled %>
15
+ <p>
16
+ <b>Start at:</b>
17
+ <%= h(@plan.schedule_record.start_at) %>
18
+ </p>
19
+ <p>
20
+ <b>Start before:</b>
21
+ <%= h(@plan.schedule_record.start_before || "-") %>
22
+ </p>
23
+ <% end %>
24
+
14
25
  <p>
15
26
  <b>Started at:</b>
16
27
  <%= h(@plan.started_at) %>