dynflow 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/doc/pages/source/documentation/index.md +47 -12
- data/dynflow.gemspec +2 -2
- data/examples/future_execution.rb +73 -0
- data/lib/dynflow.rb +4 -1
- data/lib/dynflow/action.rb +15 -0
- data/lib/dynflow/config.rb +15 -1
- data/lib/dynflow/coordinator.rb +7 -0
- data/lib/dynflow/execution_plan.rb +15 -3
- data/lib/dynflow/execution_plan/steps/plan_step.rb +5 -1
- data/lib/dynflow/middleware.rb +4 -0
- data/lib/dynflow/middleware/stack.rb +1 -1
- data/lib/dynflow/middleware/world.rb +1 -1
- data/lib/dynflow/persistence.rb +19 -0
- data/lib/dynflow/persistence_adapters/abstract.rb +16 -0
- data/lib/dynflow/persistence_adapters/sequel.rb +31 -3
- data/lib/dynflow/persistence_adapters/sequel_migrations/006_fix_data_length.rb +17 -0
- data/lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb +13 -0
- data/lib/dynflow/scheduled_plan.rb +65 -0
- data/lib/dynflow/schedulers.rb +9 -0
- data/lib/dynflow/schedulers/abstract.rb +37 -0
- data/lib/dynflow/schedulers/abstract_core.rb +65 -0
- data/lib/dynflow/schedulers/polling.rb +32 -0
- data/lib/dynflow/serializers.rb +8 -0
- data/lib/dynflow/serializers/abstract.rb +15 -0
- data/lib/dynflow/serializers/noop.rb +15 -0
- data/lib/dynflow/version.rb +1 -1
- data/lib/dynflow/web/console.rb +8 -23
- data/lib/dynflow/web/console_helpers.rb +10 -0
- data/lib/dynflow/world.rb +99 -24
- data/test/abnormal_states_recovery_test.rb +64 -0
- data/test/future_execution_test.rb +114 -0
- data/test/middleware_test.rb +8 -2
- data/test/support/middleware_example.rb +11 -0
- data/test/test_helper.rb +1 -0
- data/web/views/show.erb +11 -0
- data/web/views/worlds.erb +19 -3
- metadata +19 -6
data/lib/dynflow/world.rb
CHANGED
@@ -7,31 +7,37 @@ module Dynflow
|
|
7
7
|
attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
|
8
8
|
:transaction_adapter, :logger_adapter, :coordinator,
|
9
9
|
:persistence, :action_classes, :subscription_index,
|
10
|
-
:middleware, :auto_rescue, :clock, :meta
|
10
|
+
:middleware, :auto_rescue, :clock, :meta, :scheduler, :auto_validity_check, :validity_check_timeout
|
11
11
|
|
12
12
|
def initialize(config)
|
13
|
-
@id
|
14
|
-
@clock
|
15
|
-
config_for_world
|
13
|
+
@id = SecureRandom.uuid
|
14
|
+
@clock = spawn_and_wait(Clock, 'clock')
|
15
|
+
config_for_world = Config::ForWorld.new(config, self)
|
16
16
|
config_for_world.validate
|
17
|
-
@logger_adapter
|
18
|
-
@transaction_adapter
|
19
|
-
@persistence
|
20
|
-
@coordinator
|
21
|
-
@executor
|
22
|
-
@action_classes
|
23
|
-
@auto_rescue
|
24
|
-
@exit_on_terminate
|
25
|
-
@connector
|
26
|
-
@middleware
|
27
|
-
@client_dispatcher
|
28
|
-
@meta
|
17
|
+
@logger_adapter = config_for_world.logger_adapter
|
18
|
+
@transaction_adapter = config_for_world.transaction_adapter
|
19
|
+
@persistence = Persistence.new(self, config_for_world.persistence_adapter)
|
20
|
+
@coordinator = Coordinator.new(config_for_world.coordinator_adapter)
|
21
|
+
@executor = config_for_world.executor
|
22
|
+
@action_classes = config_for_world.action_classes
|
23
|
+
@auto_rescue = config_for_world.auto_rescue
|
24
|
+
@exit_on_terminate = config_for_world.exit_on_terminate
|
25
|
+
@connector = config_for_world.connector
|
26
|
+
@middleware = Middleware::World.new
|
27
|
+
@client_dispatcher = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self)
|
28
|
+
@meta = config_for_world.meta
|
29
|
+
@auto_validity_check = config_for_world.auto_validity_check
|
30
|
+
@validity_check_timeout = config_for_world.validity_check_timeout
|
29
31
|
calculate_subscription_index
|
30
32
|
|
31
33
|
if executor
|
32
34
|
@executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self)
|
33
35
|
executor.initialized.wait
|
34
36
|
end
|
37
|
+
self.worlds_validity_check if auto_validity_check
|
38
|
+
@scheduler = try_spawn_scheduler(config_for_world)
|
39
|
+
@meta = config_for_world.meta
|
40
|
+
@meta['scheduler'] = true if @scheduler
|
35
41
|
coordinator.register_world(registered_world)
|
36
42
|
@termination_barrier = Mutex.new
|
37
43
|
@before_termination_hooks = Queue.new
|
@@ -43,6 +49,7 @@ module Dynflow
|
|
43
49
|
end
|
44
50
|
end
|
45
51
|
self.auto_execute if config_for_world.auto_execute
|
52
|
+
@scheduler.start if @scheduler
|
46
53
|
end
|
47
54
|
|
48
55
|
def before_termination(&block)
|
@@ -90,16 +97,22 @@ module Dynflow
|
|
90
97
|
# ExecutionPlan is executed.
|
91
98
|
Triggered = type { fields! execution_plan_id: String, future: Concurrent::Edge::Future }
|
92
99
|
|
93
|
-
|
100
|
+
Scheduled = type { fields! execution_plan_id: String }
|
101
|
+
|
102
|
+
variants PlaningFailed, Triggered, Scheduled
|
94
103
|
end
|
95
104
|
|
96
105
|
module TriggerResult
|
97
106
|
def planned?
|
98
|
-
match self, PlaningFailed => false, Triggered => true
|
107
|
+
match self, PlaningFailed => false, Triggered => true, Scheduled => false
|
99
108
|
end
|
100
109
|
|
101
110
|
def triggered?
|
102
|
-
match self, PlaningFailed => false, Triggered => true
|
111
|
+
match self, PlaningFailed => false, Triggered => true, Scheduled => false
|
112
|
+
end
|
113
|
+
|
114
|
+
def scheduled?
|
115
|
+
match self, PlaningFailed => false, Triggered => false, Scheduled => true
|
103
116
|
end
|
104
117
|
|
105
118
|
def id
|
@@ -133,6 +146,20 @@ module Dynflow
|
|
133
146
|
end
|
134
147
|
end
|
135
148
|
|
149
|
+
def schedule(action_class, schedule_options, *args)
|
150
|
+
raise 'No action_class given' if action_class.nil?
|
151
|
+
execution_plan = ExecutionPlan.new self
|
152
|
+
execution_plan.schedule(action_class, {}, schedule_options, *args)
|
153
|
+
scheduled_plan = ScheduledPlan.new(self,
|
154
|
+
execution_plan.id,
|
155
|
+
schedule_options[:start_at],
|
156
|
+
schedule_options.fetch(:start_before, nil),
|
157
|
+
args,
|
158
|
+
execution_plan.entry_action.serializer)
|
159
|
+
persistence.save_scheduled_plan(scheduled_plan)
|
160
|
+
Scheduled[execution_plan.id]
|
161
|
+
end
|
162
|
+
|
136
163
|
def plan(action_class, *args)
|
137
164
|
ExecutionPlan.new(self).tap do |execution_plan|
|
138
165
|
execution_plan.prepare(action_class)
|
@@ -179,6 +206,10 @@ module Dynflow
|
|
179
206
|
begin
|
180
207
|
run_before_termination_hooks
|
181
208
|
|
209
|
+
if scheduler
|
210
|
+
logger.info "start terminating scheduler..."
|
211
|
+
scheduler.terminate.wait
|
212
|
+
end
|
182
213
|
|
183
214
|
if executor
|
184
215
|
connector.stop_receiving_new_work(self)
|
@@ -229,13 +260,15 @@ module Dynflow
|
|
229
260
|
def invalidate(world)
|
230
261
|
Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
|
231
262
|
coordinator.acquire(Coordinator::WorldInvalidationLock.new(self, world)) do
|
232
|
-
|
233
|
-
|
263
|
+
if world.is_a? Coordinator::ExecutorWorld
|
264
|
+
old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
|
265
|
+
owner_id: "world:#{world.id}")
|
234
266
|
|
235
|
-
|
267
|
+
coordinator.deactivate_world(world)
|
236
268
|
|
237
|
-
|
238
|
-
|
269
|
+
old_execution_locks.each do |execution_lock|
|
270
|
+
invalidate_execution_lock(execution_lock)
|
271
|
+
end
|
239
272
|
end
|
240
273
|
|
241
274
|
coordinator.delete_world(world)
|
@@ -267,6 +300,40 @@ module Dynflow
|
|
267
300
|
logger.error "failed to write data while invalidating execution lock #{execution_lock}"
|
268
301
|
end
|
269
302
|
|
303
|
+
def worlds_validity_check(auto_invalidate = true, worlds_filter = {})
|
304
|
+
worlds = coordinator.find_worlds(false, worlds_filter)
|
305
|
+
|
306
|
+
world_checks = worlds.reduce({}) do |hash, world|
|
307
|
+
hash.update(world => ping(world.id, self.validity_check_timeout))
|
308
|
+
end
|
309
|
+
world_checks.values.each(&:wait)
|
310
|
+
|
311
|
+
results = {}
|
312
|
+
world_checks.each do |world, check|
|
313
|
+
if check.success?
|
314
|
+
result = :valid
|
315
|
+
else
|
316
|
+
if auto_invalidate
|
317
|
+
begin
|
318
|
+
invalidate(world)
|
319
|
+
result = :invalidated
|
320
|
+
rescue => e
|
321
|
+
result = e.message
|
322
|
+
end
|
323
|
+
else
|
324
|
+
result = :invalid
|
325
|
+
end
|
326
|
+
end
|
327
|
+
results[world.id] = result
|
328
|
+
end
|
329
|
+
|
330
|
+
unless results.values.all? { |result| result == :valid }
|
331
|
+
logger.error "invalid worlds found #{results.inspect}"
|
332
|
+
end
|
333
|
+
|
334
|
+
return results
|
335
|
+
end
|
336
|
+
|
270
337
|
# executes plans that are planned/paused and haven't reported any error yet (usually when no executor
|
271
338
|
# was available by the time of planning or terminating)
|
272
339
|
def auto_execute
|
@@ -277,6 +344,14 @@ module Dynflow
|
|
277
344
|
end
|
278
345
|
end
|
279
346
|
|
347
|
+
def try_spawn_scheduler(config_for_world)
|
348
|
+
return nil if !executor || config_for_world.scheduler.nil?
|
349
|
+
coordinator.acquire(Coordinator::SchedulerLock.new(self))
|
350
|
+
config_for_world.scheduler
|
351
|
+
rescue Coordinator::LockError => e
|
352
|
+
nil
|
353
|
+
end
|
354
|
+
|
280
355
|
private
|
281
356
|
|
282
357
|
def calculate_subscription_index
|
@@ -146,6 +146,70 @@ module Dynflow
|
|
146
146
|
plan.execution_history.map { |h| [h.name, h.world_id] }.must_equal(expected_history)
|
147
147
|
end
|
148
148
|
end
|
149
|
+
|
150
|
+
describe '#worlds_validity_check' do
|
151
|
+
describe 'the auto_validity_check is enabled' do
|
152
|
+
let :invalid_world do
|
153
|
+
Coordinator::ClientWorld.new(OpenStruct.new(id: '123', meta: {}))
|
154
|
+
end
|
155
|
+
|
156
|
+
let :invalid_world_2 do
|
157
|
+
Coordinator::ClientWorld.new(OpenStruct.new(id: '456', meta: {}))
|
158
|
+
end
|
159
|
+
|
160
|
+
let :client_world do
|
161
|
+
create_world(false)
|
162
|
+
end
|
163
|
+
|
164
|
+
let :world_with_auto_validity_check do
|
165
|
+
create_world do |config|
|
166
|
+
config.auto_validity_check = true
|
167
|
+
config.validity_check_timeout = 0.2
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
it 'performs the validity check on world creation if auto_validity_check enabled' do
|
172
|
+
client_world.coordinator.register_world(invalid_world)
|
173
|
+
client_world.coordinator.find_worlds(false, id: invalid_world.id).wont_be_empty
|
174
|
+
world_with_auto_validity_check
|
175
|
+
client_world.coordinator.find_worlds(false, id: invalid_world.id).must_be_empty
|
176
|
+
end
|
177
|
+
|
178
|
+
it 'by default, the auto_validity_check is enabled only for executor words' do
|
179
|
+
create_world(false).auto_validity_check.must_equal false
|
180
|
+
create_world(true).auto_validity_check.must_equal true
|
181
|
+
end
|
182
|
+
|
183
|
+
it 'reports the validation status' do
|
184
|
+
client_world.coordinator.register_world(invalid_world)
|
185
|
+
results = client_world.worlds_validity_check
|
186
|
+
client_world.coordinator.find_worlds(false, id: invalid_world.id).must_be_empty
|
187
|
+
|
188
|
+
results[invalid_world.id].must_equal :invalidated
|
189
|
+
|
190
|
+
results[client_world.id].must_equal :valid
|
191
|
+
end
|
192
|
+
|
193
|
+
it 'allows checking only, without actual invalidation' do
|
194
|
+
client_world.coordinator.register_world(invalid_world)
|
195
|
+
results = client_world.worlds_validity_check(false)
|
196
|
+
client_world.coordinator.find_worlds(false, id: invalid_world.id).wont_be_empty
|
197
|
+
|
198
|
+
results[invalid_world.id].must_equal :invalid
|
199
|
+
end
|
200
|
+
|
201
|
+
it 'allows to filter the worlds to run the check on' do
|
202
|
+
client_world.coordinator.register_world(invalid_world)
|
203
|
+
client_world.coordinator.register_world(invalid_world_2)
|
204
|
+
client_world.coordinator.find_worlds(false, id: [invalid_world.id, invalid_world_2.id]).size.must_equal 2
|
205
|
+
|
206
|
+
results = client_world.worlds_validity_check(true, :id => invalid_world.id)
|
207
|
+
results.must_equal(invalid_world.id => :invalidated)
|
208
|
+
client_world.coordinator.find_worlds(false, id: [invalid_world.id, invalid_world_2.id]).size.must_equal 1
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
149
213
|
end
|
150
214
|
end
|
151
215
|
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
module Dynflow
|
4
|
+
module FutureExecutionTest
|
5
|
+
describe 'Future Execution' do
|
6
|
+
include PlanAssertions
|
7
|
+
include Dynflow::Testing::Assertions
|
8
|
+
include Dynflow::Testing::Factories
|
9
|
+
|
10
|
+
describe 'action scheduling' do
|
11
|
+
|
12
|
+
before do
|
13
|
+
world.persistence.delete_scheduled_plans(:execution_plan_uuid => [])
|
14
|
+
end
|
15
|
+
|
16
|
+
let(:world) { WorldFactory.create_world }
|
17
|
+
let(:plan) do
|
18
|
+
scheduled = world.schedule(::Support::DummyExample::Dummy, { :start_at => @start_at })
|
19
|
+
scheduled.must_be :scheduled?
|
20
|
+
world.persistence.load_scheduled_plan(scheduled.execution_plan_id)
|
21
|
+
end
|
22
|
+
let(:history_names) do
|
23
|
+
->(execution_plan) { execution_plan.execution_history.map(&:name) }
|
24
|
+
end
|
25
|
+
let(:execution_plan) { plan.execution_plan }
|
26
|
+
|
27
|
+
it 'schedules the action' do
|
28
|
+
@start_at = Time.now.utc + 180
|
29
|
+
execution_plan.steps.count.must_equal 1
|
30
|
+
plan.start_at.inspect.must_equal (@start_at).inspect
|
31
|
+
history_names.call(execution_plan).must_equal ['schedule']
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'finds scheduled plans' do
|
35
|
+
@start_at = Time.now.utc - 100
|
36
|
+
plan
|
37
|
+
past_scheduled_plans = world.persistence.find_past_scheduled_plans(@start_at + 10)
|
38
|
+
past_scheduled_plans.length.must_equal 1
|
39
|
+
past_scheduled_plans.first.execution_plan_uuid.must_equal execution_plan.id
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'scheduled plans can be planned and executed' do
|
43
|
+
@start_at = Time.now.utc + 180
|
44
|
+
execution_plan.state.must_equal :scheduled
|
45
|
+
plan.plan
|
46
|
+
execution_plan.state.must_equal :planned
|
47
|
+
execution_plan.result.must_equal :pending
|
48
|
+
assert_planning_success execution_plan
|
49
|
+
history_names.call(execution_plan).must_equal ['schedule']
|
50
|
+
executed = plan.execute
|
51
|
+
executed.wait
|
52
|
+
executed.value.state.must_equal :stopped
|
53
|
+
executed.value.result.must_equal :success
|
54
|
+
executed.value.execution_history.count.must_equal 3
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'expired plans can be failed' do
|
58
|
+
@start_at = Time.now.utc + 180
|
59
|
+
plan.timeout
|
60
|
+
execution_plan.state.must_equal :stopped
|
61
|
+
execution_plan.result.must_equal :error
|
62
|
+
execution_plan.errors.first.message.must_match /could not be started before set time/
|
63
|
+
history_names.call(execution_plan).must_equal %W(schedule timeout)
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
describe 'polling scheduler' do
|
69
|
+
let(:dummy_world) { Dynflow::Testing::DummyWorld.new }
|
70
|
+
let(:persistence) { MiniTest::Mock.new }
|
71
|
+
let(:options) { { :poll_interval => 15, :time_source => -> { dummy_world.clock.current_time } } }
|
72
|
+
let(:scheduler) { Schedulers::Polling.new(dummy_world, options) }
|
73
|
+
let(:klok) { dummy_world.clock }
|
74
|
+
|
75
|
+
it 'checks for scheduled plans in regular intervals' do
|
76
|
+
start_time = klok.current_time
|
77
|
+
persistence.expect(:find_past_scheduled_plans, [], [start_time])
|
78
|
+
persistence.expect(:find_past_scheduled_plans, [], [start_time + options[:poll_interval]])
|
79
|
+
dummy_world.stub :persistence, persistence do
|
80
|
+
klok.pending_pings.length.must_equal 0
|
81
|
+
scheduler.start.wait
|
82
|
+
klok.pending_pings.length.must_equal 1
|
83
|
+
klok.pending_pings.first.who.ref.must_be_same_as scheduler.core
|
84
|
+
klok.pending_pings.first.when.must_equal start_time + options[:poll_interval]
|
85
|
+
klok.progress
|
86
|
+
scheduler.terminate.wait
|
87
|
+
klok.pending_pings.length.must_equal 1
|
88
|
+
klok.pending_pings.first.who.ref.must_be_same_as scheduler.core
|
89
|
+
klok.pending_pings.first.when.must_equal start_time + 2 * options[:poll_interval]
|
90
|
+
klok.progress
|
91
|
+
klok.pending_pings.length.must_equal 0
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
describe 'serializers' do
|
97
|
+
let(:save_and_load) do
|
98
|
+
->(thing) { MultiJson.load(MultiJson.dump(thing)) }
|
99
|
+
end
|
100
|
+
let(:simulated_use) do
|
101
|
+
lambda do |serializer_class, input|
|
102
|
+
serializer = serializer_class.new
|
103
|
+
serializer.deserialize(save_and_load.call(serializer.serialize *input))
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'noop serializer [de]serializes correctly for simple types' do
|
108
|
+
input = [1, 2.0, 'three', ['four-1', 'four-2'], { 'five' => 5 }]
|
109
|
+
simulated_use.call(Dynflow::Serializers::Noop, input).must_equal input
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
data/test/middleware_test.rb
CHANGED
@@ -12,8 +12,14 @@ module Dynflow
|
|
12
12
|
end
|
13
13
|
|
14
14
|
it "wraps the action method calls" do
|
15
|
-
world.
|
16
|
-
|
15
|
+
schedule = world.schedule(Support::MiddlewareExample::LoggingAction, { :start_at => Time.now.utc - 60 }, {})
|
16
|
+
plan = world.persistence.load_scheduled_plan schedule.execution_plan_id
|
17
|
+
plan.plan
|
18
|
+
plan.execute.wait
|
19
|
+
log.must_equal %w[LogMiddleware::before_schedule
|
20
|
+
schedule
|
21
|
+
LogMiddleware::after_schedule
|
22
|
+
LogMiddleware::before_plan_phase
|
17
23
|
LogMiddleware::before_plan
|
18
24
|
plan
|
19
25
|
LogMiddleware::after_plan
|
@@ -14,6 +14,12 @@ module Support
|
|
14
14
|
LogMiddleware.log << "#{self.class.name[/\w+$/]}::#{message}"
|
15
15
|
end
|
16
16
|
|
17
|
+
def schedule(*args)
|
18
|
+
log 'before_schedule'
|
19
|
+
pass *args
|
20
|
+
log 'after_schedule'
|
21
|
+
end
|
22
|
+
|
17
23
|
def plan(args)
|
18
24
|
log 'before_plan'
|
19
25
|
pass(args)
|
@@ -72,6 +78,11 @@ module Support
|
|
72
78
|
LogMiddleware.log << message
|
73
79
|
end
|
74
80
|
|
81
|
+
def schedule(schedule_options, *args)
|
82
|
+
log 'schedule'
|
83
|
+
Dynflow::Serializers::Noop.new
|
84
|
+
end
|
85
|
+
|
75
86
|
def plan(input)
|
76
87
|
log 'plan'
|
77
88
|
plan_self(input)
|
data/test/test_helper.rb
CHANGED
@@ -88,6 +88,7 @@ module WorldFactory
|
|
88
88
|
config.persistence_adapter = persistence_adapter
|
89
89
|
config.logger_adapter = logger_adapter
|
90
90
|
config.coordinator_adapter = coordinator_adapter
|
91
|
+
config.scheduler = nil
|
91
92
|
config.auto_rescue = false
|
92
93
|
config.exit_on_terminate = false
|
93
94
|
config.auto_execute = false
|
data/web/views/show.erb
CHANGED
@@ -11,6 +11,17 @@
|
|
11
11
|
<%= h(@plan.result) %>
|
12
12
|
</p>
|
13
13
|
|
14
|
+
<% if @plan.state == :scheduled %>
|
15
|
+
<p>
|
16
|
+
<b>Start at:</b>
|
17
|
+
<%= h(@plan.schedule_record.start_at) %>
|
18
|
+
</p>
|
19
|
+
<p>
|
20
|
+
<b>Start before:</b>
|
21
|
+
<%= h(@plan.schedule_record.start_before || "-") %>
|
22
|
+
</p>
|
23
|
+
<% end %>
|
24
|
+
|
14
25
|
<p>
|
15
26
|
<b>Started at:</b>
|
16
27
|
<%= h(@plan.started_at) %>
|