dynflow 1.4.3 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{test/prepare_travis_env.sh → .github/install_dependencies.sh} +2 -2
- data/.github/workflows/ruby.yml +116 -0
- data/lib/dynflow.rb +1 -1
- data/lib/dynflow/action.rb +22 -12
- data/lib/dynflow/action/suspended.rb +4 -4
- data/lib/dynflow/action/timeouts.rb +2 -2
- data/lib/dynflow/actor.rb +20 -4
- data/lib/dynflow/clock.rb +2 -2
- data/lib/dynflow/connectors/abstract.rb +4 -0
- data/lib/dynflow/connectors/database.rb +4 -0
- data/lib/dynflow/connectors/direct.rb +5 -0
- data/lib/dynflow/director.rb +5 -1
- data/lib/dynflow/director/running_steps_manager.rb +2 -2
- data/lib/dynflow/dispatcher.rb +2 -1
- data/lib/dynflow/dispatcher/client_dispatcher.rb +8 -2
- data/lib/dynflow/dispatcher/executor_dispatcher.rb +4 -2
- data/lib/dynflow/execution_history.rb +1 -1
- data/lib/dynflow/execution_plan.rb +12 -4
- data/lib/dynflow/executors.rb +32 -10
- data/lib/dynflow/executors/abstract/core.rb +1 -1
- data/lib/dynflow/executors/parallel.rb +2 -2
- data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb +1 -1
- data/lib/dynflow/flows.rb +1 -0
- data/lib/dynflow/flows/abstract.rb +14 -0
- data/lib/dynflow/flows/abstract_composed.rb +2 -7
- data/lib/dynflow/flows/atom.rb +2 -2
- data/lib/dynflow/flows/concurrence.rb +2 -0
- data/lib/dynflow/flows/registry.rb +32 -0
- data/lib/dynflow/flows/sequence.rb +2 -0
- data/lib/dynflow/persistence.rb +8 -0
- data/lib/dynflow/persistence_adapters/abstract.rb +16 -0
- data/lib/dynflow/persistence_adapters/sequel.rb +24 -8
- data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb +30 -0
- data/lib/dynflow/rails.rb +1 -1
- data/lib/dynflow/rails/configuration.rb +16 -5
- data/lib/dynflow/testing/in_thread_executor.rb +2 -2
- data/lib/dynflow/testing/in_thread_world.rb +5 -5
- data/lib/dynflow/version.rb +1 -1
- data/lib/dynflow/world.rb +5 -5
- data/lib/dynflow/world/invalidation.rb +5 -1
- data/test/dispatcher_test.rb +6 -0
- data/test/flows_test.rb +44 -0
- data/test/future_execution_test.rb +1 -1
- data/test/persistence_test.rb +38 -2
- metadata +12 -9
- data/.travis.yml +0 -33
@@ -34,8 +34,8 @@ module Dynflow
|
|
34
34
|
@director.work_finished(work_item)
|
35
35
|
end
|
36
36
|
|
37
|
-
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
|
38
|
-
event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
|
37
|
+
def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
|
38
|
+
event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
|
39
39
|
@director.handle_event(event).each do |work_item|
|
40
40
|
@work_items << work_item
|
41
41
|
end
|
@@ -58,15 +58,15 @@ module Dynflow
|
|
58
58
|
future.reject e
|
59
59
|
end
|
60
60
|
|
61
|
-
def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
|
62
|
-
@executor.event(execution_plan_id, step_id, event, done)
|
61
|
+
def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
|
62
|
+
@executor.event(execution_plan_id, step_id, event, done, optional: optional)
|
63
63
|
end
|
64
64
|
|
65
|
-
def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
|
65
|
+
def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
|
66
66
|
if time.nil? || time < Time.now
|
67
|
-
event(execution_plan_id, step_id, event, done)
|
67
|
+
event(execution_plan_id, step_id, event, done, optional: optional)
|
68
68
|
else
|
69
|
-
clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
|
69
|
+
clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
|
70
70
|
end
|
71
71
|
end
|
72
72
|
end
|
data/lib/dynflow/version.rb
CHANGED
data/lib/dynflow/world.rb
CHANGED
@@ -219,12 +219,12 @@ module Dynflow
|
|
219
219
|
publish_request(Dispatcher::Execution[execution_plan_id], done, true)
|
220
220
|
end
|
221
221
|
|
222
|
-
def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
|
223
|
-
publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
|
222
|
+
def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
|
223
|
+
publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
|
224
224
|
end
|
225
225
|
|
226
|
-
def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
|
227
|
-
publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
|
226
|
+
def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
|
227
|
+
publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
|
228
228
|
end
|
229
229
|
|
230
230
|
def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)
|
@@ -341,7 +341,7 @@ module Dynflow
|
|
341
341
|
@terminating = Concurrent::Promises.future do
|
342
342
|
termination_future.wait(termination_timeout)
|
343
343
|
end.on_resolution do
|
344
|
-
@terminated.
|
344
|
+
@terminated.resolve
|
345
345
|
Thread.new { Kernel.exit } if @exit_on_terminate.true?
|
346
346
|
end
|
347
347
|
end
|
@@ -28,6 +28,8 @@ module Dynflow
|
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
+
pruned = persistence.prune_envelopes(world.id)
|
32
|
+
logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
|
31
33
|
coordinator.delete_world(world)
|
32
34
|
end
|
33
35
|
end
|
@@ -41,7 +43,7 @@ module Dynflow
|
|
41
43
|
else
|
42
44
|
:stopped
|
43
45
|
end
|
44
|
-
plan.update_state(state)
|
46
|
+
plan.update_state(state) if plan.state != state
|
45
47
|
|
46
48
|
coordinator.release(planning_lock)
|
47
49
|
execute(plan.id) if plan.state == :planned
|
@@ -115,6 +117,8 @@ module Dynflow
|
|
115
117
|
def perform_validity_checks
|
116
118
|
world_invalidation_result = worlds_validity_check
|
117
119
|
locks_validity_check
|
120
|
+
pruned = connector.prune_undeliverable_envelopes(self)
|
121
|
+
logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
|
118
122
|
world_invalidation_result.values.select { |result| result == :invalidated }.size
|
119
123
|
end
|
120
124
|
|
data/test/dispatcher_test.rb
CHANGED
@@ -49,6 +49,12 @@ module Dynflow
|
|
49
49
|
plan = result.finished.value
|
50
50
|
assert_equal('finish', plan.actions.first.output[:event])
|
51
51
|
end
|
52
|
+
|
53
|
+
it 'does not error on dispatching an optional event' do
|
54
|
+
request = client_world.event('123', 1, nil, optional: true)
|
55
|
+
request.wait(20)
|
56
|
+
assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
|
57
|
+
end
|
52
58
|
end
|
53
59
|
end
|
54
60
|
end
|
data/test/flows_test.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require_relative 'test_helper'
|
3
|
+
require 'mocha/minitest'
|
4
|
+
|
5
|
+
module Dynflow
|
6
|
+
describe 'flow' do
|
7
|
+
|
8
|
+
class TestRegistry < Flows::Registry
|
9
|
+
class << self
|
10
|
+
def reset!
|
11
|
+
@serialization_map = {}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
TestRegistry.reset!
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "registry" do
|
21
|
+
it "allows registering values" do
|
22
|
+
TestRegistry.register!(TestRegistry, 'TS')
|
23
|
+
TestRegistry.register!(Integer, 'I')
|
24
|
+
map = TestRegistry.instance_variable_get("@serialization_map")
|
25
|
+
_(map).must_equal({'TS' => TestRegistry, 'I' => Integer})
|
26
|
+
end
|
27
|
+
|
28
|
+
it "prevents overwriting values" do
|
29
|
+
TestRegistry.register!(Integer, 'I')
|
30
|
+
_(-> { TestRegistry.register!(Float, 'I') }).must_raise Flows::Registry::IdentifierTaken
|
31
|
+
end
|
32
|
+
|
33
|
+
it "encodes and decodes values" do
|
34
|
+
TestRegistry.register!(Integer, 'I')
|
35
|
+
_(TestRegistry.encode(Integer)).must_equal 'I'
|
36
|
+
end
|
37
|
+
|
38
|
+
it "raises an exception when unknown key is requested" do
|
39
|
+
_(-> { TestRegistry.encode(Float) }).must_raise Flows::Registry::UnknownIdentifier
|
40
|
+
_(-> { TestRegistry.decode('F') }).must_raise Flows::Registry::UnknownIdentifier
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -55,7 +55,7 @@ module Dynflow
|
|
55
55
|
|
56
56
|
it 'delays the action' do
|
57
57
|
_(execution_plan.steps.count).must_equal 1
|
58
|
-
_(delayed_plan.start_at
|
58
|
+
_(delayed_plan.start_at).must_be_within_delta(@start_at, 0.5)
|
59
59
|
_(history_names.call(execution_plan)).must_equal ['delay']
|
60
60
|
end
|
61
61
|
|
data/test/persistence_test.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
require_relative 'test_helper'
|
3
3
|
require 'tmpdir'
|
4
|
+
require 'ostruct'
|
4
5
|
|
5
6
|
module Dynflow
|
6
7
|
module PersistenceTest
|
@@ -85,7 +86,7 @@ module Dynflow
|
|
85
86
|
original.each do |key, value|
|
86
87
|
loaded_value = loaded[key.to_s]
|
87
88
|
if value.is_a?(Time)
|
88
|
-
_(loaded_value
|
89
|
+
_(loaded_value).must_be_within_delta(value, 0.5)
|
89
90
|
elsif value.is_a?(Hash)
|
90
91
|
assert_equal_attributes!(value, loaded_value)
|
91
92
|
elsif value.nil?
|
@@ -347,7 +348,7 @@ module Dynflow
|
|
347
348
|
if value.nil?
|
348
349
|
assert_nil stored.fetch(name.to_sym)
|
349
350
|
elsif value.is_a?(Time)
|
350
|
-
_(stored.fetch(name.to_sym)
|
351
|
+
_(stored.fetch(name.to_sym)).must_be_within_delta(value, 0.5)
|
351
352
|
else
|
352
353
|
_(stored.fetch(name.to_sym)).must_equal value
|
353
354
|
end
|
@@ -371,6 +372,41 @@ module Dynflow
|
|
371
372
|
assert_equal [], adapter.pull_envelopes(executor_world_id)
|
372
373
|
end
|
373
374
|
|
375
|
+
it 'supports pruning of envelopes of invalidated worlds' do
|
376
|
+
client_world_id = '5678'
|
377
|
+
executor_world_id = '1234'
|
378
|
+
envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
|
379
|
+
executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
|
380
|
+
client_envelope = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
|
381
|
+
envelopes = [client_envelope, executor_envelope]
|
382
|
+
|
383
|
+
envelopes.each { |e| adapter.push_envelope(e) }
|
384
|
+
|
385
|
+
assert_equal 1, adapter.prune_envelopes([executor_world_id])
|
386
|
+
assert_equal 0, adapter.prune_envelopes([executor_world_id])
|
387
|
+
assert_equal [], adapter.pull_envelopes(executor_world_id)
|
388
|
+
assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
|
389
|
+
end
|
390
|
+
|
391
|
+
it 'supports pruning of orphaned envelopes' do
|
392
|
+
client_world_id = '5678'
|
393
|
+
executor_world_id = '1234'
|
394
|
+
envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
|
395
|
+
executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
|
396
|
+
client_envelope = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
|
397
|
+
envelopes = [client_envelope, executor_envelope]
|
398
|
+
|
399
|
+
envelopes.each { |e| adapter.push_envelope(e) }
|
400
|
+
adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
|
401
|
+
"id" => executor_world_id, "meta" => {}, "active" => true })
|
402
|
+
|
403
|
+
assert_equal 1, adapter.prune_undeliverable_envelopes
|
404
|
+
assert_equal 0, adapter.prune_undeliverable_envelopes
|
405
|
+
assert_equal [], adapter.pull_envelopes(client_world_id)
|
406
|
+
assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
|
407
|
+
assert_equal [], adapter.pull_envelopes(executor_world_id)
|
408
|
+
end
|
409
|
+
|
374
410
|
it 'supports reading data saved prior to normalization' do
|
375
411
|
db = adapter.send(:db)
|
376
412
|
# Prepare records for saving
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ivan Necas
|
8
8
|
- Petr Chalupa
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-05-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: multi_json
|
@@ -242,10 +242,11 @@ executables: []
|
|
242
242
|
extensions: []
|
243
243
|
extra_rdoc_files: []
|
244
244
|
files:
|
245
|
+
- ".github/install_dependencies.sh"
|
246
|
+
- ".github/workflows/ruby.yml"
|
245
247
|
- ".gitignore"
|
246
248
|
- ".rubocop.yml"
|
247
249
|
- ".rubocop_todo.yml"
|
248
|
-
- ".travis.yml"
|
249
250
|
- Dockerfile
|
250
251
|
- Gemfile
|
251
252
|
- MIT-LICENSE
|
@@ -474,6 +475,7 @@ files:
|
|
474
475
|
- lib/dynflow/flows/abstract_composed.rb
|
475
476
|
- lib/dynflow/flows/atom.rb
|
476
477
|
- lib/dynflow/flows/concurrence.rb
|
478
|
+
- lib/dynflow/flows/registry.rb
|
477
479
|
- lib/dynflow/flows/sequence.rb
|
478
480
|
- lib/dynflow/logger_adapters.rb
|
479
481
|
- lib/dynflow/logger_adapters/abstract.rb
|
@@ -512,6 +514,7 @@ files:
|
|
512
514
|
- lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
|
513
515
|
- lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
|
514
516
|
- lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
|
517
|
+
- lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
|
515
518
|
- lib/dynflow/rails.rb
|
516
519
|
- lib/dynflow/rails/configuration.rb
|
517
520
|
- lib/dynflow/rails/daemon.rb
|
@@ -575,11 +578,11 @@ files:
|
|
575
578
|
- test/execution_plan_hooks_test.rb
|
576
579
|
- test/execution_plan_test.rb
|
577
580
|
- test/executor_test.rb
|
581
|
+
- test/flows_test.rb
|
578
582
|
- test/future_execution_test.rb
|
579
583
|
- test/memory_cosumption_watcher_test.rb
|
580
584
|
- test/middleware_test.rb
|
581
585
|
- test/persistence_test.rb
|
582
|
-
- test/prepare_travis_env.sh
|
583
586
|
- test/redis_locking_test.rb
|
584
587
|
- test/rescue_test.rb
|
585
588
|
- test/round_robin_test.rb
|
@@ -624,7 +627,7 @@ homepage: https://github.com/Dynflow/dynflow
|
|
624
627
|
licenses:
|
625
628
|
- MIT
|
626
629
|
metadata: {}
|
627
|
-
post_install_message:
|
630
|
+
post_install_message:
|
628
631
|
rdoc_options: []
|
629
632
|
require_paths:
|
630
633
|
- lib
|
@@ -639,8 +642,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
639
642
|
- !ruby/object:Gem::Version
|
640
643
|
version: '0'
|
641
644
|
requirements: []
|
642
|
-
rubygems_version: 3.
|
643
|
-
signing_key:
|
645
|
+
rubygems_version: 3.1.2
|
646
|
+
signing_key:
|
644
647
|
specification_version: 4
|
645
648
|
summary: DYNamic workFLOW engine
|
646
649
|
test_files:
|
@@ -658,11 +661,11 @@ test_files:
|
|
658
661
|
- test/execution_plan_hooks_test.rb
|
659
662
|
- test/execution_plan_test.rb
|
660
663
|
- test/executor_test.rb
|
664
|
+
- test/flows_test.rb
|
661
665
|
- test/future_execution_test.rb
|
662
666
|
- test/memory_cosumption_watcher_test.rb
|
663
667
|
- test/middleware_test.rb
|
664
668
|
- test/persistence_test.rb
|
665
|
-
- test/prepare_travis_env.sh
|
666
669
|
- test/redis_locking_test.rb
|
667
670
|
- test/rescue_test.rb
|
668
671
|
- test/round_robin_test.rb
|
data/.travis.yml
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
|
3
|
-
services:
|
4
|
-
- postgresql
|
5
|
-
- redis
|
6
|
-
|
7
|
-
rvm:
|
8
|
-
- "2.3.1"
|
9
|
-
- "2.4.0"
|
10
|
-
- "2.5.0"
|
11
|
-
|
12
|
-
env:
|
13
|
-
global:
|
14
|
-
- "TESTOPTS=--verbose DB=postgresql DB_CONN_STRING=postgres://postgres@localhost/travis_ci_test"
|
15
|
-
|
16
|
-
matrix:
|
17
|
-
include:
|
18
|
-
- rvm: "2.4.0"
|
19
|
-
env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
|
20
|
-
services:
|
21
|
-
- mysql
|
22
|
-
- redis
|
23
|
-
- rvm: "2.4.0"
|
24
|
-
env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
|
25
|
-
- rvm: "2.4.0"
|
26
|
-
env: "CONCURRENT_RUBY_EXT=true"
|
27
|
-
|
28
|
-
install:
|
29
|
-
- test/prepare_travis_env.sh
|
30
|
-
|
31
|
-
script:
|
32
|
-
- bundle exec rubocop
|
33
|
-
- bundle exec rake test
|