dynflow 1.4.3 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/{test/prepare_travis_env.sh → .github/install_dependencies.sh} +2 -2
  3. data/.github/workflows/ruby.yml +116 -0
  4. data/lib/dynflow.rb +1 -1
  5. data/lib/dynflow/action.rb +22 -12
  6. data/lib/dynflow/action/suspended.rb +4 -4
  7. data/lib/dynflow/action/timeouts.rb +2 -2
  8. data/lib/dynflow/actor.rb +20 -4
  9. data/lib/dynflow/clock.rb +2 -2
  10. data/lib/dynflow/connectors/abstract.rb +4 -0
  11. data/lib/dynflow/connectors/database.rb +4 -0
  12. data/lib/dynflow/connectors/direct.rb +5 -0
  13. data/lib/dynflow/director.rb +5 -1
  14. data/lib/dynflow/director/running_steps_manager.rb +2 -2
  15. data/lib/dynflow/dispatcher.rb +2 -1
  16. data/lib/dynflow/dispatcher/client_dispatcher.rb +8 -2
  17. data/lib/dynflow/dispatcher/executor_dispatcher.rb +4 -2
  18. data/lib/dynflow/execution_history.rb +1 -1
  19. data/lib/dynflow/execution_plan.rb +12 -4
  20. data/lib/dynflow/executors.rb +32 -10
  21. data/lib/dynflow/executors/abstract/core.rb +1 -1
  22. data/lib/dynflow/executors/parallel.rb +2 -2
  23. data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb +1 -1
  24. data/lib/dynflow/flows.rb +1 -0
  25. data/lib/dynflow/flows/abstract.rb +14 -0
  26. data/lib/dynflow/flows/abstract_composed.rb +2 -7
  27. data/lib/dynflow/flows/atom.rb +2 -2
  28. data/lib/dynflow/flows/concurrence.rb +2 -0
  29. data/lib/dynflow/flows/registry.rb +32 -0
  30. data/lib/dynflow/flows/sequence.rb +2 -0
  31. data/lib/dynflow/persistence.rb +8 -0
  32. data/lib/dynflow/persistence_adapters/abstract.rb +16 -0
  33. data/lib/dynflow/persistence_adapters/sequel.rb +24 -8
  34. data/lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb +30 -0
  35. data/lib/dynflow/rails.rb +1 -1
  36. data/lib/dynflow/rails/configuration.rb +16 -5
  37. data/lib/dynflow/testing/in_thread_executor.rb +2 -2
  38. data/lib/dynflow/testing/in_thread_world.rb +5 -5
  39. data/lib/dynflow/version.rb +1 -1
  40. data/lib/dynflow/world.rb +5 -5
  41. data/lib/dynflow/world/invalidation.rb +5 -1
  42. data/test/dispatcher_test.rb +6 -0
  43. data/test/flows_test.rb +44 -0
  44. data/test/future_execution_test.rb +1 -1
  45. data/test/persistence_test.rb +38 -2
  46. metadata +12 -9
  47. data/.travis.yml +0 -33
@@ -34,8 +34,8 @@ module Dynflow
34
34
  @director.work_finished(work_item)
35
35
  end
36
36
 
37
- def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
38
- event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
37
+ def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
38
+ event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
39
39
  @director.handle_event(event).each do |work_item|
40
40
  @work_items << work_item
41
41
  end
@@ -58,15 +58,15 @@ module Dynflow
58
58
  future.reject e
59
59
  end
60
60
 
61
- def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
62
- @executor.event(execution_plan_id, step_id, event, done)
61
+ def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
62
+ @executor.event(execution_plan_id, step_id, event, done, optional: optional)
63
63
  end
64
64
 
65
- def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
65
+ def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
66
66
  if time.nil? || time < Time.now
67
- event(execution_plan_id, step_id, event, done)
67
+ event(execution_plan_id, step_id, event, done, optional: optional)
68
68
  else
69
- clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
69
+ clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
70
70
  end
71
71
  end
72
72
  end
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module Dynflow
3
- VERSION = '1.4.3'
3
+ VERSION = '1.5.0'
4
4
  end
data/lib/dynflow/world.rb CHANGED
@@ -219,12 +219,12 @@ module Dynflow
219
219
  publish_request(Dispatcher::Execution[execution_plan_id], done, true)
220
220
  end
221
221
 
222
- def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
223
- publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
222
+ def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
223
+ publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
224
224
  end
225
225
 
226
- def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
227
- publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
226
+ def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
227
+ publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
228
228
  end
229
229
 
230
230
  def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)
@@ -341,7 +341,7 @@ module Dynflow
341
341
  @terminating = Concurrent::Promises.future do
342
342
  termination_future.wait(termination_timeout)
343
343
  end.on_resolution do
344
- @terminated.complete
344
+ @terminated.resolve
345
345
  Thread.new { Kernel.exit } if @exit_on_terminate.true?
346
346
  end
347
347
  end
@@ -28,6 +28,8 @@ module Dynflow
28
28
  end
29
29
  end
30
30
 
31
+ pruned = persistence.prune_envelopes(world.id)
32
+ logger.error("Pruned #{pruned} envelopes for invalidated world #{world.id}") unless pruned.zero?
31
33
  coordinator.delete_world(world)
32
34
  end
33
35
  end
@@ -41,7 +43,7 @@ module Dynflow
41
43
  else
42
44
  :stopped
43
45
  end
44
- plan.update_state(state)
46
+ plan.update_state(state) if plan.state != state
45
47
 
46
48
  coordinator.release(planning_lock)
47
49
  execute(plan.id) if plan.state == :planned
@@ -115,6 +117,8 @@ module Dynflow
115
117
  def perform_validity_checks
116
118
  world_invalidation_result = worlds_validity_check
117
119
  locks_validity_check
120
+ pruned = connector.prune_undeliverable_envelopes(self)
121
+ logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
118
122
  world_invalidation_result.values.select { |result| result == :invalidated }.size
119
123
  end
120
124
 
@@ -49,6 +49,12 @@ module Dynflow
49
49
  plan = result.finished.value
50
50
  assert_equal('finish', plan.actions.first.output[:event])
51
51
  end
52
+
53
+ it 'does not error on dispatching an optional event' do
54
+ request = client_world.event('123', 1, nil, optional: true)
55
+ request.wait(20)
56
+ assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
57
+ end
52
58
  end
53
59
  end
54
60
  end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+ require_relative 'test_helper'
3
+ require 'mocha/minitest'
4
+
5
+ module Dynflow
6
+ describe 'flow' do
7
+
8
+ class TestRegistry < Flows::Registry
9
+ class << self
10
+ def reset!
11
+ @serialization_map = {}
12
+ end
13
+ end
14
+ end
15
+
16
+ after do
17
+ TestRegistry.reset!
18
+ end
19
+
20
+ describe "registry" do
21
+ it "allows registering values" do
22
+ TestRegistry.register!(TestRegistry, 'TS')
23
+ TestRegistry.register!(Integer, 'I')
24
+ map = TestRegistry.instance_variable_get("@serialization_map")
25
+ _(map).must_equal({'TS' => TestRegistry, 'I' => Integer})
26
+ end
27
+
28
+ it "prevents overwriting values" do
29
+ TestRegistry.register!(Integer, 'I')
30
+ _(-> { TestRegistry.register!(Float, 'I') }).must_raise Flows::Registry::IdentifierTaken
31
+ end
32
+
33
+ it "encodes and decodes values" do
34
+ TestRegistry.register!(Integer, 'I')
35
+ _(TestRegistry.encode(Integer)).must_equal 'I'
36
+ end
37
+
38
+ it "raises an exception when unknown key is requested" do
39
+ _(-> { TestRegistry.encode(Float) }).must_raise Flows::Registry::UnknownIdentifier
40
+ _(-> { TestRegistry.decode('F') }).must_raise Flows::Registry::UnknownIdentifier
41
+ end
42
+ end
43
+ end
44
+ end
@@ -55,7 +55,7 @@ module Dynflow
55
55
 
56
56
  it 'delays the action' do
57
57
  _(execution_plan.steps.count).must_equal 1
58
- _(delayed_plan.start_at.inspect).must_equal (@start_at).inspect
58
+ _(delayed_plan.start_at).must_be_within_delta(@start_at, 0.5)
59
59
  _(history_names.call(execution_plan)).must_equal ['delay']
60
60
  end
61
61
 
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
  require_relative 'test_helper'
3
3
  require 'tmpdir'
4
+ require 'ostruct'
4
5
 
5
6
  module Dynflow
6
7
  module PersistenceTest
@@ -85,7 +86,7 @@ module Dynflow
85
86
  original.each do |key, value|
86
87
  loaded_value = loaded[key.to_s]
87
88
  if value.is_a?(Time)
88
- _(loaded_value.inspect).must_equal value.inspect
89
+ _(loaded_value).must_be_within_delta(value, 0.5)
89
90
  elsif value.is_a?(Hash)
90
91
  assert_equal_attributes!(value, loaded_value)
91
92
  elsif value.nil?
@@ -347,7 +348,7 @@ module Dynflow
347
348
  if value.nil?
348
349
  assert_nil stored.fetch(name.to_sym)
349
350
  elsif value.is_a?(Time)
350
- _(stored.fetch(name.to_sym).inspect).must_equal value.inspect
351
+ _(stored.fetch(name.to_sym)).must_be_within_delta(value, 0.5)
351
352
  else
352
353
  _(stored.fetch(name.to_sym)).must_equal value
353
354
  end
@@ -371,6 +372,41 @@ module Dynflow
371
372
  assert_equal [], adapter.pull_envelopes(executor_world_id)
372
373
  end
373
374
 
375
+ it 'supports pruning of envelopes of invalidated worlds' do
376
+ client_world_id = '5678'
377
+ executor_world_id = '1234'
378
+ envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
379
+ executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
380
+ client_envelope = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
381
+ envelopes = [client_envelope, executor_envelope]
382
+
383
+ envelopes.each { |e| adapter.push_envelope(e) }
384
+
385
+ assert_equal 1, adapter.prune_envelopes([executor_world_id])
386
+ assert_equal 0, adapter.prune_envelopes([executor_world_id])
387
+ assert_equal [], adapter.pull_envelopes(executor_world_id)
388
+ assert_equal [client_envelope], adapter.pull_envelopes(client_world_id)
389
+ end
390
+
391
+ it 'supports pruning of orphaned envelopes' do
392
+ client_world_id = '5678'
393
+ executor_world_id = '1234'
394
+ envelope_hash = ->(envelope) { Dynflow::Utils.indifferent_hash(Dynflow.serializer.dump(envelope)) }
395
+ executor_envelope = envelope_hash.call(Dispatcher::Envelope['123', client_world_id, executor_world_id, Dispatcher::Execution['111']])
396
+ client_envelope = envelope_hash.call(Dispatcher::Envelope['123', executor_world_id, client_world_id, Dispatcher::Accepted])
397
+ envelopes = [client_envelope, executor_envelope]
398
+
399
+ envelopes.each { |e| adapter.push_envelope(e) }
400
+ adapter.insert_coordinator_record({"class"=>"Dynflow::Coordinator::ExecutorWorld",
401
+ "id" => executor_world_id, "meta" => {}, "active" => true })
402
+
403
+ assert_equal 1, adapter.prune_undeliverable_envelopes
404
+ assert_equal 0, adapter.prune_undeliverable_envelopes
405
+ assert_equal [], adapter.pull_envelopes(client_world_id)
406
+ assert_equal [executor_envelope], adapter.pull_envelopes(executor_world_id)
407
+ assert_equal [], adapter.pull_envelopes(executor_world_id)
408
+ end
409
+
374
410
  it 'supports reading data saved prior to normalization' do
375
411
  db = adapter.send(:db)
376
412
  # Prepare records for saving
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dynflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.3
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ivan Necas
8
8
  - Petr Chalupa
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-03-04 00:00:00.000000000 Z
12
+ date: 2021-05-13 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: multi_json
@@ -242,10 +242,11 @@ executables: []
242
242
  extensions: []
243
243
  extra_rdoc_files: []
244
244
  files:
245
+ - ".github/install_dependencies.sh"
246
+ - ".github/workflows/ruby.yml"
245
247
  - ".gitignore"
246
248
  - ".rubocop.yml"
247
249
  - ".rubocop_todo.yml"
248
- - ".travis.yml"
249
250
  - Dockerfile
250
251
  - Gemfile
251
252
  - MIT-LICENSE
@@ -474,6 +475,7 @@ files:
474
475
  - lib/dynflow/flows/abstract_composed.rb
475
476
  - lib/dynflow/flows/atom.rb
476
477
  - lib/dynflow/flows/concurrence.rb
478
+ - lib/dynflow/flows/registry.rb
477
479
  - lib/dynflow/flows/sequence.rb
478
480
  - lib/dynflow/logger_adapters.rb
479
481
  - lib/dynflow/logger_adapters/abstract.rb
@@ -512,6 +514,7 @@ files:
512
514
  - lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
513
515
  - lib/dynflow/persistence_adapters/sequel_migrations/018_add_uuid_column.rb
514
516
  - lib/dynflow/persistence_adapters/sequel_migrations/019_update_mysql_time_precision.rb
517
+ - lib/dynflow/persistence_adapters/sequel_migrations/020_drop_duplicate_indices.rb
515
518
  - lib/dynflow/rails.rb
516
519
  - lib/dynflow/rails/configuration.rb
517
520
  - lib/dynflow/rails/daemon.rb
@@ -575,11 +578,11 @@ files:
575
578
  - test/execution_plan_hooks_test.rb
576
579
  - test/execution_plan_test.rb
577
580
  - test/executor_test.rb
581
+ - test/flows_test.rb
578
582
  - test/future_execution_test.rb
579
583
  - test/memory_cosumption_watcher_test.rb
580
584
  - test/middleware_test.rb
581
585
  - test/persistence_test.rb
582
- - test/prepare_travis_env.sh
583
586
  - test/redis_locking_test.rb
584
587
  - test/rescue_test.rb
585
588
  - test/round_robin_test.rb
@@ -624,7 +627,7 @@ homepage: https://github.com/Dynflow/dynflow
624
627
  licenses:
625
628
  - MIT
626
629
  metadata: {}
627
- post_install_message:
630
+ post_install_message:
628
631
  rdoc_options: []
629
632
  require_paths:
630
633
  - lib
@@ -639,8 +642,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
639
642
  - !ruby/object:Gem::Version
640
643
  version: '0'
641
644
  requirements: []
642
- rubygems_version: 3.0.3
643
- signing_key:
645
+ rubygems_version: 3.1.2
646
+ signing_key:
644
647
  specification_version: 4
645
648
  summary: DYNamic workFLOW engine
646
649
  test_files:
@@ -658,11 +661,11 @@ test_files:
658
661
  - test/execution_plan_hooks_test.rb
659
662
  - test/execution_plan_test.rb
660
663
  - test/executor_test.rb
664
+ - test/flows_test.rb
661
665
  - test/future_execution_test.rb
662
666
  - test/memory_cosumption_watcher_test.rb
663
667
  - test/middleware_test.rb
664
668
  - test/persistence_test.rb
665
- - test/prepare_travis_env.sh
666
669
  - test/redis_locking_test.rb
667
670
  - test/rescue_test.rb
668
671
  - test/round_robin_test.rb
data/.travis.yml DELETED
@@ -1,33 +0,0 @@
1
- language: ruby
2
-
3
- services:
4
- - postgresql
5
- - redis
6
-
7
- rvm:
8
- - "2.3.1"
9
- - "2.4.0"
10
- - "2.5.0"
11
-
12
- env:
13
- global:
14
- - "TESTOPTS=--verbose DB=postgresql DB_CONN_STRING=postgres://postgres@localhost/travis_ci_test"
15
-
16
- matrix:
17
- include:
18
- - rvm: "2.4.0"
19
- env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
20
- services:
21
- - mysql
22
- - redis
23
- - rvm: "2.4.0"
24
- env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
25
- - rvm: "2.4.0"
26
- env: "CONCURRENT_RUBY_EXT=true"
27
-
28
- install:
29
- - test/prepare_travis_env.sh
30
-
31
- script:
32
- - bundle exec rubocop
33
- - bundle exec rake test