dynflow 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 570d04996df908b45e9d82a5ed4a634a536cfd27db2949547de33ceca5478277
-  data.tar.gz: 169047a160646f11fae66436c79d189fbeda9dfa5d7382a6b75ae14c381d2615
+  metadata.gz: 4219e6a5f8d1e1829a43c97bb12a406c27e3a14fd9be7d29d795d63580789f9c
+  data.tar.gz: 3d9c6427c2bd1c2483844ca35e4b4a277d41d42113eac775aa9555e033a45e43
 SHA512:
-  metadata.gz: 8d8c8cc61569bb975549e5b94f2c58dba9f65f1d56c8294e83a4f5bc163ada4dcaf19e14035d338683edc9e63750d84e3a67cc288f3dcbf1b9e811fa94e67985
-  data.tar.gz: c54964b58bb14dd6c2bd6088190c598d60abfd2fd78cb6d1fd4932b1d819e865e5edc4e44f7dd960a4cc23ce33413c77523abd04d236cefceab7bd3537d17b02
+  metadata.gz: d62d9d27f2dec86e8c00f2d62bb51e3c81ae5c816fd3c59cde34518cffe7b77dc994acc999d5effd16bf19c73f35f12b6cec9ee6f9f491ba30dbb46eae149dc7
+  data.tar.gz: a6dcded79039250e858b1c89d2d0b3793c1de3e41408d9c60d9e49a053984f849c990940c674a367ba4a6a42429ff06f571f915d6de8206cc27a9e967a2c28ec

data/Gemfile

@@ -4,8 +4,11 @@ source 'https://rubygems.org'
 
 gemspec
 
+# multi_json 1.21.2 fixes json 3 compatibility, but requires Ruby 3.2+
+gem 'json', '< 3.0' if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.2')
+
 group :concurrent_ruby_ext, optional: ENV.key?('CI') && ENV['CONCURRENT_RUBY_EXT'] != 'true' do
-  gem 'concurrent-ruby-ext', '~> 1.1.3'
+  gem 'concurrent-ruby-ext', '~> 1.3.0'
 end
 
 group :pry, optional: ENV.key?('CI') do

data/dynflow.gemspec

@@ -21,9 +21,10 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 3.0.0'
 
   s.add_dependency "algebrick", '~> 0.7.0'
-  s.add_dependency "concurrent-ruby", '~> 1.1.3'
-  s.add_dependency "concurrent-ruby-edge", '~> 0.6.0'
+  s.add_dependency "concurrent-ruby", '>= 1.1.3', '< 2.0'
+  s.add_dependency "concurrent-ruby-edge", '~> 0.7.0'
   s.add_dependency "csv", "~> 3.1"
+  s.add_dependency 'logger'
   s.add_dependency "msgpack", '~> 1.3', '>= 1.3.3'
   s.add_dependency "multi_json"
   s.add_dependency "sequel", '>= 4.0.0'

data/lib/dynflow/execution_plan/steps/finalize_step.rb

@@ -24,6 +24,8 @@ module Dynflow
       end
 
       def mark_to_skip
+        return if state == :skipped
+
         self.state = :skipped
         self.save
       end

data/lib/dynflow/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dynflow
-  VERSION = '2.0.1'
+  VERSION = '2.0.2'
 end

data/lib/dynflow/world/invalidation.rb

@@ -52,17 +52,9 @@ module Dynflow
 
       def invalidate_planning_lock(planning_lock)
         with_valid_execution_plan_for_lock(planning_lock) do |plan|
-          plan.steps.values.each { |step| invalidate_step step }
-
-          state = if plan.plan_steps.any? && plan.plan_steps.all? { |step| step.state == :success }
-                    :planned
-                  else
-                    :stopped
-                  end
-          plan.update_state(state) if plan.state != state
-
+          state = invalidate_planning_plan(plan)
           coordinator.release(planning_lock)
-          execute(plan.id) if plan.state == :planned
+          execute(plan.id) if state == :planned
         end
       end
 
@@ -140,6 +132,7 @@ module Dynflow
             invalidate_execution_lock(lock)
           end
         end
+        execution_plans_validity_check
         pruned = connector.prune_undeliverable_envelopes(self)
         logger.error("Pruned #{pruned} undeliverable envelopes") unless pruned.zero?
         world_invalidation_result.values.select { |result| result == :invalidated }.size
@@ -198,8 +191,54 @@ module Dynflow
         return orphaned_locks
       end
 
+      # Invalidates execution plans left in planning without a planning lock.
+      # Acquiring a new planning lock prevents recovery from racing with another
+      # validity check or a planning process which has just acquired the lock.
+      #
+      # @return [Array<String>] ids of invalidated execution plans
+      def execution_plans_validity_check
+        planning_lock_ids = coordinator.find_locks(class: Coordinator::PlanningLock.name).map(&:execution_plan_id)
+        planning_plans = persistence.find_execution_plans(filters: { state: 'planning' })
+        invalidated_plan_ids = planning_plans.filter_map do |plan|
+          next if planning_lock_ids.include?(plan.id)
+
+          plan.id if invalidate_planning_plan_without_lock(plan.id)
+        end
+
+        unless invalidated_plan_ids.empty?
+          logger.error "execution plans in planning without a planning lock found and invalidated: #{invalidated_plan_ids.inspect}"
+        end
+
+        invalidated_plan_ids
+      end
+
       private
 
+      def invalidate_planning_plan_without_lock(execution_plan_id)
+        state = nil
+        planning_lock = Coordinator::PlanningLock.new(self, execution_plan_id)
+        coordinator.acquire(planning_lock) do
+          plan = persistence.load_execution_plan(execution_plan_id)
+          state = invalidate_planning_plan(plan) if plan.state == :planning
+        end
+        execute(execution_plan_id) if state == :planned
+        !state.nil?
+      rescue Coordinator::LockError, KeyError
+        false
+      end
+
+      def invalidate_planning_plan(plan)
+        plan.steps.values.each { |step| invalidate_step step }
+
+        state = if plan.plan_steps.any? && plan.plan_steps.all? { |step| step.state == :success }
+                  :planned
+                else
+                  :stopped
+                end
+        plan.update_state(state) if plan.state != state
+        state
+      end
+
       def invalidate_step(step)
         if step.state == :running
           step.error = ExecutionPlan::Steps::Error.new("Abnormal termination (previous state: #{step.state})")

data/lib/dynflow/world.rb

@@ -355,21 +355,20 @@ module Dynflow
             rescue Dynflow::Errors::FatalPersistenceError => e
               nil
             end
-            @terminated.resolve
             true
           rescue => e
             logger.fatal(e)
           end
         end
-        @terminating = Concurrent::Promises.future do
+        @terminating = Concurrent::Promises.resolvable_future
+        Thread.new do
           termination_future.wait(termination_timeout)
-        end.on_resolution do
           @terminated.resolve
-          Thread.new do
-            logger.info 'World terminated, exiting.'
-            Kernel.exit if @exit_on_terminate.true?
-          end
+          logger.info 'World terminated, exiting.'
+          @terminating.fulfill(true)
+          Kernel.exit if @exit_on_terminate.true?
         end
+        @terminating
       end
     end
 

data/lib/dynflow.rb

@@ -5,6 +5,7 @@ require 'set'
 require 'base64'
 require 'concurrent'
 require 'concurrent-edge'
+require 'logger'
 
 logger                          = Logger.new($stderr)
 logger.level                    = Logger::INFO

data/test/abnormal_states_recovery_test.rb

@@ -137,21 +137,48 @@ module Dynflow
           end
 
           describe 'planning locks' do
+            it 'executes fully planned execution plans left in planning without a lock' do
+              plan = client_world.plan(Support::DummyExample::Dummy)
+              plan.set_state(:planning, true)
+              plan.save
+
+              client_world_2.perform_validity_checks
+
+              wait_for do
+                plan = client_world_2.persistence.load_execution_plan(plan.id)
+                plan.state == :stopped
+              end
+            end
+
+            it 'stops incompletely planned execution plans without a lock' do
+              plan = client_world.plan(Support::DummyExample::Dummy)
+              plan.set_state(:planning, true)
+              plan.save
+              step = plan.plan_steps.first
+              step.set_state(:pending, true)
+              step.save
+
+              client_world_2.perform_validity_checks
+
+              plan = client_world_2.persistence.load_execution_plan(plan.id)
+              _(plan.state).must_equal :stopped
+            end
+
             it 'releases orphaned planning locks and executes associated execution plans' do
               plan = client_world.plan(Support::DummyExample::Dummy)
               plan.set_state(:planning, true)
               plan.save
               client_world.coordinator.acquire Coordinator::PlanningLock.new(client_world, plan.id)
               executor_world.invalidate(client_world.registered_world)
-              expected_locks = ["lock world-invalidation:#{client_world.id}",
-                                "unlock execution-plan:#{plan.id}", # planning lock
-                                "lock execution-plan:#{plan.id}", # execution lock
-                                "unlock world-invalidation:#{client_world.id}"]
-              _(executor_world.coordinator.adapter.lock_log).must_equal(expected_locks)
               wait_for do
                 plan = client_world_2.persistence.load_execution_plan(plan.id)
                 plan.state == :stopped
               end
+              lock_log = executor_world.coordinator.adapter.lock_log
+              _(lock_log).must_include "lock world-invalidation:#{client_world.id}"
+              _(lock_log).must_include "unlock execution-plan:#{plan.id}" # planning lock
+              _(lock_log).must_include "lock execution-plan:#{plan.id}" # execution lock
+              _(lock_log).must_include "unlock world-invalidation:#{client_world.id}"
             end
 
             it 'releases orphaned planning locks and stops associated execution plans which did not finish planning' do

data/test/action_test.rb

@@ -569,11 +569,8 @@ module Dynflow
                 polling_plan.sub_plans_count == total
               end
 
-              # Moving the clock to make the parent check on sub plans
-              _(clock.pending_pings.count).must_equal 1
-              clock.progress
-
               wait_for('the parent to realise the sub plans failed') do
+                clock.progress
                 polling_plan = world.persistence.load_execution_plan(triggered_plan.id)
                 polling_plan.state == :paused
               end
@@ -586,11 +583,8 @@ module Dynflow
                 polling_plan.sub_plans_count == 2 * total
               end
 
-              # Move the clock again
-              _(clock.pending_pings.count).must_equal 1
-              clock.progress
-
               wait_for('everything to finish successfully') do
+                clock.progress
                 polling_plan = world.persistence.load_execution_plan(triggered_plan.id)
                 polling_plan.state == :stopped && polling_plan.result == :success
               end
@@ -611,7 +605,7 @@ module Dynflow
               end
 
               # Moving the clock to make the parent check on sub plans
-              _(clock.pending_pings.count).must_equal 1
+              wait_for('the parent poll to be scheduled') { clock.pending_pings.count == 1 }
               clock.progress
               _(clock.pending_pings.count).must_equal 0
 
@@ -694,7 +688,7 @@ module Dynflow
               plan.sub_plans_count == total &&
                 plan.sub_plans.all? { |sub| sub.result == :success }
             end
-            _(clock.pending_pings.count).must_equal 1
+            wait_for('the parent poll to be scheduled') { clock.pending_pings.count == 1 }
             clock.progress
             wait_for do
               plan = world.persistence.load_execution_plan(plan.id)
@@ -716,7 +710,7 @@ module Dynflow
               plan.entry_action.output[:planning_finished] == 1
             end
             # Poll was set during #initiate
-            _(clock.pending_pings.count).must_equal 1
+            wait_for('the parent poll to be scheduled') { clock.pending_pings.count == 1 }
 
             # Wait for the sub plans to finish
             wait_for do

data/test/bats/sidekiq-orchestrator.bats

@@ -96,7 +96,10 @@ teardown() {
   wait_for 5 1 grep 'dynflow: Finished performing validity checks' "$(bg_output_file o1)"
 
   podman stop "$POSTGRES_CONTAINER_NAME"
-  wait_for 60 1 grep 'dynflow: World terminated, exiting.' "$(bg_output_file o1)"
+  # With concurrent-ruby 1.3 the persistence error surfaces after
+  # MAX_RETRIES polling rounds, which pushes the termination log past a
+  # 60s window (observed ~58s); allow a generous margin.
+  wait_for 180 1 grep 'dynflow: World terminated, exiting.' "$(bg_output_file o1)"
 }
 
 @test "active orchestrator can withstand temporary pg connection drop" {

data/test/rescue_test.rb

@@ -34,6 +34,25 @@ module Dynflow
           end
         end
 
+        describe 'when skipping a run step fails' do
+          let :execution_plan do
+            execute(Example::ActionWithSkip, 1, :error_on_skip)
+          end
+
+          it 'allows retrying the skip' do
+            2.times do
+              plan = world.persistence.load_execution_plan(execution_plan.id)
+              plan.skip(plan.failed_steps.first)
+              world.execute(plan.id).value
+            end
+
+            plan = world.persistence.load_execution_plan(execution_plan.id)
+            _(plan.state).must_equal :paused
+            _(plan.run_steps.first.state).must_equal :error
+            _(plan.finalize_steps.first.state).must_equal :skipped
+          end
+        end
+
         describe 'of simple skippable action in finalize phase' do
           let :execution_plan do
             execute(Example::ActionWithSkip, 1, :error_on_finalize)

data/test/world_test.rb

@@ -2,6 +2,7 @@
 
 require_relative 'test_helper'
 require 'fileutils'
+require 'mocha/minitest'
 
 module Dynflow
   module WorldTest
@@ -50,6 +51,29 @@ module Dynflow
           terminated_event.wait(10)
           _(terminated_event.resolved?).must_equal true
         end
+
+        it 'finishes when the global IO executor is saturated' do
+          io_executor = Concurrent::FixedThreadPool.new(1)
+          io_started = Concurrent::Event.new
+          io_release = Concurrent::Event.new
+          saturated_world = WorldFactory.create_world { |config| config.termination_timeout = 0.1 }
+          blocker = Concurrent::Promises.future_on(io_executor) do
+            io_started.set
+            io_release.wait
+          end
+          assert io_started.wait(1)
+          Concurrent.stubs(:global_io_executor).returns(io_executor)
+
+          termination = saturated_world.terminate
+
+          assert termination.wait(1)
+          _(saturated_world.terminated.resolved?).must_equal true
+        ensure
+          io_release&.set
+          blocker&.wait(1)
+          io_executor&.shutdown
+          io_executor&.wait_for_termination(10)
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Ivan Necas
@@ -28,30 +28,36 @@ dependencies:
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby-edge
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
@@ -745,7 +765,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.20
 specification_version: 4
 summary: DYNamic workFLOW engine
 test_files: []