dynflow 1.8.2 → 1.8.4

data/lib/dynflow/action/missing.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
+
 module Dynflow
   # for cases the serialized action was renamed and it's not available
   # in the code base anymore.
   class Action::Missing < Dynflow::Action
-
     def self.generate(action_name)
       Class.new(self).tap do |klass|
         klass.singleton_class.send(:define_method, :name) do
@@ -14,17 +14,17 @@ module Dynflow
 
     def plan(*args)
       raise StandardError,
-            "The action class was not found and therefore plan phase failed, this can happen if the action was added/renamed but the executor was not restarted."
+        "The action class was not found and therefore plan phase failed, this can happen if the action was added/renamed but the executor was not restarted."
     end
 
     def run
       raise StandardError,
-            "The action class was not found and therefore run phase failed, this can happen if the action was added/renamed but the executor was not restarted."
+        "The action class was not found and therefore run phase failed, this can happen if the action was added/renamed but the executor was not restarted."
     end
 
     def finalize
       raise StandardError,
-            "The action class was not found and therefore finalize phase failed, this can happen if the action was added/renamed but the executor was not restarted."
+        "The action class was not found and therefore finalize phase failed, this can happen if the action was added/renamed but the executor was not restarted."
     end
   end
 end

data/lib/dynflow/action/polling.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
+
 require 'dynflow/action/timeouts'
 
 module Dynflow
   module Action::Polling
-
     def self.included(base)
       base.send :include, Action::Timeouts
     end
@@ -71,7 +71,7 @@ module Dynflow
 
     # Returns the time to wait between two polling intervals.
     def poll_interval
-      interval_level = poll_attempts[:total]/attempts_before_next_interval
+      interval_level = poll_attempts[:total] / attempts_before_next_interval
       poll_intervals[interval_level] || poll_intervals.last
     end
 
@@ -116,6 +116,5 @@ module Dynflow
         raise error
       end
     end
-
   end
 end

data/lib/dynflow/action/progress.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
-module Dynflow
 
+module Dynflow
   # Methods for specifying the progress of the action
   # the +*_progress+ methods should return number in 0..1.
   # The weight is there to increase/decrease the portion of this task
@@ -10,9 +10,7 @@ module Dynflow
   # The +*_progress+ is run only when the action is in running/suspend state. Otherwise
   # the progress is 1 for success/skipped actions and 0 for errorneous ones.
   module Action::Progress
-
     class Calculate < Middleware
-
       def run(*args)
         with_progress_calculation(*args) do
           [action.run_progress, action.run_progress_weight]
@@ -61,4 +59,3 @@ module Dynflow
     attr_accessor :calculated_progress
   end
 end
-

data/lib/dynflow/action/rescue.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Action::Rescue
-
     Strategy = Algebrick.type do
       variants Skip = atom, Pause = atom, Fail = atom
     end
@@ -65,4 +65,3 @@ module Dynflow
     end
   end
 end
-

data/lib/dynflow/action/singleton.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   class Action
     module Singleton

data/lib/dynflow/action/suspended.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   class Action::Suspended
     attr_reader :execution_plan_id, :step_id

data/lib/dynflow/action/timeouts.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Action::Timeouts
     Timeout = Algebrick.atom
@@ -10,5 +11,5 @@ module Dynflow
     def schedule_timeout(seconds, optional: false)
       plan_event(Timeout, seconds, optional: optional)
     end
- end
+  end
 end

data/lib/dynflow/action/with_bulk_sub_plans.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Action::WithBulkSubPlans
     include Dynflow::Action::Cancellable

data/lib/dynflow/action/with_polling_sub_plans.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Action::WithPollingSubPlans
-
     REFRESH_INTERVAL = 10
     Poll = Algebrick.atom
 

data/lib/dynflow/action/with_sub_plans.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Action::WithSubPlans
     include Dynflow::Action::Cancellable
@@ -16,23 +17,23 @@ module Dynflow
 
     def run(event = nil)
       match event,
-            (on nil do
-               if output[:total_count]
-                 resume
-               else
-                 initiate
-               end
-             end),
-            (on SubPlanFinished do
-               mark_as_done(event.execution_plan_id, event.success)
-               try_to_finish or suspend
-             end),
-            (on Action::Cancellable::Cancel do
-               cancel!
-             end),
-            (on Action::Cancellable::Abort do
-               abort!
-             end)
+        (on nil do
+           if output[:total_count]
+             resume
+           else
+             initiate
+           end
+         end),
+        (on SubPlanFinished do
+           mark_as_done(event.execution_plan_id, event.success)
+           try_to_finish or suspend
+         end),
+        (on Action::Cancellable::Cancel do
+           cancel!
+         end),
+        (on Action::Cancellable::Abort do
+           abort!
+         end)
     end
 
     def initiate
@@ -109,8 +110,8 @@ module Dynflow
         # Assume concurrency level 1 unless stated otherwise
         level = input[:concurrency_control].fetch(:level, {}).fetch(:free, 1)
         semaphore = ::Dynflow::Semaphores::Stateful.new(nil, level,
-                                                        :interval => time.to_f / (count * level),
-                                                        :time_span => time)
+          :interval => time.to_f / (count * level),
+          :time_span => time)
         input[:concurrency_control][:time] = semaphore.to_hash
       end
     end

data/lib/dynflow/action.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
+
 module Dynflow
   # rubocop:disable Metrics/ClassLength
   class Action < Serializable
-
     OutputReference = ExecutionPlan::OutputReference
 
     include Algebrick::TypeCheck
@@ -68,9 +68,9 @@ module Dynflow
     Skip    = Algebrick.atom
     Phase   = Algebrick.type do
       Executable = type do
-        variants Plan     = atom,
-                 Run      = atom,
-                 Finalize = atom
+        variants Plan = atom,
+          Run      = atom,
+          Finalize = atom
       end
       variants Executable, Present = atom
     end
@@ -78,9 +78,9 @@ module Dynflow
     module Executable
       def execute_method_name
         match self,
-              (on Plan, :execute_plan),
-              (on Run, :execute_run),
-              (on Finalize, :execute_finalize)
+          (on Plan, :execute_plan),
+          (on Run, :execute_run),
+          (on Finalize, :execute_finalize)
       end
     end
 
@@ -105,9 +105,9 @@ module Dynflow
     end
 
     attr_reader :world, :phase, :execution_plan_id, :id, :input,
-                :plan_step_id, :run_step_id, :finalize_step_id,
-                :caller_execution_plan_id, :caller_action_id,
-                :pending_output_chunks
+      :plan_step_id, :run_step_id, :finalize_step_id,
+      :caller_execution_plan_id, :caller_action_id,
+      :pending_output_chunks
 
     middleware.use Action::Progress::Calculate
 
@@ -124,12 +124,12 @@ module Dynflow
       @run_step_id       = Type! attributes.fetch(:run_step_id), Integer, NilClass
       @finalize_step_id  = Type! attributes.fetch(:finalize_step_id), Integer, NilClass
 
-      @execution_plan    = Type!(attributes.fetch(:execution_plan), ExecutionPlan) if phase? Present
+      @execution_plan = Type!(attributes.fetch(:execution_plan), ExecutionPlan) if phase? Present
 
       @caller_execution_plan_id  = Type!(attributes.fetch(:caller_execution_plan_id, nil), String, NilClass)
       @caller_action_id          = Type!(attributes.fetch(:caller_action_id, nil), Integer, NilClass)
 
-      getter =-> key, required do
+      getter = ->key, required do
         required ? attributes.fetch(key) : attributes.fetch(key, {})
       end
 
@@ -236,10 +236,10 @@ module Dynflow
     # returned actions are in Present phase
     def planned_actions(filter = Action)
       phase! Present
-      plan_step.
-          planned_steps(execution_plan).
-          map { |s| s.action(execution_plan) }.
-          select { |a| a.is_a?(filter) }
+      plan_step
+        .planned_steps(execution_plan)
+        .map { |s| s.action(execution_plan) }
+        .select { |a| a.is_a?(filter) }
     end
 
     # @param [Class] filter_class return only actions which are kind of `filter_class`
@@ -248,8 +248,8 @@ module Dynflow
     def all_planned_actions(filter_class = Action)
       phase! Present
       mine = planned_actions
-      (mine + mine.reduce([]) { |arr, action| arr + action.all_planned_actions }).
-          select { |a| a.is_a?(filter_class) }
+      (mine + mine.reduce([]) { |arr, action| arr + action.all_planned_actions })
+        .select { |a| a.is_a?(filter_class) }
     end
 
     def run_step
@@ -268,18 +268,19 @@ module Dynflow
 
     def to_hash
       recursive_to_hash(
-          { class:                     self.class.name,
-            execution_plan_id:         execution_plan_id,
-            id:                        id,
-            plan_step_id:              plan_step_id,
-            run_step_id:               run_step_id,
-            finalize_step_id:          finalize_step_id,
-            caller_execution_plan_id:  caller_execution_plan_id,
-            caller_action_id:          caller_action_id,
-            input:                     input },
-          if phase? Run, Finalize, Present
-            { output: output }
-          end)
+        { class:                     self.class.name,
+          execution_plan_id:         execution_plan_id,
+          id:                        id,
+          plan_step_id:              plan_step_id,
+          run_step_id:               run_step_id,
+          finalize_step_id:          finalize_step_id,
+          caller_execution_plan_id:  caller_execution_plan_id,
+          caller_action_id:          caller_action_id,
+          input:                     input },
+        if phase? Run, Finalize, Present
+          { output: output }
+        end
+      )
     end
 
     def state
@@ -307,7 +308,7 @@ module Dynflow
     # @return [Array<Integer>] - ids of steps referenced from action
     def required_step_ids(input = self.input)
       results   = []
-      recursion =-> value do
+      recursion = ->value do
         case value
         when Hash
           value.values.each { |v| recursion.(v) }
@@ -364,9 +365,9 @@ module Dynflow
     def state=(state)
       phase! Executable
       @world.logger.debug format('%13s %s:%2d %9s >> %9s in phase %8s %s',
-                                 'Step', execution_plan_id, @step.id,
-                                 self.state, state,
-                                 phase.to_s_humanized, self.class)
+        'Step', execution_plan_id, @step.id,
+        self.state, state,
+        phase.to_s_humanized, self.class)
       @step.state = state
     end
 
@@ -502,6 +503,7 @@ module Dynflow
       when :skipping
         self.state = :skipped
       when :suspended, :error
+        # Do nothing
       else
         raise "wrong state #{self.state}"
       end
@@ -551,11 +553,10 @@ module Dynflow
     end
 
     # TODO: This is getting out of hand, refactoring needed
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def execute_run(event)
       phase! Run
       @world.logger.debug format('%13s %s:%2d got event %s',
-                                 'Step', execution_plan_id, @step.id, event) if event
+        'Step', execution_plan_id, @step.id, event) if event
 
       case
       when state == :running
@@ -599,7 +600,6 @@ module Dynflow
         raise "wrong state #{state} when event:#{event}"
       end
     end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     def execute_finalize
       phase! Finalize

data/lib/dynflow/active_job/queue_adapter.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module ActiveJob
     module QueueAdapters
@@ -13,7 +14,7 @@ module Dynflow
         def enqueue_at(job, timestamp)
           job.provider_job_id = job.job_id
           ::Rails.application.dynflow.world
-            .delay_with_options(id: job.provider_job_id,
+                 .delay_with_options(id: job.provider_job_id,
                                 action_class: JobWrapper,
                                 delay_options: { :start_at => Time.at(timestamp) },
                                 args: [job.serialize])

data/lib/dynflow/actor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
-module Dynflow
 
+module Dynflow
   FULL_BACKTRACE = %w[1 y yes].include?((ENV['DYNFLOW_FULL_BACKTRACE'] || '').downcase)
   BACKTRACE_LIMIT = begin
                       limit = ENV['DYNFLOW_BACKTRACE_LIMIT'].to_i
@@ -94,7 +94,7 @@ module Dynflow
         # takes an array of backtrace lines and replaces each chunk
         def filter_backtrace(backtrace)
           trace = backtrace.map { |line| filter_line(line) }
-                           .chunk_while { |l1, l2| l1 == l2}
+                           .chunk_while { |l1, l2| l1 == l2 }
                            .map(&:first)
           if BACKTRACE_LIMIT
             count = trace.count

data/lib/dynflow/actors/execution_plan_cleaner.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Actors
     class ExecutionPlanCleaner

data/lib/dynflow/actors.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Actors
     require 'dynflow/actors/execution_plan_cleaner'

data/lib/dynflow/clock.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
+
 module Dynflow
   class Clock < Actor
-
     include Algebrick::Types
 
     Timer = Algebrick.type do
@@ -13,7 +13,7 @@ module Dynflow
 
     module Timer
       def self.[](*fields)
-        super(*fields).tap { |v| Match! v.who, -> who { who.respond_to? v.where } }
+        super(*fields).tap { |v| Match! v.who, ->who { who.respond_to? v.where } }
       end
 
       include Comparable
@@ -116,7 +116,7 @@ module Dynflow
 
     def ping(who, time, with_what = nil, where = :<<, optional: false)
       Type! time, Time, Numeric
-      time  = current_time + time if time.is_a? Numeric
+      time = current_time + time if time.is_a? Numeric
       if who.is_a?(Action::Suspended)
         who.plan_event(with_what, time, optional: optional)
       else
@@ -125,5 +125,4 @@ module Dynflow
       end
     end
   end
-
 end

data/lib/dynflow/config.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'socket'
 
 module Dynflow
@@ -6,11 +7,11 @@ module Dynflow
     include Algebrick::TypeCheck
 
     def self.config_attr(name, *types, &default)
-      self.send(:define_method, "validate_#{ name }!") do |value|
+      self.send(:define_method, "validate_#{name}!") do |value|
         Type! value, *types unless types.empty?
       end
       self.send(:define_method, name) do
-        var_name = "@#{ name }"
+        var_name = "@#{name}"
         if instance_variable_defined?(var_name)
           return instance_variable_get(var_name)
         else
@@ -41,7 +42,7 @@ module Dynflow
         return @cache[name] if @cache.key?(name)
         value = @config.send(name)
         value = value.call(@world, self) if value.is_a? Proc
-        validation_method = "validate_#{ name }!"
+        validation_method = "validate_#{name}!"
         @config.send(validation_method, value) if @config.respond_to?(validation_method)
         @cache[name] = value
       end
@@ -51,7 +52,7 @@ module Dynflow
       attr_reader :queues
 
       def initialize
-        @queues = {:default => {}}
+        @queues = { :default => {} }
       end
 
       # Add a new queue to the configuration
@@ -204,7 +205,7 @@ module Dynflow
                                                "it's #{ar_pool_size} but there is #{config_for_world.pool_size} " +
                                                'threads in Dynflow pool.'
           end
-        rescue ActiveRecord::ConnectionNotEstablished # rubocop:disable Lint/HandleExceptions
+        rescue ActiveRecord::ConnectionNotEstablished
           # If in tests or in an environment where ActiveRecord doesn't have a
           # real DB connection, we want to skip AR configuration altogether
         end

data/lib/dynflow/connectors/abstract.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Connectors
     class Abstract
@@ -35,16 +36,16 @@ module Dynflow
         Type! envelope, Dispatcher::Envelope
         Telemetry.with_instance { |t| t.increment_counter(:dynflow_connector_envelopes, 1, :world => world.id, :direction => 'incoming') }
         match(envelope.message,
-              (on Dispatcher::Ping do
-                 response_envelope = envelope.build_response_envelope(Dispatcher::Pong, world)
-                 send(response_envelope)
-               end),
-              (on Dispatcher::Request do
-                 world.executor_dispatcher.tell([:handle_request, envelope])
-               end),
-              (on Dispatcher::Response do
-                 world.client_dispatcher.tell([:dispatch_response, envelope])
-               end))
+          (on Dispatcher::Ping do
+             response_envelope = envelope.build_response_envelope(Dispatcher::Pong, world)
+             send(response_envelope)
+           end),
+          (on Dispatcher::Request do
+             world.executor_dispatcher.tell([:handle_request, envelope])
+           end),
+          (on Dispatcher::Response do
+             world.client_dispatcher.tell([:dispatch_response, envelope])
+           end))
       end
     end
   end

data/lib/dynflow/connectors/database.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Connectors
     class Database < Abstract
-
       class PostgresListerner
         def initialize(core, world_id, db)
           @core     = core
@@ -22,7 +22,7 @@ module Dynflow
         def start
           @started.set true
           @thread = Thread.new do
-            @db.listen("world:#{ @world_id }", :loop => true) do
+            @db.listen("world:#{@world_id}", :loop => true) do
               if started?
                 @core << :check_inbox
               else

data/lib/dynflow/connectors/direct.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Connectors
     class Direct < Abstract
-
       class Core < Actor
-
         def initialize(connector)
           @connector = connector
           @worlds = {}
@@ -30,7 +29,7 @@ module Dynflow
           if world = find_receiver(envelope)
             @connector.receive(world, envelope)
           else
-            log(Logger::ERROR, "Receiver for envelope #{ envelope } not found")
+            log(Logger::ERROR, "Receiver for envelope #{envelope} not found")
           end
         end
 

data/lib/dynflow/connectors.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module Connectors
     require 'dynflow/connectors/abstract'

data/lib/dynflow/coordinator.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
+
 require 'dynflow/coordinator_adapters'
 
 module Dynflow
   class Coordinator
-
     include Algebrick::TypeCheck
 
     class DuplicateRecordError < Dynflow::Error
@@ -41,7 +41,7 @@ module Dynflow
       end
 
       def initialize(*args)
-        @data ||= {}
+        @data = {}
         @data = Utils.indifferent_hash(@data.merge(class: self.class.name))
       end
 
@@ -146,8 +146,8 @@ module Dynflow
         Type! owner_id, String
       end
 
-      def to_s
-        "#{self.class.name}: #{id} by #{owner_id}"
+      def unlock_on_shutdown?
+        true
       end
     end
 
@@ -293,6 +293,10 @@ module Dynflow
       def request_id
         @data[:request_id]
       end
+
+      def unlock_on_shutdown?
+        false
+      end
     end
 
     class PlanningLock < LockByWorld
@@ -309,6 +313,10 @@ module Dynflow
       def execution_plan_id
         @data[:execution_plan_id]
       end
+
+      def unlock_on_shutdown?
+        false
+      end
     end
 
     attr_reader :adapter
@@ -324,8 +332,12 @@ module Dynflow
       if block
         begin
           block.call
+        # We are looking for ::Sidekiq::Shutdown, but that may not be defined. We rely on it being a subclass of Interrupt
+        # We don't really want to rescue it, but we need to bind it somehow so that we can check it in ensure
+        rescue Interrupt => e
+          raise e
         ensure
-          release(lock)
+          release(lock) if !(defined?(::Sidekiq) && e.is_a?(::Sidekiq::Shutdown)) || lock.unlock_on_shutdown?
         end
       end
     rescue DuplicateRecordError => e
@@ -337,8 +349,8 @@ module Dynflow
       adapter.delete_record(lock)
     end
 
-    def release_by_owner(owner_id)
-      find_locks(owner_id: owner_id).map { |lock| release(lock) }
+    def release_by_owner(owner_id, on_termination = false)
+      find_locks(owner_id: owner_id).map { |lock| release(lock) if !on_termination || lock.unlock_on_shutdown? }
     end
 
     def find_locks(filter_options)
@@ -383,9 +395,9 @@ module Dynflow
       create_record(world)
     end
 
-    def delete_world(world)
+    def delete_world(world, on_termination = false)
       Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
-      release_by_owner("world:#{world.id}")
+      release_by_owner("world:#{world.id}", on_termination)
       delete_record(world)
     end
 

data/lib/dynflow/coordinator_adapters/abstract.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Dynflow
   module CoordinatorAdapters
     class Abstract