dynflow 1.4.5 → 1.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 531c1207ec6665f5773b2e71887e79d0259c4edd068d976f655766b5646ca9b0
-  data.tar.gz: ae88bf9434ebc2b7672751e24b95a60d1c4a38c1ff9396e68342c0529b68022c
+  metadata.gz: 4f677a2e5d7f119264258b7b36fe187203db60b8b4df4136beb73954857d59bd
+  data.tar.gz: a7cd75185cc99ce4a1e3e18e07565542ebf24bec8109a3291e78c56345bbc0db
 SHA512:
-  metadata.gz: 3f2f5f4ccdc36b9bc0439fc0a864412126877f4da6f01eaa00b6bcadf6af05d0467d1eb1e6fde002ee35d962ea04f3c46df0e369b32e17c56fca9e26450ab1be
-  data.tar.gz: ac3c24d8f31492c3d053877a03162e54c0c2f2b7ebe3673cb4cb6134f2cc0ea83be6f54ea4251d4f801e2dacc4f90c6cfe16b2ffd818e0f46a573da2fe054189
+  metadata.gz: c00baa2c020df6b035aa5a45aaa1b58a08ccc310752f4de0ab54da0207a03439cfde09132a7e8a6f4fe7911058a027e87f5cdb7b2563da035725ba5b482d0ab4
+  data.tar.gz: 4189a3f752642000d4eda05a8d82013d350eb77f7325450dfa01e774672aa33c61f3b32a1c8d5eb75e030d2fae1cb42a1e92da3278278569ba250245b06c3aec

data/lib/dynflow/action.rb

@@ -93,7 +93,8 @@ module Dynflow
       fields! execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              time:              type { variants Time, NilClass }
+              time:              type { variants Time, NilClass },
+              optional:          Algebrick::Types::Boolean
     end
 
     def self.constantize(action_name)
@@ -332,9 +333,9 @@ module Dynflow
 
     # Plan an +event+ to be send to the action defined by +action+, what defaults to be self.
     # if +time+ is not passed, event is sent as soon as possible.
-    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id)
+    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id, optional: false)
       time = @world.clock.current_time + time if time.is_a?(Numeric)
-      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time]
+      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time, optional]
     end
 
     def delayed_events
@@ -352,15 +353,12 @@ module Dynflow
       @step.state = state
     end
 
+    # If this save returns an integer, it means it was an update. The number
+    # represents the number of updated records. If it is 0, then the step was in
+    # an unexpected state and couldn't be updated
     def save_state(conditions = {})
       phase! Executable
-      # If this save returns an integer, it means it was an update. The number
-      #   represents the number of updated records. If it is 0, then the step
-      #   was in an unexpected state and couldn't be updated, in which case we
-      #   raise an exception and crash hard to prevent the step from being
-      #   executed twice
-      count = @step.save(conditions)
-      raise 'Could not save state' if count.kind_of?(Integer) && !count.positive?
+      @step.save(conditions)
     end
 
     def delay(delay_options, *args)
@@ -536,11 +534,11 @@ module Dynflow
     end
 
     # TODO: This is getting out of hand, refactoring needed
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def execute_run(event)
       phase! Run
       @world.logger.debug format('%13s %s:%2d got event %s',
                                  'Step', execution_plan_id, @step.id, event) if event
-      @input = OutputReference.dereference @input, world.persistence
 
       case
       when state == :running
@@ -551,8 +549,19 @@ module Dynflow
           raise 'event can be processed only when in suspended state'
         end
 
+        old_state = self.state
         self.state = :running unless self.state == :skipping
-        save_state(:state => %w(pending error skipping suspended))
+        saved = save_state(:state => %w(pending error skipping suspended))
+        if saved.kind_of?(Integer) && !saved.positive?
+          # The step was already in a state we're trying to transition to, most
+          # likely we were about to execute it for the second time after first
+          # execution was forcefully interrupted.
+          # Set error and return to prevent the step from being executed twice
+          set_error "Could not transition step from #{old_state} to #{self.state}, step already in #{self.state}."
+          return
+        end
+
+        @input = OutputReference.dereference @input, world.persistence
         with_error_handling do
           event = Skip if state == :skipping
 
@@ -573,6 +582,7 @@ module Dynflow
         raise "wrong state #{state} when event:#{event}"
       end
     end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     def execute_finalize
       phase! Finalize

data/lib/dynflow/action/suspended.rb

@@ -9,14 +9,14 @@ module Dynflow
       @step_id           = action.run_step_id
     end
 
-    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future)
-      @world.plan_event(execution_plan_id, step_id, event, time, sent)
+    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future, optional: false)
+      @world.plan_event(execution_plan_id, step_id, event, time, sent, optional: optional)
     end
 
-    def event(event, sent = Concurrent::Promises.resolvable_future)
+    def event(event, sent = Concurrent::Promises.resolvable_future, optional: false)
       # TODO: deprecate 2 levels backtrace (to know it's called from clock or internaly)
       # remove lib/dynflow/clock.rb ClockReference#ping branch condition on removal.
-      plan_event(event, nil, sent)
+      plan_event(event, nil, sent, optional: optional)
     end
 
     def <<(event = nil)

data/lib/dynflow/action/timeouts.rb

@@ -7,8 +7,8 @@ module Dynflow
       fail("Timeout exceeded.")
     end
 
-    def schedule_timeout(seconds)
-      plan_event(Timeout, seconds)
+    def schedule_timeout(seconds, optional: false)
+      plan_event(Timeout, seconds, optional: optional)
     end
  end
 end

data/lib/dynflow/actor.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 module Dynflow
 
+  FULL_BACKTRACE = %w[1 y yes].include?((ENV['DYNFLOW_FULL_BACKTRACE'] || '').downcase)
+  BACKTRACE_LIMIT = begin
+                      limit = ENV['DYNFLOW_BACKTRACE_LIMIT'].to_i
+                      limit.zero? ? nil : limit
+                    end
+
   module MethodicActor
     def on_message(message)
       method, *args = message
@@ -44,7 +50,11 @@ module Dynflow
       include LogWithFullBacktrace
 
       def on_envelope(envelope)
-        Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        if FULL_BACKTRACE
+          Actor::BacktraceCollector.with_backtrace(envelope.origin_backtrace) { super }
+        else
+          super
+        end
       end
     end
 
@@ -83,9 +93,15 @@ module Dynflow
 
         # takes an array of backtrace lines and replaces each chunk
         def filter_backtrace(backtrace)
-          backtrace.map { |line| filter_line(line) }
-            .chunk_while { |l1, l2| l1 == l2}
-            .map(&:first)
+          trace = backtrace.map { |line| filter_line(line) }
+                           .chunk_while { |l1, l2| l1 == l2}
+                           .map(&:first)
+          if BACKTRACE_LIMIT
+            count = trace.count
+            trace = trace.take(BACKTRACE_LIMIT)
+            trace << "[ backtrace omitted #{count - BACKTRACE_LIMIT} lines ]" if trace.count < count
+          end
+          trace
         end
       end
     end

data/lib/dynflow/clock.rb

@@ -114,11 +114,11 @@ module Dynflow
       Time.now
     end
 
-    def ping(who, time, with_what = nil, where = :<<)
+    def ping(who, time, with_what = nil, where = :<<, optional: false)
       Type! time, Time, Numeric
       time  = current_time + time if time.is_a? Numeric
       if who.is_a?(Action::Suspended)
-        who.plan_event(with_what, time)
+        who.plan_event(with_what, time, optional: optional)
       else
         timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
         self.tell([:add_timer, timer])

data/lib/dynflow/director.rb

@@ -15,7 +15,8 @@ module Dynflow
               execution_plan_id: String,
               step_id:           Integer,
               event:             Object,
-              result:            Concurrent::Promises::ResolvableFuture
+              result:            Concurrent::Promises::ResolvableFuture,
+              optional:          Algebrick::Types::Boolean
     end
 
     UnprocessableEvent = Class.new(Dynflow::Error)
@@ -163,6 +164,9 @@ module Dynflow
       execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
       if execution_plan_manager
         execution_plan_manager.event(event)
+      elsif event.optional
+        event.result.reject "no manager for #{event.inspect}"
+        []
       else
         raise Dynflow::Error, "no manager for #{event.inspect}"
       end

data/lib/dynflow/director/running_steps_manager.rb

@@ -20,8 +20,8 @@ module Dynflow
       def terminate
         pending_work = @work_items.clear.values.flatten(1)
         pending_work.each do |w|
-          if EventWorkItem === w
-            w.event.result.reject UnprocessableEvent.new("dropping due to termination")
+          finish_event_result(w) do |result|
+            result.reject UnprocessableEvent.new("dropping due to termination")
           end
         end
       end

data/lib/dynflow/dispatcher.rb

@@ -6,7 +6,8 @@ module Dynflow
         fields! execution_plan_id: String,
                 step_id:           Integer,
                 event:             Object,
-                time:              type { variants Time, NilClass }
+                time:              type { variants Time, NilClass },
+                optional:          Algebrick::Types::Boolean
       end
 
       Execution = type do

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -132,11 +132,13 @@ module Dynflow
       end
 
       def dispatch_request(request, client_world_id, request_id)
+        ignore_unknown = false
         executor_id = match request,
                             (on ~Execution do |execution|
                                AnyExecutor
                              end),
                             (on ~Event do |event|
+                               ignore_unknown = event.optional
                                find_executor(event.execution_plan_id)
                              end),
                             (on Ping.(~any, ~any) | Status.(~any, ~any) do |receiver_id, _|
@@ -144,7 +146,11 @@ module Dynflow
                              end)
         envelope = Envelope[request_id, client_world_id, executor_id, request]
         if Dispatcher::UnknownWorld === envelope.receiver_id
-          raise Dynflow::Error, "Could not find an executor for #{envelope}"
+          raise Dynflow::Error, "Could not find an executor for #{envelope}" unless ignore_unknown
+
+          message = "Could not find an executor for optional #{envelope}, discarding."
+          log(Logger::DEBUG, message)
+          return respond(envelope, Failed[message])
         end
         connector.send(envelope).value!
       rescue => e
@@ -252,7 +258,7 @@ module Dynflow
           future.fulfill(true)
         else
           if @ping_cache.executor?(request.receiver_id)
-            future.reject
+            future.reject false
           else
             yield
           end

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -52,12 +52,14 @@ module Dynflow
           end
         end
         if event_request.time.nil? || event_request.time < Time.now
-          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future)
+          @world.executor.event(envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, future,
+                                optional: event_request.optional)
         else
           @world.clock.ping(
             @world.executor,
             event_request.time,
-            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future],
+            Director::Event[envelope.request_id, event_request.execution_plan_id, event_request.step_id, event_request.event, Concurrent::Promises.resolvable_future,
+                            event_request.optional],
             :delayed_event
           )
           # resolves the future right away - currently we do not wait for the clock ping

data/lib/dynflow/executors/abstract/core.rb

@@ -37,7 +37,7 @@ module Dynflow
 
         def plan_events(delayed_events)
           delayed_events.each do |event|
-            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time)
+            @world.plan_event(event.execution_plan_id, event.step_id, event.event, event.time, optional: event.optional)
           end
         end
 

data/lib/dynflow/executors/parallel.rb

@@ -33,8 +33,8 @@ module Dynflow
         raise e
       end
 
-      def event(request_id, execution_plan_id, step_id, event, future = nil)
-        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future]])
+      def event(request_id, execution_plan_id, step_id, event, future = nil, optional: false)
+        @core.ask([:handle_event, Director::Event[request_id, execution_plan_id, step_id, event, future, optional]])
         future
       end
 

data/lib/dynflow/rails.rb

@@ -38,8 +38,8 @@ module Dynflow
       init_world.tap do |world|
         @world = world
         config.run_on_init_hooks(false, world)
+        config.increase_db_pool_size(world)
         unless config.remote?
-          config.increase_db_pool_size(world)
           config.run_on_init_hooks(true, world)
           # leave this just for long-running executors
           unless config.rake_task_with_executor?

data/lib/dynflow/rails/configuration.rb

@@ -96,7 +96,11 @@ module Dynflow
       end
 
       def increase_db_pool_size?
-        !::Rails.env.test? && !remote?
+        !::Rails.env.test? && (!remote? || sidekiq_worker?)
+      end
+
+      def sidekiq_worker?
+        defined?(::Sidekiq) && ::Sidekiq.options[:queues].any?
       end
 
       def calculate_db_pool_size(world)

data/lib/dynflow/testing/in_thread_executor.rb

@@ -34,8 +34,8 @@ module Dynflow
         @director.work_finished(work_item)
       end
 
-      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future)
-        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future])
+      def event(execution_plan_id, step_id, event, future = Concurrent::Promises.resolvable_future, optional: false)
+        event = (Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, future, optional])
         @director.handle_event(event).each do |work_item|
           @work_items << work_item
         end

data/lib/dynflow/testing/in_thread_world.rb

@@ -58,15 +58,15 @@ module Dynflow
         future.reject e
       end
 
-      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-        @executor.event(execution_plan_id, step_id, event, done)
+      def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+        @executor.event(execution_plan_id, step_id, event, done, optional: optional)
       end
 
-      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future)
+      def plan_event(execution_plan_id, step_id, event, time, done = Concurrent::Promises.resolvable_future, optional: false)
         if time.nil? || time < Time.now
-          event(execution_plan_id, step_id, event, done)
+          event(execution_plan_id, step_id, event, done, optional: optional)
         else
-          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done], :delayed_event)
+          clock.ping(executor, time, Director::Event[SecureRandom.uuid, execution_plan_id, step_id, event, done, optional], :delayed_event)
         end
       end
     end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.4.5'
+  VERSION = '1.4.9'
 end

data/lib/dynflow/world.rb

@@ -219,12 +219,12 @@ module Dynflow
       publish_request(Dispatcher::Execution[execution_plan_id], done, true)
     end
 
-    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event], done, false)
+    def event(execution_plan_id, step_id, event, done = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, nil, optional], done, false)
     end
 
-    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future)
-      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time], accepted, false)
+    def plan_event(execution_plan_id, step_id, event, time, accepted = Concurrent::Promises.resolvable_future, optional: false)
+      publish_request(Dispatcher::Event[execution_plan_id, step_id, event, time, optional], accepted, false)
     end
 
     def ping(world_id, timeout, done = Concurrent::Promises.resolvable_future)
@@ -341,7 +341,7 @@ module Dynflow
         @terminating = Concurrent::Promises.future do
           termination_future.wait(termination_timeout)
         end.on_resolution do
-          @terminated.complete
+          @terminated.resolve
           Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end

data/lib/dynflow/world/invalidation.rb

@@ -43,7 +43,7 @@ module Dynflow
                   else
                     :stopped
                   end
-          plan.update_state(state)
+          plan.update_state(state) if plan.state != state
 
           coordinator.release(planning_lock)
           execute(plan.id) if plan.state == :planned

data/test/dispatcher_test.rb

@@ -49,6 +49,12 @@ module Dynflow
               plan = result.finished.value
               assert_equal('finish', plan.actions.first.output[:event])
             end
+
+            it 'does not error on dispatching an optional event' do
+              request = client_world.event('123', 1, nil, optional: true)
+              request.wait(20)
+              assert_match /Could not find an executor for optional .*, discarding/, request.reason.message
+            end
           end
         end
       end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.4.5
+  version: 1.4.9
 platform: ruby
 authors:
 - Ivan Necas
 - Petr Chalupa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-17 00:00:00.000000000 Z
+date: 2021-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -625,7 +625,7 @@ homepage: https://github.com/Dynflow/dynflow
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -640,8 +640,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: DYNamic workFLOW engine
 test_files: