dynflow 1.3.0 → 1.4.0

data/examples/termination.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require_relative 'example_helper'
 

data/lib/dynflow.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'apipie-params'
 require 'algebrick'
 require 'thread'
@@ -17,8 +18,27 @@ end
 # TODO profiling, find bottlenecks
 # FIND change ids to uuid, uuid-<action_id>, uuid-<action_id-(plan, run, finalize)
 module Dynflow
+  class << self
+    # Return the world that representing this process - this is mainly used by
+    # Sidekiq deployments, where there is a need for a global-level context.
+    #
+    # @return [Dynflow::World, nil]
+    def process_world
+      return @process_world if defined? @process_world
+      @process_world = Sidekiq.options[:dynflow_world]
+      raise "process world is not set" unless @process_world
+      @process_world
+    end
+  end
 
   class Error < StandardError
+    def to_hash
+      { class: self.class.name, message: message, backtrace: backtrace }
+    end
+
+    def self.from_hash(hash)
+      self.new(hash[:message]).tap { |e| e.set_backtrace(hash[:backtrace]) }
+    end
   end
 
   require 'dynflow/utils'

data/lib/dynflow/action.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   # rubocop:disable Metrics/ClassLength
   class Action < Serializable
@@ -88,6 +89,13 @@ module Dynflow
       end
     end
 
+    DelayedEvent = Algebrick.type do
+      fields! execution_plan_id: String,
+              step_id:           Integer,
+              event:             Object,
+              time:              type { variants Time, NilClass }
+    end
+
     def self.constantize(action_name)
       super action_name
     rescue NameError
@@ -322,6 +330,17 @@ module Dynflow
     def queue
     end
 
+    # Plan an +event+ to be send to the action defined by +action+, what defaults to be self.
+    # if +time+ is not passed, event is sent as soon as possible.
+    def plan_event(event, time = nil, execution_plan_id: self.execution_plan_id, step_id: self.run_step_id)
+      time = @world.clock.current_time + time if time.is_a?(Numeric)
+      delayed_events << DelayedEvent[execution_plan_id, step_id, event, time]
+    end
+
+    def delayed_events
+      @delayed_events ||= []
+    end
+
     protected
 
     def state=(state)
@@ -333,9 +352,15 @@ module Dynflow
       @step.state = state
     end
 
-    def save_state
+    def save_state(conditions = {})
       phase! Executable
-      @step.save
+      # If this save returns an integer, it means it was an update. The number
+      #   represents the number of updated records. If it is 0, then the step
+      #   was in an unexpected state and couldn't be updated, in which case we
+      #   raise an exception and crash hard to prevent the step from being
+      #   executed twice
+      count = @step.save(conditions)
+      raise 'Could not save state' if count.kind_of?(Integer) && !count.positive?
     end
 
     def delay(delay_options, *args)
@@ -527,7 +552,7 @@ module Dynflow
         end
 
         self.state = :running unless self.state == :skipping
-        save_state
+        save_state(:state => %w(pending error skipping suspended))
         with_error_handling do
           event = Skip if state == :skipping
 

data/lib/dynflow/action/cancellable.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::Cancellable
     Cancel = Algebrick.atom

data/lib/dynflow/action/format.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
 
   # Input/output format validation logic calling

data/lib/dynflow/action/missing.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   # for cases the serialized action was renamed and it's not available
   # in the code base anymore.

data/lib/dynflow/action/polling.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'dynflow/action/timeouts'
 
 module Dynflow
@@ -90,7 +91,8 @@ module Dynflow
     end
 
     def suspend_and_ping
-      suspend { |suspended_action| world.clock.ping suspended_action, poll_interval, Poll }
+      plan_event(Poll, poll_interval)
+      suspend
     end
 
     def poll_external_task_with_rescue

data/lib/dynflow/action/progress.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
 
   # Methods for specifying the progress of the action

data/lib/dynflow/action/rescue.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::Rescue
 

data/lib/dynflow/action/singleton.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Action
     module Singleton

data/lib/dynflow/action/suspended.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   class Action::Suspended
     attr_reader :execution_plan_id, :step_id
@@ -8,8 +9,14 @@ module Dynflow
       @step_id           = action.run_step_id
     end
 
-    def event(event, future = Concurrent::Promises.resolvable_future)
-      @world.event execution_plan_id, step_id, event, future
+    def plan_event(event, time, sent = Concurrent::Promises.resolvable_future)
+      @world.plan_event(execution_plan_id, step_id, event, time, sent)
+    end
+
+    def event(event, sent = Concurrent::Promises.resolvable_future)
+      # TODO: deprecate 2 levels backtrace (to know it's called from clock or internaly)
+      # remove lib/dynflow/clock.rb ClockReference#ping branch condition on removal.
+      plan_event(event, nil, sent)
     end
 
     def <<(event = nil)

data/lib/dynflow/action/timeouts.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::Timeouts
     Timeout = Algebrick.atom
@@ -7,7 +8,7 @@ module Dynflow
     end
 
     def schedule_timeout(seconds)
-      world.clock.ping suspended_action, seconds, Timeout
+      plan_event(Timeout, seconds)
     end
  end
 end

data/lib/dynflow/action/with_bulk_sub_plans.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::WithBulkSubPlans
     include Dynflow::Action::Cancellable
@@ -69,7 +70,7 @@ module Dynflow
     def spawn_plans
       super
     ensure
-      suspended_action << PlanNextBatch
+      plan_event(PlanNextBatch)
     end
 
     def cancel!(force = false)

data/lib/dynflow/action/with_polling_sub_plans.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::WithPollingSubPlans
 
@@ -19,7 +20,7 @@ module Dynflow
     end
 
     def initiate
-      ping suspended_action
+      ping
       super
     end
 
@@ -38,7 +39,7 @@ module Dynflow
       else
         if self.is_a?(::Dynflow::Action::WithBulkSubPlans) && can_spawn_next_batch?
           # Not everything was spawned
-          ping suspended_action
+          ping
           spawn_plans
           suspend
         else
@@ -52,11 +53,12 @@ module Dynflow
     end
 
     def suspend_and_ping
-      suspend { |suspended_action| ping suspended_action }
+      ping
+      suspend
     end
 
-    def ping(suspended_action)
-      world.clock.ping suspended_action, REFRESH_INTERVAL, Poll
+    def ping(_suspended_action = nil)
+      plan_event(Poll, REFRESH_INTERVAL)
     end
 
     def recalculate_counts

data/lib/dynflow/action/with_sub_plans.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Action::WithSubPlans
     include Dynflow::Action::Cancellable

data/lib/dynflow/active_job/queue_adapter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module ActiveJob
     module QueueAdapters

data/lib/dynflow/actor.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
 
   module MethodicActor
@@ -48,7 +49,8 @@ module Dynflow
     end
 
     class BacktraceCollector
-      CONCURRENT_RUBY_LINE = '[ concurrent-ruby ]'.freeze
+      CONCURRENT_RUBY_LINE = '[ concurrent-ruby ]'
+      SIDEKIQ_LINE = '[ sidekiq ]'
 
       class << self
         def with_backtrace(backtrace)
@@ -69,14 +71,20 @@ module Dynflow
 
         private
 
-        def filter_line?(line)
-          %w[concurrent-ruby gems/logging actor.rb].any? { |pattern| line.include?(pattern) }
+        def filter_line(line)
+          if %w[concurrent-ruby gems/logging actor.rb].any? { |pattern| line.include?(pattern) }
+            CONCURRENT_RUBY_LINE
+          elsif line.include?('lib/sidekiq')
+            SIDEKIQ_LINE
+          else
+            line
+          end
         end
 
         # takes an array of backtrace lines and replaces each chunk
         def filter_backtrace(backtrace)
-          backtrace.map { |line| filter_line?(line) ? CONCURRENT_RUBY_LINE : line }
-            .chunk_while { |l1, l2| l1.equal?(CONCURRENT_RUBY_LINE) && l2.equal?(CONCURRENT_RUBY_LINE) }
+          backtrace.map { |line| filter_line(line) }
+            .chunk_while { |l1, l2| l1 == l2}
             .map(&:first)
         end
       end

data/lib/dynflow/actors.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Actors
     require 'dynflow/actors/execution_plan_cleaner'

data/lib/dynflow/actors/execution_plan_cleaner.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Dynflow
   module Actors
     class ExecutionPlanCleaner

data/lib/dynflow/clock.rb

@@ -1,9 +1,9 @@
+# frozen_string_literal: true
 module Dynflow
   class Clock < Actor
 
     include Algebrick::Types
 
-    Tick  = Algebrick.atom
     Timer = Algebrick.type do
       fields! who:   Object, # to ping back
               when:  Time, # to deliver
@@ -40,19 +40,9 @@ module Dynflow
       end
     end
 
-    Pills = Algebrick.type do
-      variants None = atom,
-               Took = atom,
-               Pill = type { fields Float }
-    end
-
     def initialize(logger = nil)
-      @logger        = logger
-      @timers        = Utils::PriorityQueue.new { |a, b| b <=> a }
-      @sleeping_pill = None
-      @sleep_barrier = Mutex.new
-      @sleeper       = Thread.new { sleeping }
-      Thread.pass until @sleep_barrier.locked? || @sleeper.status == 'sleep'
+      @logger = logger
+      @timers = Utils::PriorityQueue.new { |a, b| b <=> a }
     end
 
     def default_reference_class
@@ -60,9 +50,7 @@ module Dynflow
     end
 
     def on_event(event)
-      if event == :terminated
-        @sleeper.kill
-      end
+      wakeup if event == :terminated
     end
 
     def tick
@@ -97,52 +85,44 @@ module Dynflow
     end
 
     def wakeup
-      while @sleep_barrier.synchronize { Pill === @sleeping_pill }
-        Thread.pass
-      end
-      @sleep_barrier.synchronize do
-        @sleeper.wakeup if Took === @sleeping_pill
+      if @timer
+        @timer.cancel
+        tick unless terminating?
       end
     end
 
     def sleep_to(timer)
       return unless timer
-      sec = [timer.when - Time.now, 0.0].max
-      @sleep_barrier.synchronize do
-        @sleeping_pill = Pill[sec]
-        @sleeper.wakeup
-      end
+      schedule(timer.when - Time.now) { reference.tell(:tick) unless terminating? }
+      nil
     end
 
-    def sleeping
-      @sleep_barrier.synchronize do
-        loop do
-          @sleeping_pill = None
-          @sleep_barrier.sleep
-          pill           = @sleeping_pill
-          @sleeping_pill = Took
-          @sleep_barrier.sleep pill.value
-          reference.tell(:tick)
-        end
-      end
+    def schedule(delay, &block)
+      @timer = if delay.positive?
+                 Concurrent::ScheduledTask.execute(delay, &block)
+               else
+                 yield
+                 nil
+               end
     end
   end
 
   class ClockReference < Concurrent::Actor::Reference
     include Algebrick::Types
 
+    def current_time
+      Time.now
+    end
+
     def ping(who, time, with_what = nil, where = :<<)
       Type! time, Time, Numeric
-      time  = Time.now + time if time.is_a? Numeric
-      timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
-      # if self.ask!(:terminated?) # FIXME not thread safe
-      #   Thread.new do
-      #     sleep [timer.when - Time.now, 0].max
-      #     timer.apply
-      #   end
-      # else
-      self.tell([:add_timer, timer])
-      # end
+      time  = current_time + time if time.is_a? Numeric
+      if who.is_a?(Action::Suspended)
+        who.plan_event(with_what, time)
+      else
+        timer = Clock::Timer[who, time, with_what.nil? ? Algebrick::Types::None : Some[Object][with_what], where]
+        self.tell([:add_timer, timer])
+      end
     end
   end
 

data/lib/dynflow/config.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'socket'
 
 module Dynflow
@@ -97,8 +98,16 @@ module Dynflow
       5
     end
 
-    config_attr :executor, Executors::Abstract, FalseClass do |world, config|
-      Executors::Parallel.new(world, config.executor_heartbeat_interval, config.queues)
+    config_attr :executor do |world, config|
+      Executors::Parallel::Core
+    end
+
+    def validate_executor!(value)
+      accepted_executors = [Executors::Parallel::Core]
+      accepted_executors << Executors::Sidekiq::Core if defined? Executors::Sidekiq::Core
+      if value && !accepted_executors.include?(value)
+        raise ArgumentError, "Executor #{value} is expected to be one of #{accepted_executors.inspect}"
+      end
     end
 
     config_attr :executor_semaphore, Semaphores::Abstract, FalseClass do |world, config|