dynflow 0.8.9 → 0.8.10

data/lib/dynflow/semaphores/dummy.rb

@@ -0,0 +1,32 @@
+module Dynflow
+  module Semaphores
+    class Dummy < Abstract
+
+      def wait(thing)
+        true
+      end
+
+      def get_waiting
+        nil
+      end
+
+      def has_waiting?
+        false
+      end
+
+      def release(*args)
+      end
+
+      def save
+      end
+
+      def get(n)
+        n
+      end
+
+      def free
+        1
+      end
+    end
+  end
+end

data/lib/dynflow/semaphores/stateful.rb

@@ -0,0 +1,70 @@
+module Dynflow
+  module Semaphores
+    class Stateful < Abstract
+
+      attr_reader :free, :tickets, :waiting, :meta
+
+      def initialize(tickets, free = tickets, meta = {})
+        @tickets = tickets
+        @free = free
+        @waiting = []
+        @meta = meta
+      end
+
+      def wait(thing)
+        if get > 0
+          true
+        else
+          @waiting << thing
+          false
+        end
+      end
+
+      def get_waiting
+        @waiting.shift
+      end
+
+      def has_waiting?
+        !@waiting.empty?
+      end
+
+      def release(n = 1)
+        @free += n
+        @free = @tickets unless @tickets.nil? || @free <= @tickets
+        save
+      end
+
+      def save
+      end
+
+      def get(n = 1)
+        if n > @free
+          drain
+        else
+          @free -= n
+          save
+          n
+        end
+      end
+
+      def drain
+        @free.tap do
+          @free = 0
+          save
+        end
+      end
+
+      def to_hash
+        {
+          :tickets => @tickets,
+          :free => @free,
+          :meta => @meta
+        }
+      end
+
+      def self.new_from_hash(hash)
+        self.new(*hash.values_at(:tickets, :free, :meta))
+      end
+    end
+  end
+end

data/lib/dynflow/throttle_limiter.rb

@@ -0,0 +1,135 @@
+module Dynflow
+  class ThrottleLimiter
+
+    attr_reader :core
+
+    def initialize(world)
+      @world = world
+      spawn
+    end
+
+    def handle_plans!(*args)
+      core.ask!([:handle_plans, *args])
+    end
+
+    def cancel!(plan_id)
+      core.tell([:cancel, plan_id])
+    end
+
+    def terminate
+      core.ask(:terminate!)
+    end
+
+    def observe(parent_id = nil)
+      core.ask!([:observe, parent_id])
+    end
+
+    def core_class
+      Core
+    end
+
+    private
+
+    def spawn
+      Concurrent.future.tap do |initialized|
+        @core = core_class.spawn(:name => 'throttle-limiter',
+                                 :args => [@world],
+                                 :initialized => initialized)
+      end
+    end
+
+    class Core < Actor
+      def initialize(world)
+        @world = world
+        @semaphores = {}
+      end
+
+      def handle_plans(parent_id, planned_ids, failed_ids, semaphores_hash)
+        @semaphores[parent_id] = create_semaphores(semaphores_hash)
+        set_up_clock_for(parent_id, true)
+
+        failed = failed_ids.map do |plan_id|
+          ::Dynflow::World::Triggered[plan_id, Concurrent.future].tap do |triggered|
+            execute_triggered(triggered)
+          end
+        end
+
+        planned_ids.map do |child_id|
+          ::Dynflow::World::Triggered[child_id, Concurrent.future].tap do |triggered|
+            triggered.future.on_completion! { self << [:release, parent_id] }
+            execute_triggered(triggered) if @semaphores[parent_id].wait(triggered)
+          end
+        end + failed
+      end
+
+      def observe(parent_id = nil)
+        if parent_id.nil?
+          @semaphores.reduce([]) do |acc, cur|
+            acc << { cur.first => cur.last.waiting }
+          end
+        elsif @semaphores.key? parent_id
+          @semaphores[parent_id].waiting
+        else
+          []
+        end
+      end
+
+      def release(plan_id, key = :level)
+        return unless @semaphores.key? plan_id
+        set_up_clock_for(plan_id) if key == :time
+        semaphore = @semaphores[plan_id]
+        semaphore.release(1, key) if semaphore.children.key?(key)
+        if semaphore.has_waiting? && semaphore.get == 1
+          execute_triggered(semaphore.get_waiting)
+        end
+        @semaphores.delete(plan_id) unless semaphore.has_waiting?
+      end
+
+      def cancel(parent_id, reason = nil)
+        if @semaphores.key?(parent_id)
+          reason ||= 'The task was cancelled.'
+          @semaphores[parent_id].waiting.each do |triggered|
+            cancel_plan_id(triggered.execution_plan_id, reason)
+            triggered.future.fail(reason)
+          end
+          @semaphores.delete(parent_id)
+        end
+      end
+
+      private
+
+      def cancel_plan_id(plan_id, reason)
+        plan = @world.persistence.load_execution_plan(plan_id)
+        steps = plan.steps.values.select { |step| step.is_a?(::Dynflow::ExecutionPlan::Steps::RunStep) }
+        steps.each do |step|
+          step.state = :error
+          step.error = ::Dynflow::ExecutionPlan::Steps::Error.new(reason)
+          step.save
+        end
+        plan.update_state(:stopped)
+        plan.save
+      end
+
+      def execute_triggered(triggered)
+        @world.execute(triggered.execution_plan_id, triggered.finished)
+      end
+
+      def set_up_clock_for(plan_id, initial = false)
+        if @semaphores[plan_id].children.key? :time
+          timeout_message = 'The task could not be started within the maintenance window.'
+          interval = @semaphores[plan_id].children[:time].meta[:interval]
+          timeout = @semaphores[plan_id].children[:time].meta[:time_span]
+          @world.clock.ping(self, interval, [:release, plan_id, :time])
+          @world.clock.ping(self, timeout, [:cancel, plan_id, timeout_message]) if initial
+        end
+      end
+
+      def create_semaphores(hash)
+        semaphores = hash.keys.reduce(Utils.indifferent_hash({})) do |acc, key|
+          acc.merge(key => ::Dynflow::Semaphores::Stateful.new_from_hash(hash[key]))
+        end
+        ::Dynflow::Semaphores::Aggregating.new(semaphores)
+      end
+    end
+  end
+end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.9'
+  VERSION = '0.8.10'
 end

data/lib/dynflow/world.rb

@@ -7,7 +7,7 @@ module Dynflow
     attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
         :transaction_adapter, :logger_adapter, :coordinator,
         :persistence, :action_classes, :subscription_index,
-        :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout
+        :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter
 
     def initialize(config)
       @id                     = SecureRandom.uuid
@@ -21,7 +21,7 @@ module Dynflow
       @executor               = config_for_world.executor
       @action_classes         = config_for_world.action_classes
       @auto_rescue            = config_for_world.auto_rescue
-      @exit_on_terminate      = config_for_world.exit_on_terminate
+      @exit_on_terminate      = Concurrent::AtomicBoolean.new(config_for_world.exit_on_terminate)
       @connector              = config_for_world.connector
       @middleware             = Middleware::World.new
       @middleware.use Middleware::Common::Transaction if @transaction_adapter
@@ -29,10 +29,11 @@ module Dynflow
       @meta                   = config_for_world.meta
       @auto_validity_check    = config_for_world.auto_validity_check
       @validity_check_timeout = config_for_world.validity_check_timeout
+      @throttle_limiter       = config_for_world.throttle_limiter
       calculate_subscription_index
 
       if executor
-        @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self)
+        @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self, config_for_world.executor_semaphore)
         executor.initialized.wait
       end
       if auto_validity_check
@@ -48,7 +49,7 @@ module Dynflow
 
       if config_for_world.auto_terminate
         at_exit do
-          @exit_on_terminate = false # make sure we don't terminate twice
+          @exit_on_terminate.make_false # make sure we don't terminate twice
           self.terminate.wait
         end
       end
@@ -150,10 +151,14 @@ module Dynflow
       end
     end
 
-    def delay(action_class, delay_options, *args)
+    def delay(*args)
+      delay_with_caller(nil, *args)
+    end
+
+    def delay_with_caller(caller_action, action_class, delay_options, *args)
       raise 'No action_class given' if action_class.nil?
-      execution_plan = ExecutionPlan.new self
-      execution_plan.delay(action_class, delay_options, *args)
+      execution_plan = ExecutionPlan.new(self)
+      execution_plan.delay(caller_action, action_class, delay_options, *args)
       Scheduled[execution_plan.id]
     end
 
@@ -208,6 +213,9 @@ module Dynflow
               delayed_executor.terminate.wait
             end
 
+            logger.info "start terminating throttle_limiter..."
+            throttle_limiter.terminate.wait
+
             if executor
               connector.stop_receiving_new_work(self)
 
@@ -239,7 +247,7 @@ module Dynflow
             logger.fatal(e)
           end
         end.on_completion do
-          Kernel.exit if @exit_on_terminate
+          Thread.new { Kernel.exit } if @exit_on_terminate.true?
         end
       end
 

data/test/action_test.rb

@@ -351,9 +351,9 @@ module Dynflow
           input[:count].times.map { trigger(ChildAction, suspend: input[:suspend]) }
         end
 
-        def resume
+        def resume(*args)
           output[:custom_resume] = true
-          super
+          super *args
         end
       end
 

data/test/concurrency_control_test.rb

@@ -0,0 +1,241 @@
+require_relative 'test_helper'
+
+module Dynflow
+  module ConcurrencyControlTest
+    describe 'Concurrency Control' do
+      include PlanAssertions
+      include Dynflow::Testing::Assertions
+      include Dynflow::Testing::Factories
+      include TestHelpers
+
+      class FailureSimulator
+        def self.should_fail?
+          @should_fail || false
+        end
+
+        def self.will_fail!
+          @should_fail = true
+        end
+
+        def self.wont_fail!
+          @should_fail = false
+        end
+
+        def self.will_sleep!
+          @should_sleep = true
+        end
+
+        def self.wont_sleep!
+          @should_sleep = false
+        end
+
+        def self.should_sleep?
+          @should_sleep
+        end
+      end
+
+      before do
+        FailureSimulator.wont_fail!
+        FailureSimulator.wont_sleep!
+      end
+
+      after do
+        klok.clear
+      end
+
+      class ChildAction < ::Dynflow::Action
+        def plan(should_sleep = false)
+          raise "Simulated failure" if FailureSimulator.should_fail?
+          plan_self :should_sleep => should_sleep
+        end
+
+        def run(event = nil)
+          unless output[:slept]
+            output[:slept] = true
+            puts "SLEEPING" if input[:should_sleep]
+            suspend { |suspended| world.clock.ping(suspended, 100, [:run])  } if input[:should_sleep]
+          end
+        end
+      end
+
+      class ParentAction < ::Dynflow::Action
+        include Dynflow::Action::WithSubPlans
+
+        def plan(count, concurrency_level = nil, time_span = nil, should_sleep = nil)
+          limit_concurrency_level(concurrency_level) unless concurrency_level.nil?
+          distribute_over_time(time_span) unless time_span.nil?
+          plan_self :count => count, :should_sleep => should_sleep
+        end
+
+        def create_sub_plans
+          input[:count].times.map { |i| trigger(::Dynflow::ConcurrencyControlTest::ChildAction, input[:should_sleep]) }
+        end
+      end
+
+      let(:klok) { Dynflow::Testing::ManagedClock.new }
+      let(:world) do
+        WorldFactory.create_world do |config|
+          config.throttle_limiter = proc { |world| LoggingThrottleLimiter.new world }
+        end
+      end
+
+      def check_step(plan, total, finished)
+        world.throttle_limiter.observe(plan.id).length.must_equal (total - finished)
+        plan.sub_plans.select { |sub| planned? sub }.count.must_equal (total - finished)
+        plan.sub_plans.select { |sub| successful? sub }.count.must_equal finished
+      end
+
+      def planned?(plan)
+        plan.state == :planned && plan.result == :pending
+      end
+
+      def successful?(plan)
+        plan.state == :stopped && plan.result == :success
+      end
+
+      class LoggingThrottleLimiter < Dynflow::ThrottleLimiter
+
+        class LoggingCore < Dynflow::ThrottleLimiter::Core
+
+          attr_reader :running
+
+          def initialize(*args)
+            @running = [0]
+            super *args
+          end
+
+          def release(*args)
+            # Discard semaphores without tickets, find the one with least tickets from the rest
+            if @semaphores.key? args.first
+              tickets = @semaphores[args.first].children.values.map { |sem| sem.tickets }.compact.min
+              # Add running count to the log
+              @running << (tickets - @semaphores[args.first].free) unless tickets.nil?
+            end
+            super(*args)
+          end
+        end
+
+        def core_class
+          LoggingThrottleLimiter::LoggingCore
+        end
+      end
+
+      it 'can be disabled' do
+        total = 10
+        plan = world.plan(ParentAction, 10)
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        plan.sub_plans.all? { |sub| successful? sub }
+        world.throttle_limiter.core.ask!(:running).must_equal [0]
+      end
+
+      it 'limits by concurrency level' do
+        total = 10
+        level = 4
+        plan = world.plan(ParentAction, total, level)
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        world.throttle_limiter.core.ask!(:running).max.must_be :<=, level
+      end
+
+      it 'allows to cancel' do
+        total = 5
+        world.stub :clock, klok do
+          plan = world.plan(ParentAction, total, 0)
+          triggered = world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total }
+          world.event(plan.id, plan.steps.values.last.id, ::Dynflow::Action::Cancellable::Cancel)
+          wait_for { triggered.completed? }
+          plan.entry_action.output[:failed_count].must_equal total
+          world.throttle_limiter.core.ask!(:running).max.must_be :<=, 0
+        end
+      end
+
+      it 'calculates time interval correctly' do
+        world.stub :clock, klok do
+          total = 10
+          get_interval = ->(plan) { plan.entry_action.input[:concurrency_control][:time][:meta][:interval] }
+
+          plan = world.plan(ParentAction, total, 1, 10)
+          future = world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total }
+          wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
+          # 10 tasks over 10 seconds, one task at a time, 1 task every second
+          get_interval.call(plan).must_equal 1.0
+
+          plan = world.plan(ParentAction, total, 4, 10)
+          world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total }
+          wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
+          # 10 tasks over 10 seconds, four tasks at a time, 1 task every 0.25 second
+          get_interval.call(plan).must_equal 0.25
+
+          plan = world.plan(ParentAction, total, nil, 10)
+          world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total }
+          wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
+          # 1o tasks over 10 seconds, one task at a time (default), 1 task every second
+          get_interval.call(plan).must_equal 1.0
+        end
+      end
+
+      it 'uses the throttle limiter to handle the plans' do
+        world.stub :clock, klok do
+          time_span = 10.0
+          total = 10
+          level = 2
+          plan = world.plan(ParentAction, total, level, time_span)
+          start_time = klok.current_time
+          world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans.select { |sub| successful? sub }.count == level }
+          finished = 2
+          check_step(plan, total, finished)
+          world.throttle_limiter.observe(plan.id).dup.each do |triggered|
+            triggered.future.tap do |future|
+              klok.progress
+              wait_for { future.completed? }
+            end
+            finished += 1
+            check_step(plan, total, finished)
+          end
+          end_time = klok.current_time
+          (end_time - start_time).must_equal 4
+          world.throttle_limiter.observe(plan.id).must_equal []
+          world.throttle_limiter.core.ask!(:running).max.must_be :<=, level
+        end
+      end
+
+      it 'fails tasks which failed to plan immediately' do
+        FailureSimulator.will_fail!
+        total = 5
+        level = 1
+        time_span = 10
+        plan = world.plan(ParentAction, total, level, time_span)
+        future = world.execute(plan.id)
+        wait_for { future.completed? }
+        plan.sub_plans.all? { |sub| sub.result == :error }.must_equal true
+      end
+
+      it 'cancels tasks which could not be started within the time window' do
+        world.stub :clock, klok do
+          time_span = 10.0
+          level = 1
+          total = 10
+          plan = world.plan(ParentAction, total, level, time_span, true)
+          future = world.execute(plan.id)
+          wait_for { plan.sub_plans.count == total && plan.sub_plans.all? { |sub| sub.result == :pending } }
+          planned, running = plan.sub_plans.partition { |sub| planned? sub }
+          planned.count.must_equal total - level
+          running.count.must_equal level
+          world.throttle_limiter.observe(plan.id).length.must_equal (total - 1)
+          4.times { klok.progress }
+          wait_for { future.completed? }
+          finished, stopped = plan.sub_plans.partition { |sub| successful? sub }
+          finished.count.must_equal level
+          stopped.count.must_equal (total - level)
+        end
+      end
+    end
+  end
+end