dynflow 0.8.16 → 0.8.17

data/lib/dynflow/executors/parallel/pool.rb

@@ -47,8 +47,8 @@ module Dynflow
           distribute_jobs
         end
 
-        def worker_done(worker, step)
-          @executor_core.tell([:finish_step, step])
+        def worker_done(worker, work)
+          @executor_core.tell([:work_finished, work])
           @free_workers << worker
           distribute_jobs
         end

data/lib/dynflow/executors/parallel/worker.rb

@@ -2,25 +2,17 @@ module Dynflow
   module Executors
     class Parallel < Abstract
       class Worker < Actor
-
         def initialize(pool, transaction_adapter)
           @pool                = Type! pool, Concurrent::Actor::Reference
           @transaction_adapter = Type! transaction_adapter, TransactionAdapters::Abstract
         end
 
-        def on_message(message)
-          match message,
-                (on Work::Step.(step: ~any) |
-                        Work::Event.(step: ~any, event: Parallel::Event.(event: ~any)) do |step, event|
-                  step.execute event
-                end),
-                (on Work::Finalize.(~any, any) do |sequential_manager|
-                   sequential_manager.finalize
-                 end)
+        def on_message(work_item)
+          work_item.execute
         rescue Errors::PersistenceError => e
           @pool.tell([:handle_persistence_error, e])
         ensure
-          @pool.tell([:worker_done, reference, message])
+          @pool.tell([:worker_done, reference, work_item])
           @transaction_adapter.cleanup
         end
       end

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -266,7 +266,6 @@ module Dynflow
         records.map { |record| load_data(record) }
       end
 
-
       def load_data(record)
         Utils.indifferent_hash(MultiJson.load(record[:data]))
       end
@@ -277,7 +276,7 @@ module Dynflow
 
       def extract_metadata(what, value)
         meta_keys = META_DATA.fetch(what)
-        value         = Utils.indifferent_hash(value)
+        value     = Utils.indifferent_hash(value)
         meta_keys.inject({}) { |h, k| h.update k.to_sym => value[k] }
       end
 

data/lib/dynflow/testing.rb

@@ -23,6 +23,8 @@ module Dynflow
     require 'dynflow/testing/dummy_execution_plan'
     require 'dynflow/testing/dummy_step'
     require 'dynflow/testing/dummy_planned_action'
+    require 'dynflow/testing/in_thread_executor'
+    require 'dynflow/testing/in_thread_world'
     require 'dynflow/testing/assertions'
     require 'dynflow/testing/factories'
 

data/lib/dynflow/testing/in_thread_executor.rb

@@ -0,0 +1,52 @@
+module Dynflow
+  module Testing
+    class InThreadExecutor < Dynflow::Executors::Abstract
+      def initialize(world)
+        @world = world
+        @director = Director.new(@world)
+        @work_items = Queue.new
+      end
+
+      def execute(execution_plan_id, finished = Concurrent.future, _wait_for_acceptance = true)
+        feed_queue(@director.start_execution(execution_plan_id, finished))
+        process_work_items
+        finished
+      end
+
+      def process_work_items
+        until @work_items.empty?
+          feed_queue(handle_work(@work_items.pop))
+          clock_tick
+        end
+      end
+
+      def handle_work(work_item)
+        work_item.execute
+        @director.work_finished(work_item)
+      end
+
+      def event(execution_plan_id, step_id, event, future = Concurrent.future)
+        event = (Director::Event[execution_plan_id, step_id, event, future])
+        @director.handle_event(event).each do |work_item|
+          @work_items << work_item
+        end
+        future
+      end
+
+      def clock_tick
+        @world.clock.progress
+      end
+
+      def feed_queue(work_items)
+        work_items.each { |work_item| @work_items.push(work_item) }
+      end
+
+      def terminate(future = Concurrent.future)
+        @director.terminate
+        future.success true
+      rescue => e
+        future.fail e
+      end
+    end
+  end
+end

data/lib/dynflow/testing/in_thread_world.rb

@@ -0,0 +1,64 @@
+module Dynflow
+  module Testing
+    class InThreadWorld < Dynflow::World
+      def self.test_world_config
+        config                     = Dynflow::Config.new
+        config.persistence_adapter = persistence_adapter
+        config.logger_adapter      = logger_adapter
+        config.coordinator_adapter = coordinator_adapter
+        config.delayed_executor    = nil
+        config.auto_rescue         = false
+        config.auto_validity_check = false
+        config.exit_on_terminate   = false
+        config.auto_execute        = false
+        config.auto_terminate      = false
+        yield config if block_given?
+        return config
+      end
+
+      def self.persistence_adapter
+        @persistence_adapter ||= begin
+                                   db_config = ENV['DB_CONN_STRING'] || 'sqlite:/'
+                                   puts "Using database configuration: #{db_config}"
+                                   Dynflow::PersistenceAdapters::Sequel.new(db_config)
+                                 end
+      end
+
+      def self.logger_adapter
+        @adapter ||= Dynflow::LoggerAdapters::Simple.new $stderr, 4
+      end
+
+      def self.coordinator_adapter
+        ->(world, _) { CoordiationAdapterWithLog.new(world) }
+      end
+
+      # The worlds created by this method are getting terminated after each test run
+      def self.instance(&block)
+        @instance ||= self.new(test_world_config(&block))
+      end
+
+      def initialize(*args)
+        super
+        @clock = ManagedClock.new
+        @executor = InThreadExecutor.new(self)
+      end
+
+      def execute(execution_plan_id, done = Concurrent.future)
+        @executor.execute(execution_plan_id, done)
+      end
+
+      def terminate(future = Concurrent.future)
+        run_before_termination_hooks
+        @executor.terminate
+        coordinator.delete_world(registered_world)
+        future.success true
+      rescue => e
+        future.fail e
+      end
+
+      def event(execution_plan_id, step_id, event, done = Concurrent.future)
+        @executor.event(execution_plan_id, step_id, event, done)
+      end
+    end
+  end
+end

data/lib/dynflow/testing/managed_clock.rb

@@ -11,7 +11,7 @@ module Dynflow
       end
 
       def ping(who, time, with_what = nil, where = :<<)
-        time  = current_time + time if time.is_a? Numeric
+        time = current_time + time if time.is_a? Numeric
         with = with_what.nil? ? None : Some[Object][with_what]
         @pending_pings << Clock::Timer[who, time, with, where]
         @pending_pings.sort!

data/lib/dynflow/throttle_limiter.rb

@@ -100,7 +100,7 @@ module Dynflow
 
       def cancel_plan_id(plan_id, reason)
         plan = @world.persistence.load_execution_plan(plan_id)
-        steps = plan.steps.values.select { |step| step.is_a?(::Dynflow::ExecutionPlan::Steps::RunStep) }
+        steps = plan.run_steps
         steps.each do |step|
           step.state = :error
           step.error = ::Dynflow::ExecutionPlan::Steps::Error.new(reason)

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.16'
+  VERSION = '0.8.17'
 end

data/lib/dynflow/world.rb

@@ -5,9 +5,9 @@ module Dynflow
     include Algebrick::Matching
 
     attr_reader :id, :client_dispatcher, :executor_dispatcher, :executor, :connector,
-        :transaction_adapter, :logger_adapter, :coordinator,
-        :persistence, :action_classes, :subscription_index,
-        :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter
+                :transaction_adapter, :logger_adapter, :coordinator,
+                :persistence, :action_classes, :subscription_index,
+                :middleware, :auto_rescue, :clock, :meta, :delayed_executor, :auto_validity_check, :validity_check_timeout, :throttle_limiter
 
     def initialize(config)
       @id                     = SecureRandom.uuid
@@ -141,7 +141,7 @@ module Dynflow
       else
         execution_plan = plan(action_class, *args)
       end
-      planned        = execution_plan.state == :planned
+      planned = execution_plan.state == :planned
 
       if planned
         done = execute(execution_plan.id, Concurrent.future)
@@ -280,7 +280,13 @@ module Dynflow
     end
 
     def invalidate_execution_lock(execution_lock)
-      plan = persistence.load_execution_plan(execution_lock.execution_plan_id)
+      begin
+        plan = persistence.load_execution_plan(execution_lock.execution_plan_id)
+      rescue KeyError => e
+        logger.error "invalidated execution plan #{execution_lock.execution_plan_id} missing, skipping"
+        coordinator.release(execution_lock)
+        return
+      end
       plan.execution_history.add('terminate execution', execution_lock.world_id)
 
       plan.steps.values.each do |step|
@@ -291,7 +297,7 @@ module Dynflow
         end
       end
 
-      plan.update_state(:paused) unless [:paused, :stopped].include?(plan.state)
+      plan.update_state(:paused) if plan.state == :running
       plan.save
       coordinator.release(execution_lock)
       unless plan.error?
@@ -322,6 +328,7 @@ module Dynflow
               invalidate(world)
               result = :invalidated
             rescue => e
+              logger.error e
               result = e.message
             end
           else
@@ -374,7 +381,6 @@ module Dynflow
     end
 
     private
-
     def calculate_subscription_index
       @subscription_index =
           action_classes.each_with_object(Hash.new { |h, k| h[k] = [] }) do |klass, index|

data/test/abnormal_states_recovery_test.rb

@@ -85,6 +85,16 @@ module Dynflow
                               "unlock world-invalidation:#{executor_world.id}"]
             client_world.coordinator.adapter.lock_log.must_equal(expected_locks)
           end
+
+          it "handles missing execution plans" do
+            lock = Coordinator::ExecutionLock.new(executor_world, "missing", nil, nil)
+            executor_world.coordinator.acquire(lock)
+            client_world.invalidate(executor_world.registered_world)
+            expected_locks = ["lock world-invalidation:#{executor_world.id}",
+                              "unlock execution-plan:missing",
+                              "unlock world-invalidation:#{executor_world.id}"]
+            client_world.coordinator.adapter.lock_log.must_equal(expected_locks)
+          end
         end
       end
 

data/test/action_test.rb

@@ -81,7 +81,7 @@ module Dynflow
       end
 
       it 'fails when output is not serializable' do
-        klass  = Class.new(Dynflow::Action) do
+        klass = Class.new(Dynflow::Action) do
           def run
             output.update key: Object.new
           end
@@ -148,7 +148,7 @@ module Dynflow
 
         class Config
           attr_accessor :external_service, :poll_max_retries,
-              :poll_intervals, :attempts_before_next_interval
+                        :poll_intervals, :attempts_before_next_interval
 
           def initialize
             @external_service              = ExternalService.new
@@ -235,13 +235,13 @@ module Dynflow
         end
 
         it 'initiates the external task' do
-          action   = run_action plan
+          action = run_action plan
 
           action.output[:task][:task_id].must_equal 123
         end
 
         it 'polls till the task is done' do
-          action   = run_action plan
+          action = run_action plan
 
           9.times { progress_action_time action }
           action.done?.must_equal false
@@ -255,14 +255,14 @@ module Dynflow
         end
 
         it 'tries to poll for the old task when resuming' do
-          action   = run_action plan
+          action = run_action plan
           action.output[:task][:progress].must_equal 0
           run_action action
           action.output[:task][:progress].must_equal 10
         end
 
         it 'invokes the external task again when polling on the old one fails' do
-          action   = run_action plan
+          action = run_action plan
           action.world.silence_logger!
           action.external_service.will_fail
           action.output[:task][:progress].must_equal 0
@@ -271,7 +271,7 @@ module Dynflow
         end
 
         it 'tolerates some failure while polling' do
-          action   = run_action plan
+          action = run_action plan
           action.external_service.will_fail
           action.world.silence_logger!
 
@@ -293,7 +293,7 @@ module Dynflow
           TestPollingAction.config.poll_intervals = [1, 2]
           TestPollingAction.config.attempts_before_next_interval = 2
 
-          action   = run_action plan
+          action = run_action plan
           pings = []
           pings << next_ping(action)
           progress_action_time action
@@ -317,7 +317,7 @@ module Dynflow
         end
 
         it 'timesout' do
-          action   = run_action plan
+          action = run_action plan
           iterations = 0
           while progress_action_time action
             # we count the number of iterations till the timeout occurs

data/test/clock_test.rb

@@ -12,7 +12,6 @@ describe clock_class do
     clock.ping [], 0.1, :pong
   end
 
-
   it 'pongs' do
     q = Queue.new
     start = Time.now
@@ -51,5 +50,4 @@ describe clock_class do
     threads.each &:join
   end
 
-
 end

data/test/concurrency_control_test.rb

@@ -53,7 +53,7 @@ module Dynflow
           unless output[:slept]
             output[:slept] = true
             puts "SLEEPING" if input[:should_sleep]
-            suspend { |suspended| world.clock.ping(suspended, 100, [:run])  } if input[:should_sleep]
+            suspend { |suspended| world.clock.ping(suspended, 100, [:run]) } if input[:should_sleep]
           end
         end
       end

data/test/execution_plan_test.rb

@@ -61,7 +61,6 @@ module Dynflow
 
         end
 
-
         describe 'for error in running phase' do
 
           before do
@@ -224,7 +223,6 @@ module Dynflow
           end
         end
 
-
         describe 'finalize flow' do
 
           let :execution_plan do

data/test/executor_test.rb

@@ -113,9 +113,8 @@ module Dynflow
             TestPause.when_paused do
               plan = world.persistence.load_execution_plan(execution_plan.id)
               plan.state.must_equal :running
-              triage = plan.steps.values.find do |s|
-                s.is_a?(Dynflow::ExecutionPlan::Steps::RunStep) &&
-                    s.action_class == Support::CodeWorkflowExample::Triage
+              triage = plan.run_steps.find do |s|
+                s.action_class == Support::CodeWorkflowExample::Triage
               end
               triage.state.must_equal :running
               world.persistence.
@@ -204,10 +203,7 @@ module Dynflow
             it 'fails' do
               assert_equal :error, result.result
               assert_equal :paused, result.state
-              assert_equal :error,
-                           result.steps.values.
-                               find { |s| s.is_a? Dynflow::ExecutionPlan::Steps::RunStep }.
-                               state
+              assert_equal :error, result.run_steps.first.state
             end
           end
 
@@ -292,14 +288,12 @@ module Dynflow
             specify do
               assert_equal :paused, result.state
               assert_equal :error, result.result
-              assert_equal :error, result.steps.values.
-                  find { |s| s.is_a? Dynflow::ExecutionPlan::Steps::RunStep }.state
+              assert_equal :error, result.run_steps.first.state
 
               ep = world.execute(result.id).value
               assert_equal :stopped, ep.state
               assert_equal :success, ep.result
-              assert_equal :success, ep.steps.values.
-                  find { |s| s.is_a? Dynflow::ExecutionPlan::Steps::RunStep }.state
+              assert_equal :success, ep.run_steps.first.state
             end
           end
 
@@ -536,7 +530,7 @@ module Dynflow
           world.plan(Support::CodeWorkflowExample::IncomingIssues, issues_data)
         end
 
-        let(:manager) { Executors::Parallel::FlowManager.new execution_plan, execution_plan.run_flow }
+        let(:manager) { Director::FlowManager.new execution_plan, execution_plan.run_flow }
 
         def assert_next_steps(expected_next_step_ids, finished_step_id = nil, success = true)
           if finished_step_id
@@ -569,7 +563,6 @@ module Dynflow
             assert_next_steps([], 4, false)
           end
 
-
           it "is not done while other steps can be finished" do
             assert_next_steps([4, 13])
             assert_next_steps([], 4, false)