dynflow 0.8.33 → 0.8.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 331deb991c6fe8d4617af49ce8072fe0edceef6f
-  data.tar.gz: 8de8181ec0b0451fefab469ee496172b8f1938dc
+  metadata.gz: 7ff5e3fce545cac6d1d299717c2fa11b54ba9ef6
+  data.tar.gz: 90214fe9c441dfae1780f591cca27de7eaa89da3
 SHA512:
-  metadata.gz: 90d6626cc3af5a24784985ac6efd3beb029a50524b899f3ba9028248ddd0952cab7d72f46284cb1595a2d8ec74512a4d0247a77e41011306ec5a237d51c18e43
-  data.tar.gz: 5db089dc8165ea98ad5677e8a4849e369c6329b856023084cdd6cc4b9ef4427a6bd39a59e8921be689ba3b03b7427138647b636ec02c386ff03610d2d5009631
+  metadata.gz: e1084dc1ef5c5d8d923df3fc92d93b1b6e92edd85d29ab4fd5fe045908315a692b2cbf62615d0d34aedbf18b077d522b9eff4b368944c31c5689d1f8484820d2
+  data.tar.gz: eb449ff93ccc6720e13a4cc539dbec6a1296a4d81d58b3a4129a2ef8530fae769c0a68fe870317d0775fe094919de460d4c719c5bcc2a1318ee3b7939c90a9d7

data/lib/dynflow/action/with_bulk_sub_plans.rb

@@ -30,7 +30,8 @@ module Dynflow
 
     def initiate
       output[:planned_count] = 0
-      output[:total_count]   = total_count
+      output[:cancelled_count] = 0
+      output[:total_count] = total_count
       super
     end
 
@@ -58,7 +59,8 @@ module Dynflow
     # The same logic as in Action::WithSubPlans, but calculated using the expected total count
     def run_progress
       if counts_set?
-        (output[:success_count] + output[:failed_count]).to_f / total_count
+        sum = output.values_at(:success_count, :cancelled_count, :failed_count).reduce(:+)
+        sum.to_f / total_count
       else
         0.1
       end
@@ -71,8 +73,8 @@ module Dynflow
     end
 
     def cancel!(force = false)
-      # Count the not-yet-planned tasks as failed
-      output[:failed_count] += total_count - output[:planned_count]
+      # Count the not-yet-planned tasks as cancelled
+      output[:cancelled_count] = total_count - output[:planned_count]
       if uses_concurrency_control
         # Tell the throttle limiter to cancel the tasks its managing
         world.throttle_limiter.cancel!(execution_plan_id)
@@ -92,8 +94,11 @@ module Dynflow
     end
 
     def can_spawn_next_batch?
-      total_count - output[:success_count] - output[:pending_count] - output[:failed_count] > 0
+      remaining_count > 0
     end
 
+    def remaining_count
+      total_count - output[:cancelled_count] - output[:planned_count]
+    end
   end
 end

data/lib/dynflow/action/with_polling_sub_plans.rb

@@ -60,11 +60,11 @@ module Dynflow
     end
 
     def recalculate_counts
-      total      = sub_plans.count
-      failed     = sub_plans('state' => %w(paused stopped), 'result' => 'error').count
-      success    = sub_plans('state' => 'stopped', 'result' => 'success').count
+      total      = sub_plans_count
+      failed     = sub_plans_count('state' => %w(paused stopped), 'result' => 'error')
+      success    = sub_plans_count('state' => 'stopped', 'result' => 'success')
       output.update(:total_count   => total - output.fetch(:resumed_count, 0),
-                    :pending_count => 0,
+                    :pending_count => total - failed - success,
                     :failed_count  => failed - output.fetch(:resumed_count, 0),
                     :success_count => success)
     end

data/lib/dynflow/action/with_sub_plans.rb

@@ -162,6 +162,12 @@ module Dynflow
       end
     end
 
+    def sub_plans_count(filter = {})
+      filters = { 'caller_execution_plan_id' => execution_plan_id,
+                  'caller_action_id' => self.id }
+      world.persistence.find_execution_plan_counts(filters: filters.merge(filter))
+    end
+
     def notify_on_finish(plans)
       suspend do |suspended_action|
         plans.each do |plan|

data/lib/dynflow/execution_plan.rb

@@ -1,7 +1,7 @@
 require 'securerandom'
 
 module Dynflow
-
+  # rubocop:disable Metrics/ClassLength
   # TODO extract planning logic to an extra class ExecutionPlanner
   class ExecutionPlan < Serializable
 
@@ -162,6 +162,10 @@ module Dynflow
       persistence.find_execution_plans(filters: { 'caller_execution_plan_id' => self.id })
     end
 
+    def sub_plans_count
+      persistence.find_execution_plan_counts(filters: { 'caller_execution_plan_id' => self.id })
+    end
+
     def rescue_plan_id
       case rescue_strategy
       when Action::Rescue::Pause
@@ -519,4 +523,5 @@ module Dynflow
 
     private_class_method :steps_from_hash
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/dynflow/persistence.rb

@@ -40,6 +40,10 @@ module Dynflow
       end
     end
 
+    def find_execution_plan_counts(options)
+      adapter.find_execution_plan_counts(options)
+    end
+
     def delete_execution_plans(filters, batch_size = 1000, enforce_backup_dir = nil)
       backup_dir = enforce_backup_dir || current_backup_dir
       adapter.delete_execution_plans(filters, batch_size, backup_dir)

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -39,6 +39,12 @@ module Dynflow
         raise NotImplementedError
       end
 
+      # @option options [Hash{ String => Object,Array<object> }] filters hash represents
+      #   set of allowed values for a given key representing column
+      def find_execution_plan_counts(options = {})
+        filter(:execution_plan, options[:filters]).count
+      end
+
       # @param filters [Hash{ String => Object }] filters to determine
       #   what to delete
       # @param batch_size the size of the chunks to iterate over when

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -60,6 +60,10 @@ module Dynflow
         data_set.all.map { |record| load_data(record) }
       end
 
+      def find_execution_plan_counts(options = {})
+        filter(:execution_plan, table(:execution_plan), options[:filters]).count
+      end
+
       def delete_execution_plans(filters, batch_size = 1000, backup_dir = nil)
         count = 0
         filter(:execution_plan, table(:execution_plan), filters).each_slice(batch_size) do |plans|

data/lib/dynflow/rails/configuration.rb

@@ -118,6 +118,7 @@ module Dynflow
 
       def default_sequel_adapter_options
         db_config            = ::ActiveRecord::Base.configurations[::Rails.env].dup
+        db_config['adapter'] = db_config['adapter'].gsub(/_?makara_?/, '')
         db_config['adapter'] = 'postgres' if db_config['adapter'] == 'postgresql'
         db_config['max_connections'] = db_pool_size if increase_db_pool_size?
 

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.33'
+  VERSION = '0.8.34'
 end

data/test/action_test.rb

@@ -433,6 +433,7 @@ module Dynflow
       specify "the sub-plan stores the information about its parent" do
         sub_plans = execution_plan.sub_plans
         sub_plans.size.must_equal 2
+        execution_plan.sub_plans_count.must_equal 2
         sub_plans.each { |sub_plan| sub_plan.caller_execution_plan_id.must_equal execution_plan.id }
       end
 
@@ -505,7 +506,7 @@ module Dynflow
 
               wait_for do # Waiting for the sub plans to be spawned
                 polling_plan = world.persistence.load_execution_plan(triggered_plan.id)
-                polling_plan.sub_plans.count == total
+                polling_plan.sub_plans_count == total
               end
 
               # Moving the clock to make the parent check on sub plans
@@ -522,7 +523,7 @@ module Dynflow
               world.execute(polling_plan.id) # The actual resume
 
               wait_for do # Waiting for new generation of sub plans to be spawned
-                polling_plan.sub_plans.count == 2 * total
+                polling_plan.sub_plans_count == 2 * total
               end
 
               # Move the clock again
@@ -545,7 +546,7 @@ module Dynflow
 
               wait_for do # Waiting for the sub plans to be spawned
                 polling_plan = world.persistence.load_execution_plan(triggered_plan.id)
-                polling_plan.sub_plans.count == total &&
+                polling_plan.sub_plans_count == total &&
                   polling_plan.sub_plans.all? { |sub| sub.state == :paused }
               end
 
@@ -629,7 +630,7 @@ module Dynflow
             world.execute(plan.id)
             clock.pending_pings.count.must_equal 0
             wait_for do
-              plan.sub_plans.count == total &&
+              plan.sub_plans_count == total &&
                 plan.sub_plans.all? { |sub| sub.result == :success }
             end
             clock.pending_pings.count.must_equal 1
@@ -658,7 +659,7 @@ module Dynflow
 
             # Wait for the sub plans to finish
             wait_for do
-              plan.sub_plans.count == total &&
+              plan.sub_plans_count == total &&
                 plan.sub_plans.all? { |sub| sub.result == :success }
             end
 

data/test/batch_sub_tasks_test.rb

@@ -93,6 +93,19 @@ module Dynflow
         action.output[:success_count].must_equal action.total_count
       end
 
+      it 'is controlled only by total_count and output[:planned_count]' do
+        plan = world.plan(ParentAction, 10)
+        future = world.execute plan.id
+        wait_for { future.completed? }
+        action = plan.entry_action
+        action.send(:can_spawn_next_batch?).must_equal false
+        action.current_batch.must_be :empty?
+        action.output[:pending_count] = 0
+        action.output[:success_count] = 5
+        action.send(:can_spawn_next_batch?).must_equal false
+        action.current_batch.must_be :empty?
+      end
+
     end
   end
 end

data/test/concurrency_control_test.rb

@@ -143,7 +143,7 @@ module Dynflow
         world.stub :clock, klok do
           plan = world.plan(ParentAction, total, 0)
           triggered = world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans_count == total }
           world.event(plan.id, plan.steps.values.last.id, ::Dynflow::Action::Cancellable::Cancel)
           wait_for { triggered.completed? }
           plan.entry_action.output[:failed_count].must_equal total
@@ -158,21 +158,21 @@ module Dynflow
 
           plan = world.plan(ParentAction, total, 1, 10)
           future = world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans_count == total }
           wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
           # 10 tasks over 10 seconds, one task at a time, 1 task every second
           get_interval.call(plan).must_equal 1.0
 
           plan = world.plan(ParentAction, total, 4, 10)
           world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans_count == total }
           wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
           # 10 tasks over 10 seconds, four tasks at a time, 1 task every 0.25 second
           get_interval.call(plan).must_equal 0.25
 
           plan = world.plan(ParentAction, total, nil, 10)
           world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans_count == total }
           wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
           # 1o tasks over 10 seconds, one task at a time (default), 1 task every second
           get_interval.call(plan).must_equal 1.0
@@ -187,7 +187,7 @@ module Dynflow
           plan = world.plan(ParentAction, total, level, time_span)
           start_time = klok.current_time
           world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total }
+          wait_for { plan.sub_plans_count == total }
           wait_for { plan.sub_plans.select { |sub| successful? sub }.count == level }
           finished = 2
           check_step(plan, total, finished)
@@ -224,7 +224,7 @@ module Dynflow
           total = 10
           plan = world.plan(ParentAction, total, level, time_span, true)
           future = world.execute(plan.id)
-          wait_for { plan.sub_plans.count == total && plan.sub_plans.all? { |sub| sub.result == :pending } }
+          wait_for { plan.sub_plans_count == total && plan.sub_plans.all? { |sub| sub.result == :pending } }
           planned, running = plan.sub_plans.partition { |sub| planned? sub }
           planned.count.must_equal total - level
           running.count.must_equal level

data/test/persistence_test.rb

@@ -123,6 +123,45 @@ module Dynflow
           end
         end
 
+        describe '#def find_execution_plan_counts' do
+          before do
+            # the tests expect clean field
+            adapter.delete_execution_plans({})
+          end
+
+          it 'supports filtering' do
+            prepare_plans
+            if adapter.ordering_by.include?('state')
+              loaded_plans = adapter.find_execution_plan_counts(filters: { label: ['test1'] })
+              loaded_plans.must_equal 1
+              loaded_plans = adapter.find_execution_plan_counts(filters: { state: ['paused'] })
+              loaded_plans.must_equal 3
+
+              loaded_plans = adapter.find_execution_plan_counts(filters: { state: ['stopped'] })
+              loaded_plans.must_equal 1
+
+              loaded_plans = adapter.find_execution_plan_counts(filters: { state: [] })
+              loaded_plans.must_equal 0
+
+              loaded_plans = adapter.find_execution_plan_counts(filters: { state: ['stopped', 'paused'] })
+              loaded_plans.must_equal 4
+
+              loaded_plans = adapter.find_execution_plan_counts(filters: { 'state' => ['stopped', 'paused'] })
+              loaded_plans.must_equal 4
+
+              loaded_plans = adapter.find_execution_plan_counts(filters: { label: ['test1'], :delayed => true })
+              loaded_plans.must_equal 0
+
+              adapter.save_delayed_plan('plan1',
+                                        :execution_plan_uuid => 'plan1',
+                                        :start_at => format_time(Time.now + 60),
+                                        :start_before => format_time(Time.now - 60))
+              loaded_plans = adapter.find_execution_plan_counts(filters: { label: ['test1'], :delayed => true })
+              loaded_plans.must_equal 1
+            end
+          end
+        end
+
         describe '#load_execution_plan and #save_execution_plan' do
           it 'serializes/deserializes the plan data' do
             -> { adapter.load_execution_plan('plan1') }.must_raise KeyError

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 0.8.33
+  version: 0.8.34
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-29 00:00:00.000000000 Z
+date: 2017-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json