dynflow 0.8.35 → 0.8.36

data/lib/dynflow/serializers/abstract.rb

@@ -1,37 +1,66 @@
 module Dynflow
   module Serializers
+    # @abstract
+    # Used to serialize and deserialize arguments for storage in a database.
+    # Used by {DelayedPlan} to store arguments which should be passed into
+    # the {Dynflow::Action}'s #plan method when the plan is executed.
     class Abstract
 
       attr_reader :args, :serialized_args
 
+      # @param args [Array] arguments to be serialized
+      # @param serialized_args [nil, Array] arguments in their serialized form
       def initialize(args, serialized_args = nil)
         @args = args
         @serialized_args = serialized_args
       end
 
-      def args
+      # Retrieves the arguments
+      #
+      # @raise [RuntimeError] if the deserialized arguments are not available
+      # @return [Array] the arguments
+      def args!
         raise "@args not set" if @args.nil?
         return @args
       end
 
-      def serialized_args
+      # Retrieves the arguments in the serialized form
+      #
+      # @raise [RuntimeError] if the serialized arguments are not available
+      # @return [Array] the serialized arguments
+      def serialized_args!
         raise "@serialized_args not set" if @serialized_args.nil?
         return @serialized_args
       end
 
+      # Converts arguments into their serialized form, iterates over deserialized
+      # arguments, applying {#serialize} to each of them
+      #
+      # @raise [RuntimeError] if the deserialized arguments are not available
+      # @return [Array] the serialized arguments
       def perform_serialization!
-        @serialized_args = args.map { |arg| serialize arg }
+        @serialized_args = args!.map { |arg| serialize arg }
       end
 
+      # Converts arguments into their deserialized form, iterates over serialized
+      # arguments, applying {#deserialize} to each of them
+      #
+      # @raise [RuntimeError] if the serialized arguments are not available
+      # @return [Array] the deserialized arguments
       def perform_deserialization!
-        raise "@serialized_args not set" if @serialized_args.nil?
-        @args = serialized_args.map { |arg| deserialize arg }
+        @args = serialized_args!.map { |arg| deserialize arg }
       end
 
+      # Converts an argument into it serialized form
+      #
+      # @param arg the argument to be serialized
       def serialize(arg)
         raise NotImplementedError
       end
 
+      # Converts a serialized argument into its deserialized form
+      #
+      # @param arg the argument to be deserialized
       def deserialize(arg)
         raise NotImplementedError
       end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.35'
+  VERSION = '0.8.36'.freeze
 end

data/test/batch_sub_tasks_test.rb

@@ -69,6 +69,7 @@ module Dynflow
         plan = world.plan(ParentAction, 20)
         future = world.execute plan.id
         wait_for { future.completed? }
+        plan = world.persistence.load_execution_plan(plan.id)
         action = plan.entry_action
 
         action.output[:batch_count].must_equal action.total_count / action.batch_size
@@ -79,6 +80,7 @@ module Dynflow
         plan = world.plan(ParentAction, 20)
         future = world.execute plan.id
         wait_for { future.completed? }
+        plan = world.persistence.load_execution_plan(plan.id)
         action = plan.entry_action
         action.output[:batch_count].must_equal 1
         future.value.state.must_equal :paused
@@ -97,6 +99,7 @@ module Dynflow
         plan = world.plan(ParentAction, 10)
         future = world.execute plan.id
         wait_for { future.completed? }
+        plan = world.persistence.load_execution_plan(plan.id)
         action = plan.entry_action
         action.send(:can_spawn_next_batch?).must_equal false
         action.current_batch.must_be :empty?

data/test/concurrency_control_test.rb

@@ -146,6 +146,7 @@ module Dynflow
           wait_for { plan.sub_plans_count == total }
           world.event(plan.id, plan.steps.values.last.id, ::Dynflow::Action::Cancellable::Cancel)
           wait_for { triggered.completed? }
+          plan = world.persistence.load_execution_plan(plan.id)
           plan.entry_action.output[:failed_count].must_equal total
           world.throttle_limiter.core.ask!(:running).max.must_be :<=, 0
         end
@@ -154,10 +155,13 @@ module Dynflow
       it 'calculates time interval correctly' do
         world.stub :clock, klok do
           total = 10
-          get_interval = ->(plan) { plan.entry_action.input[:concurrency_control][:time][:meta][:interval] }
+          get_interval = ->(plan) do
+            plan = world.persistence.load_execution_plan(plan.id)
+            plan.entry_action.input[:concurrency_control][:time][:meta][:interval]
+          end
 
           plan = world.plan(ParentAction, total, 1, 10)
-          future = world.execute(plan.id)
+          world.execute(plan.id)
           wait_for { plan.sub_plans_count == total }
           wait_for { klok.progress; plan.sub_plans.all? { |sub| successful? sub } }
           # 10 tasks over 10 seconds, one task at a time, 1 task every second

data/test/execution_plan_hooks_test.rb

@@ -0,0 +1,86 @@
+require_relative 'test_helper'
+
+module Dynflow
+  class ExecutionPlan
+    describe Hooks do
+      include PlanAssertions
+
+      let(:world) { WorldFactory.create_world }
+
+      class Flag
+        class << self
+          def raise!
+            @raised = true
+          end
+
+          def raised?
+            @raised
+          end
+
+          def lower!
+            @raised = false
+          end
+        end
+      end
+
+      module FlagHook
+        def raise_flag(_execution_plan)
+          Flag.raise!
+        end
+
+        def controlled_failure(_execution_plan)
+          Flag.raise!
+          raise "A controlled failure"
+        end
+      end
+
+      class ActionWithHooks < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :raise_flag, :on => :success
+      end
+
+      class ActionOnStop < ::Dynflow::Action
+        include FlagHook
+
+        execution_plan_hooks.use :controlled_failure, :on => :stopped
+      end
+
+      class Inherited < ActionWithHooks; end
+      class Overriden < ActionWithHooks
+        execution_plan_hooks.do_not_use :raise_flag
+      end
+
+      before { Flag.lower! }
+
+      it 'runs the on_success hook' do
+        refute Flag.raised?
+        plan = world.trigger(ActionWithHooks)
+        plan.finished.wait!
+        assert Flag.raised?
+      end
+
+      it 'does not alter the execution plan when exception happens in the hook' do
+        refute Flag.raised?
+        plan = world.plan(ActionOnStop)
+        plan = world.execute(plan.id).wait!.value
+        assert Flag.raised?
+        plan.result.must_equal :success
+      end
+
+      it 'inherits the hooks when subclassing' do
+        refute Flag.raised?
+        plan = world.trigger(Inherited)
+        plan.finished.wait!
+        assert Flag.raised?
+      end
+
+      it 'can override the hooks from the child' do
+        refute Flag.raised?
+        plan = world.trigger(Overriden)
+        plan.finished.wait!
+        refute Flag.raised?
+      end
+    end
+  end
+end

data/test/executor_test.rb

@@ -221,8 +221,11 @@ module Dynflow
               result.started_at.wont_be_nil
               result.ended_at.wont_be_nil
               result.execution_time.must_be :<, result.real_time
-              result.execution_time.must_equal(
-                  result.steps.inject(0) { |sum, (_, step)| sum + step.execution_time })
+
+              step_sum = result.steps.values.map(&:execution_time).reduce(:+)
+
+              # Storing floats can lead to slight deviations, 1ns precision should be enough
+              result.execution_time.must_be_close_to step_sum, 0.000_001
 
               plan_step = result.steps[1]
               plan_step.started_at.wont_be_nil

data/test/future_execution_test.rb

@@ -115,6 +115,9 @@ module Dynflow
       end
 
       describe 'serializers' do
+        let(:args) { %w(arg1 arg2) }
+        let(:serialized_serializer) { Dynflow::Serializers::Noop.new(nil, args) }
+        let(:deserialized_serializer) { Dynflow::Serializers::Noop.new(args, nil) }
         let(:save_and_load) do
           ->(thing) { MultiJson.load(MultiJson.dump(thing)) }
         end
@@ -134,6 +137,40 @@ module Dynflow
           input = [1, 2.0, 'three', ['four-1', 'four-2'], { 'five' => 5 }]
           simulated_use.call(Dynflow::Serializers::Noop, input).must_equal input
         end
+
+        it 'args! raises if not deserialized' do
+          proc { serialized_serializer.args! }.must_raise RuntimeError
+          deserialized_serializer.args! # Must not raise
+        end
+
+        it 'serialized_args! raises if not serialized' do
+          proc { deserialized_serializer.serialized_args! }.must_raise RuntimeError
+          serialized_serializer.serialized_args! # Must not raise
+        end
+
+        it 'performs_serialization!' do
+          deserialized_serializer.perform_serialization!
+          deserialized_serializer.serialized_args!.must_equal args
+        end
+
+        it 'performs_deserialization!' do
+          serialized_serializer.perform_deserialization!
+          serialized_serializer.args.must_equal args
+        end
+      end
+
+      describe 'delayed plan' do
+        let(:args) { %w(arg1 arg2) }
+        let(:serializer) { Dynflow::Serializers::Noop.new(nil, args) }
+        let(:delayed_plan) do
+          Dynflow::DelayedPlan.new(Dynflow::World.allocate, 'an uuid', nil, nil, serializer)
+        end
+
+        it "allows access to serializer's args" do
+          serializer.args.must_be :nil?
+          delayed_plan.args.must_equal args
+          serializer.args.must_equal args
+        end
       end
     end
   end

data/test/persistence_test.rb

@@ -6,21 +6,31 @@ module Dynflow
     describe 'persistence adapters' do
 
       let :execution_plans_data do
-        [{ id: 'plan1', :label => 'test1', state: 'paused' },
-         { id: 'plan2', :label => 'test2', state: 'stopped' },
-         { id: 'plan3', :label => 'test3', state: 'paused' },
-         { id: 'plan4', :label => 'test4', state: 'paused' }]
+        [{ id: 'plan1', :label => 'test1', root_plan_step_id: 1, class: 'Dynflow::ExecutionPlan', state: 'paused' },
+         { id: 'plan2', :label => 'test2', root_plan_step_id: 1, class: 'Dynflow::ExecutionPlan', state: 'stopped' },
+         { id: 'plan3', :label => 'test3', root_plan_step_id: 1, class: 'Dynflow::ExecutionPlan', state: 'paused' },
+         { id: 'plan4', :label => 'test4', root_plan_step_id: 1, class: 'Dynflow::ExecutionPlan', state: 'paused' }]
       end
 
       let :action_data do
-        { id: 1, caller_execution_plan_id: nil, caller_action_id: nil }
+        {
+         id: 1,
+         caller_execution_plan_id: nil,
+         caller_action_id: nil,
+         class: 'Dynflow::Action',
+         input: {key: 'value'},
+         output: {something: 'else'},
+         plan_step_id: 1,
+         run_step_id: 2,
+         finalize_step_id: 3
+        }
       end
 
       let :step_data do
         { id: 1,
           state: 'success',
-          started_at: '2015-02-24 10:00',
-          ended_at: '2015-02-24 10:01',
+          started_at: Time.now.utc - 60,
+          ended_at: Time.now.utc - 30,
           real_time: 1.1,
           execution_time: 0.1,
           action_id: 1,
@@ -30,13 +40,15 @@ module Dynflow
 
       def prepare_plans
         execution_plans_data.map do |h|
-          h.merge result:    nil, started_at: (Time.now-20).to_s, ended_at: (Time.now-10).to_s,
+          h.merge result:    nil, started_at: Time.now.utc - 20, ended_at: Time.now.utc - 10,
               real_time: 0.0, execution_time: 0.0
-        end.tap do |plans|
-          plans.each { |plan| adapter.save_execution_plan(plan[:id], plan) }
         end
       end
 
+      def prepare_and_save_plans
+        prepare_plans.each { |plan| adapter.save_execution_plan(plan[:id], plan) }
+      end
+
       def format_time(time)
         time.strftime('%Y-%m-%d %H:%M:%S')
       end
@@ -46,18 +58,38 @@ module Dynflow
       end
 
       def prepare_step(plan)
-        adapter.save_step(plan, step_data[:id], step_data)
+        step = step_data.dup
+        step[:execution_plan_uuid] = plan
+        step
+      end
+
+      def prepare_and_save_step(plan)
+        step = prepare_step(plan)
+        adapter.save_step(plan, step[:id], step)
       end
 
       def prepare_plans_with_actions
-        prepare_plans.each do |plan|
+        prepare_and_save_plans.each do |plan|
           prepare_action(plan[:id])
         end
       end
 
       def prepare_plans_with_steps
         prepare_plans_with_actions.map do |plan|
-          prepare_step(plan[:id])
+          prepare_and_save_step(plan[:id])
+        end
+      end
+
+      def assert_equal_attributes!(original, loaded)
+        original.each do |key, value|
+          loaded_value = loaded[key.to_s]
+          if value.is_a?(Time)
+            loaded_value.inspect.must_equal value.inspect
+          elsif value.is_a?(Hash)
+            assert_equal_attributes!(value, loaded_value)
+          else
+            loaded[key.to_s].must_equal value
+          end
         end
       end
 
@@ -69,7 +101,7 @@ module Dynflow
         end
         describe '#find_execution_plans' do
           it 'supports pagination' do
-            prepare_plans
+            prepare_and_save_plans
             if adapter.pagination?
               loaded_plans = adapter.find_execution_plans(page: 0, per_page: 1)
               loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
@@ -80,7 +112,7 @@ module Dynflow
           end
 
           it 'supports ordering' do
-            prepare_plans
+            prepare_and_save_plans
             if adapter.ordering_by.include?('state')
               loaded_plans = adapter.find_execution_plans(order_by: 'state')
               loaded_plans.map { |h| h[:id] }.must_equal %w(plan1 plan3 plan4 plan2)
@@ -91,7 +123,7 @@ module Dynflow
           end
 
           it 'supports filtering' do
-            prepare_plans
+            prepare_and_save_plans
             if adapter.ordering_by.include?('state')
               loaded_plans = adapter.find_execution_plans(filters: { label: ['test1'] })
               loaded_plans.map { |h| h[:id] }.must_equal ['plan1']
@@ -130,7 +162,7 @@ module Dynflow
           end
 
           it 'supports filtering' do
-            prepare_plans
+            prepare_and_save_plans
             if adapter.ordering_by.include?('state')
               loaded_plans = adapter.find_execution_plan_counts(filters: { label: ['test1'] })
               loaded_plans.must_equal 1
@@ -165,10 +197,12 @@ module Dynflow
         describe '#load_execution_plan and #save_execution_plan' do
           it 'serializes/deserializes the plan data' do
             -> { adapter.load_execution_plan('plan1') }.must_raise KeyError
-            prepare_plans
-            adapter.load_execution_plan('plan1')[:id].must_equal 'plan1'
-            adapter.load_execution_plan('plan1')['id'].must_equal 'plan1'
-            adapter.load_execution_plan('plan1').keys.size.must_equal 8
+            plan = prepare_and_save_plans.first
+            loaded_plan = adapter.load_execution_plan('plan1')
+            loaded_plan[:id].must_equal 'plan1'
+            loaded_plan['id'].must_equal 'plan1'
+
+            assert_equal_attributes!(plan, loaded_plan)
 
             adapter.save_execution_plan('plan1', nil)
             -> { adapter.load_execution_plan('plan1') }.must_raise KeyError
@@ -214,37 +248,60 @@ module Dynflow
 
         describe '#load_action and #save_action' do
           it 'serializes/deserializes the action data' do
-            prepare_plans
+            prepare_and_save_plans
+            action = action_data.dup
             action_id = action_data[:id]
             -> { adapter.load_action('plan1', action_id) }.must_raise KeyError
 
             prepare_action('plan1')
             loaded_action = adapter.load_action('plan1', action_id)
             loaded_action[:id].must_equal action_id
-            loaded_action.must_equal(Utils.stringify_keys(action_data))
+
+            assert_equal_attributes!(action, loaded_action)
 
             adapter.save_action('plan1', action_id, nil)
             -> { adapter.load_action('plan1', action_id) }.must_raise KeyError
 
             adapter.save_execution_plan('plan1', nil)
           end
+
+          it 'allow to retrieve specific attributes using #load_actions_attributes' do
+            prepare_and_save_plans
+            prepare_action('plan1')
+            loaded_data = adapter.load_actions_attributes('plan1', [:id, :run_step_id]).first
+            loaded_data.keys.count.must_equal 2
+            loaded_data[:id].must_equal action_data[:id]
+            loaded_data[:run_step_id].must_equal action_data[:run_step_id]
+          end
+
+          it 'allows to load actions in bulk using #load_actions' do
+            prepare_and_save_plans
+            prepare_action('plan1')
+            action = action_data.dup
+            loaded_actions = adapter.load_actions('plan1', [1])
+            loaded_actions.count.must_equal 1
+            loaded_action = loaded_actions.first
+
+            assert_equal_attributes!(action, loaded_action)
+          end
         end
 
         describe '#load_step and #save_step' do
           it 'serializes/deserializes the step data' do
             prepare_plans_with_actions
             step_id = step_data[:id]
-            prepare_step('plan1')
+            prepare_and_save_step('plan1')
             loaded_step = adapter.load_step('plan1', step_id)
             loaded_step[:id].must_equal step_id
-            loaded_step.must_equal(Utils.stringify_keys(step_data))
+
+            assert_equal_attributes!(step_data, loaded_step)
           end
         end
 
         describe '#find_past_delayed_plans' do
           it 'finds plans with start_before in past' do
-            start_time = Time.now
-            prepare_plans
+            start_time = Time.now.utc
+            prepare_and_save_plans
             adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => format_time(start_time + 60),
                                       :start_before => format_time(start_time - 60))
             adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => format_time(start_time - 60))
@@ -264,15 +321,16 @@ module Dynflow
         it_acts_as_persistence_adapter
 
         it 'allows inspecting the persisted content' do
-          plans = prepare_plans
+          plans = prepare_and_save_plans
 
           plans.each do |original|
             stored = adapter.to_hash.fetch(:execution_plans).find { |ep| ep[:uuid].strip == original[:id] }
-            stored.each { |k, v| stored[k] = v.to_s if v.is_a? Time }
             adapter.class::META_DATA.fetch(:execution_plan).each do |name|
               value = original.fetch(name.to_sym)
               if value.nil?
                 stored.fetch(name.to_sym).must_be_nil
+              elsif value.is_a?(Time)
+                stored.fetch(name.to_sym).inspect.must_equal value.inspect
               else
                 stored.fetch(name.to_sym).must_equal value
               end
@@ -296,6 +354,63 @@ module Dynflow
           assert_equal [], adapter.pull_envelopes(executor_world_id)
         end
 
+        it 'supports reading data saved prior to normalization' do
+          db = adapter.send(:db)
+          # Prepare records for saving
+          plan = prepare_plans.first
+          step_data = prepare_step(plan[:id])
+
+          # We used to store times as strings
+          plan[:started_at] = format_time plan[:started_at]
+          plan[:ended_at] = format_time plan[:ended_at]
+          step_data[:started_at] = format_time step_data[:started_at]
+          step_data[:ended_at] = format_time step_data[:ended_at]
+
+          plan_record = adapter.send(:prepare_record, :execution_plan, plan.merge(:uuid => plan[:id]))
+          action_record = adapter.send(:prepare_record, :action, action_data.dup)
+          step_record = adapter.send(:prepare_record, :step, step_data)
+
+          # Insert the records
+          db[:dynflow_execution_plans].insert plan_record.merge(:uuid => plan[:id])
+          db[:dynflow_actions].insert action_record.merge(:execution_plan_uuid => plan[:id], :id => action_data[:id])
+          db[:dynflow_steps].insert step_record.merge(:execution_plan_uuid => plan[:id], :id => step_data[:id])
+
+          # Load the saved records
+          loaded_plan   = adapter.load_execution_plan(plan[:id])
+          loaded_action = adapter.load_action(plan[:id], action_data[:id])
+          loaded_step   = adapter.load_step(plan[:id], step_data[:id])
+
+          # Test
+          assert_equal_attributes!(plan, loaded_plan)
+          assert_equal_attributes!(action_data, loaded_action)
+          assert_equal_attributes!(step_data, loaded_step)
+        end
+
+        it 'support updating data saved prior to normalization' do
+          db = adapter.send(:db)
+          plan = prepare_plans.first
+          plan_data = plan.dup
+          plan[:started_at] = format_time plan[:started_at]
+          plan[:ended_at] = format_time plan[:ended_at]
+          plan_record = adapter.send(:prepare_record, :execution_plan, plan.merge(:uuid => plan[:id]))
+
+          # Save the plan the old way
+          db[:dynflow_execution_plans].insert plan_record.merge(:uuid => plan[:id])
+
+          # Update and save the plan
+          plan_data[:state] = 'stopped'
+          plan_data[:result] = 'success'
+          adapter.save_execution_plan(plan[:id], plan_data)
+
+          # Check the plan has the changed columns populated
+          raw_plan = db[:dynflow_execution_plans].where(:uuid => 'plan1').first
+          raw_plan[:state].must_equal 'stopped'
+          raw_plan[:result].must_equal 'success'
+
+          # Load the plan and assert it doesn't read attributes from data
+          loaded_plan = adapter.load_execution_plan(plan[:id])
+          assert_equal_attributes!(plan_data, loaded_plan)
+        end
       end
     end
   end