dynflow 0.1.0 → 0.2.0

data/lib/dynflow/execution_plan/steps.rb

@@ -0,0 +1,12 @@
+module Dynflow
+  module ExecutionPlan::Steps
+
+    require 'dynflow/execution_plan/steps/error'
+    require 'dynflow/execution_plan/steps/abstract'
+    require 'dynflow/execution_plan/steps/abstract_flow_step'
+    require 'dynflow/execution_plan/steps/plan_step'
+    require 'dynflow/execution_plan/steps/run_step'
+    require 'dynflow/execution_plan/steps/finalize_step'
+
+  end
+end

data/lib/dynflow/execution_plan/steps/abstract.rb

@@ -0,0 +1,121 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class Abstract < Serializable
+      include Algebrick::TypeCheck
+      include Stateful
+
+      attr_reader :execution_plan_id, :id, :state, :action_class, :action_id, :world, :started_at,
+                  :ended_at, :execution_time, :real_time
+      attr_accessor :error
+
+      def initialize(execution_plan_id,
+          id,
+          state,
+          action_class,
+          action_id,
+          error,
+          world,
+          started_at = nil,
+          ended_at = nil,
+          execution_time = 0.0,
+          real_time = 0.0)
+
+        @id                = id || raise(ArgumentError, 'missing id')
+        @execution_plan_id = Type! execution_plan_id, String
+        @world             = Type! world, World
+        @error             = Type! error, ExecutionPlan::Steps::Error, NilClass
+        @started_at        = Type! started_at, Time, NilClass
+        @ended_at          = Type! ended_at, Time, NilClass
+        @execution_time    = Type! execution_time, Float
+        @real_time         = Type! real_time, Float
+
+        self.state = state.to_sym
+
+        Type! action_class, Class
+        raise ArgumentError, 'action_class is not an child of Action' unless action_class < Action
+        raise ArgumentError, 'action_class must not be phase' if action_class.phase?
+        @action_class = action_class
+
+        @action_id = action_id || raise(ArgumentError, 'missing action_id')
+      end
+
+      def action_logger
+        @world.action_logger
+      end
+
+      def phase
+        raise NotImplementedError
+      end
+
+      def persistence
+        world.persistence
+      end
+
+      def save
+        persistence.save_step(self)
+      end
+
+      def self.states
+        @states ||= [:pending, :running, :success, :suspended, :skipped, :error]
+      end
+
+      def execute(*args)
+        raise NotImplementedError
+      end
+
+      def to_s
+        "[#{self.class.name}:#{execution_plan_id}:#{id}]"
+      end
+
+      def to_hash
+        recursive_to_hash execution_plan_id: execution_plan_id,
+                          id:                id,
+                          state:             state,
+                          class:             self.class.to_s,
+                          action_class:      action_class.to_s,
+                          action_id:         action_id,
+                          error:             error,
+                          started_at:        time_to_str(started_at),
+                          ended_at:          time_to_str(ended_at),
+                          execution_time:    execution_time,
+                          real_time:         real_time
+      end
+
+      # @return [Array<[0..100], Fixnum>] the percentage of the step progress
+      # and the weight - how time-consuming the task is comparing the others.
+      # @see [Action::Progress] for more details
+      def progress
+        raise NotImplementedError, "Expected to be implemented in RunStep and FinalizeStep"
+      end
+
+      protected
+
+      def self.new_from_hash(hash, execution_plan_id, world)
+        check_class_matching hash
+        new execution_plan_id,
+            hash[:id],
+            hash[:state],
+            Action.constantize(hash[:action_class]),
+            hash[:action_id],
+            hash_to_error(hash[:error]),
+            world,
+            string_to_time(hash[:started_at]),
+            string_to_time(hash[:ended_at]),
+            hash[:execution_time],
+            hash[:real_time]
+      end
+
+      private
+
+      def with_time_calculation(&block)
+        start       = Time.now
+        @started_at ||= start
+        block.call
+      ensure
+        @ended_at       = Time.now
+        @execution_time += @ended_at - start
+        @real_time      = @ended_at - @started_at
+      end
+    end
+  end
+end

data/lib/dynflow/execution_plan/steps/abstract_flow_step.rb

@@ -0,0 +1,52 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class AbstractFlowStep < Abstract
+
+      def execute(*args)
+        return self if [:skipped, :success].include? self.state
+        open_action do |action|
+          action.indifferent_access_hash_variable_set :input, dereference(action.input)
+          with_time_calculation do
+            action.execute(*args)
+          end
+        end
+      end
+
+      def clone
+        self.class.from_hash(to_hash, execution_plan_id, world)
+      end
+
+
+      def progress
+        action = persistence.load_action(self)
+        [action.progress_done, action.progress_weight]
+      end
+
+      private
+
+      def open_action
+        action = persistence.load_action(self)
+        yield action
+        persistence.save_action(execution_plan_id, action)
+        save
+
+        return self
+      end
+
+      def dereference(input)
+        case input
+        when Hash
+          input.reduce(HashWithIndifferentAccess.new) do |h, (key, val)|
+            h.update(key => dereference(val))
+          end
+        when Array
+          input.map { |val| dereference(val) }
+        when ExecutionPlan::OutputReference
+          input.dereference(persistence, execution_plan_id)
+        else
+          input
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/execution_plan/steps/error.rb

@@ -0,0 +1,33 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class Error < Serializable
+
+      attr_reader :exception_class, :message, :backtrace
+
+      def initialize(exception_class, message, backtrace)
+        @exception_class = exception_class
+        @message         = message
+        @backtrace       = backtrace
+      end
+
+      def self.new_from_hash(hash)
+        self.new(hash[:exception_class], hash[:message], hash[:backtrace])
+      end
+
+      def to_hash
+        { class:           self.class.name,
+          exception_class: exception_class,
+          message:         message,
+          backtrace:       backtrace }
+      end
+
+      def to_s
+        "#{message} (#{exception_class})\n#{(backtrace || []).join("\n")}"
+      end
+
+      def exception
+        exception_class.constantize.exception(message).tap { |e| e.set_backtrace backtrace }
+      end
+    end
+  end
+end

data/lib/dynflow/execution_plan/steps/finalize_step.rb

@@ -0,0 +1,23 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class FinalizeStep < AbstractFlowStep
+
+      def self.state_transitions
+        @state_transitions ||= {
+            pending:   [:running, :skipped], # :skipped when its run_step is skipped
+            running:   [:success, :error],
+            success:   [:pending], # when restarting finalize phase
+            suspended: [],
+            skipped:   [],
+            error:     [:pending, :skipped] # pending when restarting finalize phase
+        }
+      end
+
+
+      def phase
+        :finalize_phase
+      end
+
+    end
+  end
+end

data/lib/dynflow/execution_plan/steps/plan_step.rb

@@ -0,0 +1,81 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class PlanStep < Abstract
+      attr_reader :children
+
+      # @param [Array] children is a private API parameter
+      def initialize(execution_plan_id,
+          id,
+          state,
+          action_class,
+          action_id,
+          error,
+          world,
+          started_at = nil,
+          ended_at = nil,
+          execution_time = 0.0,
+          real_time = 0.0,
+          children = [])
+
+        super execution_plan_id, id, state, action_class, action_id, error, world, started_at,
+              ended_at, execution_time, real_time
+        children.all? { |child| Type! child, Integer }
+        @children = children
+      end
+
+      def phase
+        :plan_phase
+      end
+
+      def to_hash
+        super.merge recursive_to_hash(:children => children)
+      end
+
+      # @return [Action]
+      def execute(execution_plan, trigger, *args)
+        Type! execution_plan, ExecutionPlan
+        attributes = { execution_plan_id: execution_plan.id,
+                       id:                action_id,
+                       step:              self,
+                       plan_step_id:      self.id }
+        action     = action_class.plan_phase.new(attributes, execution_plan, trigger)
+        persistence.save_action(execution_plan_id, action)
+
+        with_time_calculation do
+          action.execute(*args)
+        end
+
+        execution_plan.update_execution_time execution_time
+
+        persistence.save_action(execution_plan_id, action)
+        return action
+      end
+
+      def self.state_transitions
+        @state_transitions ||= { pending:   [:running],
+                                 running:   [:success, :error],
+                                 success:   [],
+                                 suspended: [],
+                                 skipped:   [],
+                                 error:     [] }
+      end
+
+
+      def self.new_from_hash(hash, execution_plan_id, world)
+        check_class_matching hash
+        new execution_plan_id,
+            hash[:id],
+            hash[:state],
+            Action.constantize(hash[:action_class]),
+            hash[:action_id],
+            hash_to_error(hash[:error]),
+            world,
+            string_to_time(hash[:started_at]),
+            string_to_time(hash[:ended_at]),
+            hash[:execution_time],
+            hash[:real_time],
+            hash[:children]
+      end
+    end
+  end
+end

data/lib/dynflow/execution_plan/steps/run_step.rb

@@ -0,0 +1,21 @@
+module Dynflow
+  module ExecutionPlan::Steps
+    class RunStep < AbstractFlowStep
+
+      def self.state_transitions
+        @state_transitions ||= {
+            pending:   [:running, :skipped], # :skipped when it cannot be run because it depends on skipped step
+            running:   [:success, :error, :suspended],
+            success:   [:suspended], # after not-done process_update
+            suspended: [:running, :error], # process_update, e.g. error in setup_progress_updates
+            skipped:   [],
+            error:     [:skipped, :running]
+        }
+      end
+
+      def phase
+        :run_phase
+      end
+    end
+  end
+end

data/lib/dynflow/executors.rb

@@ -0,0 +1,9 @@
+module Dynflow
+  module Executors
+
+    require 'dynflow/executors/abstract'
+    require 'dynflow/executors/parallel'
+    require 'dynflow/executors/remote_via_socket'
+
+  end
+end

data/lib/dynflow/executors/abstract.rb

@@ -0,0 +1,32 @@
+module Dynflow
+  module Executors
+    class Abstract
+      include Algebrick::TypeCheck
+      attr_reader :world, :logger
+
+      def initialize(world)
+        @world  = Type! world, World
+        @logger = world.logger
+      end
+
+      # @return [Future]
+      # @raise when execution_plan_id is not accepted
+      def execute(execution_plan_id)
+        raise NotImplementedError
+      end
+
+      def event(execution_plan_id, step_id, event, future = Future)
+        raise NotImplementedError
+      end
+
+      def terminate(future = Future.new)
+        raise NotImplementedError
+      end
+
+      # @return [Future]
+      def initialized
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/dynflow/executors/parallel.rb

@@ -0,0 +1,88 @@
+module Dynflow
+  module Executors
+    class Parallel < Abstract
+
+      require 'dynflow/executors/parallel/sequence_cursor'
+      require 'dynflow/executors/parallel/flow_manager'
+      require 'dynflow/executors/parallel/work_queue'
+      require 'dynflow/executors/parallel/execution_plan_manager'
+      require 'dynflow/executors/parallel/sequential_manager'
+      require 'dynflow/executors/parallel/running_steps_manager'
+      require 'dynflow/executors/parallel/core'
+      require 'dynflow/executors/parallel/pool'
+      require 'dynflow/executors/parallel/worker'
+
+      UnprocessableEvent = Class.new(Dynflow::Error)
+
+      # actor messages
+      Algebrick.types do
+        Boolean = type { variants TrueClass, FalseClass }
+
+        Execution = type do
+          fields! execution_plan_id: String,
+                  finished:          Future
+        end
+
+        Event = type do
+          fields! execution_plan_id: String,
+                  step_id:           Fixnum,
+                  event:             Object,
+                  result:            Future
+        end
+
+        Work = type do |work|
+          work::Finalize = type do
+            fields! sequential_manager: SequentialManager,
+                    execution_plan_id:  String
+          end
+
+          work::Step = type do
+            fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
+                    execution_plan_id: String
+          end
+
+          work::Event = type do
+            fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
+                    execution_plan_id: String,
+                    event:             Event
+          end
+
+          variants work::Step, work::Event, work::Finalize
+        end
+
+        PoolDone   = type do
+          fields! work: Work
+        end
+        WorkerDone = type do
+          fields! work: Work, worker: Worker
+        end
+      end
+
+      def initialize(world, pool_size = 10)
+        super(world)
+        @core = Core.new world, pool_size
+      end
+
+      def execute(execution_plan_id, finished = Future.new)
+        @core.ask(Execution[execution_plan_id, finished]).value!
+        finished
+      rescue => e
+        finished.fail e unless finished.ready?
+        raise e
+      end
+
+      def event(execution_plan_id, step_id, event, future = Future.new)
+        @core << Event[execution_plan_id, step_id, event, future]
+        future
+      end
+
+      def terminate(future = Future.new)
+        @core.ask(MicroActor::Terminate, future)
+      end
+
+      def initialized
+        @core.initialized
+      end
+    end
+  end
+end