dynflow 0.1.0 → 0.2.0

data/lib/dynflow/executors/parallel/sequence_cursor.rb

@@ -0,0 +1,97 @@
+module Dynflow
+  module Executors
+    class Parallel < Abstract
+      class SequenceCursor
+
+        def initialize(flow_manager, sequence, parent_cursor = nil)
+          @flow_manager    = flow_manager
+          @sequence        = sequence
+          @parent_cursor   = parent_cursor
+          @todo            = []
+          @index           = -1 # starts before first element
+          @no_error_so_far = true
+        end
+
+        # @param [ExecutionPlan::Steps::Abstract, SequenceCursor] work
+        #   step or sequence cursor that was done
+        # @param [true, false] success was the work finished successfully
+        # @return [Array<Integer>] new step_ids that can be done next
+        def what_is_next(work = nil, success = true)
+          unless work.nil? || @todo.delete(work)
+            raise "marking as done work that was not expected: #{work.inspect}"
+          end
+
+          @no_error_so_far &&= success
+
+          if done_here?
+            return next_steps
+          else
+            return []
+          end
+        end
+
+
+        # return true if we can't move the cursor further, either when
+        # everyting is done in the sequence or there was some failure
+        # that prevents us from moving
+        def done?
+          (!@no_error_so_far && done_here?) || @index == @sequence.size
+        end
+
+        protected
+
+        # steps we can do right now without waiting for anything
+        def steps_todo
+          @todo.map do |item|
+            case item
+            when SequenceCursor
+              item.steps_todo
+            else
+              item
+            end
+          end.flatten
+        end
+
+        def move
+          @index    += 1
+          next_flow = @sequence.sub_flows[@index]
+          add_todo(next_flow)
+        end
+
+        private
+
+        def done_here?
+          @todo.empty?
+        end
+
+        def next_steps
+          move if @no_error_so_far
+          if done?
+            if @parent_cursor
+              return @parent_cursor.what_is_next(self, @no_error_so_far)
+            else
+              return []
+            end
+          else
+            return steps_todo
+          end
+        end
+
+        def add_todo(flow)
+          case flow
+          when Flows::Sequence
+            @todo << SequenceCursor.new(@flow_manager, flow, self).tap do |cursor|
+              cursor.move
+            end
+          when Flows::Concurrence
+            flow.sub_flows.each { |sub_flow| add_todo(sub_flow) }
+          when Flows::Atom
+            @flow_manager.cursor_index[flow.step_id] = self
+            @todo << @flow_manager.execution_plan.steps[flow.step_id]
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/dynflow/executors/parallel/sequential_manager.rb

@@ -0,0 +1,81 @@
+module Dynflow
+  module Executors
+    class Parallel::SequentialManager
+      attr_reader :execution_plan, :world
+
+      def initialize(world, execution_plan)
+        @world          = world
+        @execution_plan = execution_plan
+        @done           = false
+      end
+
+      def run
+        with_state_updates do
+          dispatch(execution_plan.run_flow)
+          finalize
+        end
+
+        return execution_plan
+      end
+
+      def finalize
+        reset_finalize_steps
+        unless execution_plan.error?
+          world.transaction_adapter.transaction do
+            unless dispatch(execution_plan.finalize_flow)
+              world.transaction_adapter.rollback
+            end
+          end
+        end
+        @done = true
+      end
+
+      def reset_finalize_steps
+        execution_plan.finalize_flow.all_step_ids.each do |step_id|
+          step       = execution_plan.steps[step_id]
+          step.state = :pending if [:success, :error].include? step.state
+        end
+      end
+
+      def done?
+        @done
+      end
+
+      private
+
+      def dispatch(flow)
+        case flow
+        when Flows::Sequence
+          run_in_sequence(flow.flows)
+        when Flows::Concurrence
+          run_in_concurrence(flow.flows)
+        when Flows::Atom
+          run_step(execution_plan.steps[flow.step_id])
+        else
+          raise ArgumentError, "Don't know how to run #{flow}"
+        end
+      end
+
+      def run_in_sequence(steps)
+        steps.all? { |s| dispatch(s) }
+      end
+
+      def run_in_concurrence(steps)
+        run_in_sequence(steps)
+      end
+
+      def run_step(step)
+        step.execute
+        execution_plan.update_execution_time step.execution_time
+        execution_plan.save
+        return step.state != :error
+      end
+
+      def with_state_updates(&block)
+        execution_plan.update_state(:running)
+        block.call
+        execution_plan.update_state(execution_plan.error? ? :paused : :stopped)
+      end
+    end
+  end
+end

data/lib/dynflow/executors/parallel/work_queue.rb

@@ -0,0 +1,44 @@
+module Dynflow
+  module Executors
+    class Parallel < Abstract
+      class WorkQueue
+        include Algebrick::TypeCheck
+
+        def initialize(key_type = Object, work_type = Object)
+          @key_type  = key_type
+          @work_type = work_type
+          @stash     = Hash.new { |hash, key| hash[key] = [] }
+        end
+
+        def push(key, work)
+          Type! key, @key_type
+          Type! work, @work_type
+          @stash[key].push work
+        end
+
+        def shift(key)
+          return nil unless present? key
+          @stash[key].shift.tap { |work| @stash.delete(key) if @stash[key].empty? }
+        end
+
+        def present?(key)
+          @stash.key?(key)
+        end
+
+        def empty?(key)
+          !present?(key)
+        end
+
+        def size(key)
+          return 0 if empty?(key)
+          @stash[key].size
+        end
+
+        def first(key)
+          return nil if empty?(key)
+          @stash[key].first
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/parallel/worker.rb

@@ -0,0 +1,30 @@
+module Dynflow
+  module Executors
+    class Parallel < Abstract
+      class Worker < MicroActor
+        def initialize(pool, transaction_adapter)
+          super(pool.logger, pool, transaction_adapter)
+        end
+
+        private
+
+        def delayed_initialize(pool, transaction_adapter)
+          @pool                = pool
+          @transaction_adapter = Type! transaction_adapter, TransactionAdapters::Abstract
+        end
+
+        def on_message(message)
+          match message,
+                Work::Step.(step: ~any) | Work::Event.(step: ~any, event: Event.(event: ~any)) >-> step, event do
+                  step.execute event
+                end,
+                Work::Finalize.(~any, any) >-> sequential_manager do
+                  sequential_manager.finalize
+                end
+          @pool << WorkerDone[work: message, worker: self]
+          @transaction_adapter.cleanup
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/remote_via_socket.rb

@@ -0,0 +1,38 @@
+require 'multi_json'
+require 'socket'
+
+module Dynflow
+  module Executors
+    class RemoteViaSocket < Abstract
+      require 'dynflow/executors/remote_via_socket/core'
+
+      include Listeners::Serialization
+      include Algebrick::Matching
+
+      def initialize(world, socket_path)
+        super world
+        @core = Core.new world, socket_path
+      end
+
+      def execute(execution_plan_id, finished = Future.new)
+        @core.ask(Core::Execute[execution_plan_id, finished]).value!.value!
+        finished
+      rescue => e
+        finished.fail e unless finished.ready?
+        raise e
+      end
+
+      def event(execution_plan_id, step_id, event, future = Future)
+        raise 'events are handled in a process with real executor'
+      end
+
+      def terminate(future = Future.new)
+        @core.ask(MicroActor::Terminate, future)
+      end
+
+      def initialized
+        @core.initialized
+      end
+    end
+  end
+end

data/lib/dynflow/executors/remote_via_socket/core.rb

@@ -0,0 +1,150 @@
+module Dynflow
+  module Executors
+    class RemoteViaSocket < Abstract
+      class Core < MicroActor
+        include Listeners::Serialization
+
+        Message = Algebrick.type do
+          variants Closed   = atom,
+                   Received = type { fields message: SocketMessage },
+                   Execute  = type { fields execution_plan_uuid: String, future: Future }
+        end
+
+        Execution = Algebrick.type do
+          fields! id: Integer, accepted: Future, finished: Future
+        end
+
+        module Execution
+          def accept!
+            accepted.resolve true
+            self
+          end
+
+          def reject!(error)
+            accepted.fail error
+            finished.fail error
+            self
+          end
+
+          def success!(value)
+            raise unless accepted.ready?
+            finished.resolve value
+            self
+          end
+
+          def fail!(error)
+            if accepted.ready?
+              finished.fail error
+            else
+              reject! error
+            end
+            self
+          end
+        end
+
+        def initialize(world, socket_path)
+          super(world.logger, world, socket_path)
+        end
+
+        private
+
+        def delayed_initialize(world, socket_path)
+          @socket_path = Type! socket_path, String
+          @world       = Type! world, World
+          @socket      = nil
+          @last_id     = 0
+          @executions  = {}
+          connect
+        end
+
+        def termination
+          disconnect
+        end
+
+        def on_message(message)
+          match message,
+
+                (on Core::Execute.(~any, ~any) do |execution_plan_uuid, future|
+                  raise 'terminating' if terminating?
+                  id, accepted = add_execution future
+                  success      = connect && begin
+                    send_message @socket, RemoteViaSocket::Execute[id, execution_plan_uuid]
+                    true
+                  rescue IOError => error
+                    logger.warn error
+                    false
+                  end
+
+                  unless success
+                    @executions[id].reject! Dynflow::Error.new(
+                                                'No connection to RemoteViaSocket::Listener')
+                  end
+
+                  return accepted
+                end),
+
+                (on Received.(Accepted.(~any)) do |id|
+                  @executions[id].accept!
+                end),
+
+                (on Received.(Failed.(~any, ~any)) do |id, error|
+                  @executions.delete(id).reject! Dynflow::Error.new(error)
+                end),
+
+                (on Received.(Done.(~any, ~any)) do |id, uuid|
+                  @executions.delete(id).success! @world.persistence.load_execution_plan(uuid)
+                end),
+
+                (on Closed do
+                  @socket = nil
+                  logger.info 'Disconnected from server.'
+                  @executions.each { |_, c| c.fail! 'No connection to RemoteViaSocket::Listener' }
+                  @executions.clear
+                  terminate! if terminating?
+                end)
+        end
+
+        def add_execution(finished)
+          @executions[id = (@last_id += 1)] = Execution[id, accepted = Future.new, finished]
+          return id, accepted
+        end
+
+        def connect
+          return true if @socket
+          @socket = UNIXSocket.new @socket_path
+          logger.info 'Connected to server.'
+          read_socket_until_closed
+          true
+        rescue IOError => error
+          logger.warn error
+          false
+        end
+
+        def disconnect
+          return true unless @socket
+          @socket.shutdown :RDWR
+          true
+        end
+
+        def read_socket_until_closed
+          Thread.new do
+            catch(:stop_reading) do
+              loop { read_socket }
+            end
+          end
+        end
+
+        def read_socket
+          match message = receive_message(@socket),
+                SocketMessage >-> { self << Received[message] },
+                NilClass.to_m >-> do
+                  self << Closed
+                  throw :stop_reading
+                end
+        rescue => error
+          logger.fatal error
+        end
+      end
+    end
+  end
+end