dynflow 0.2.0 → 0.3.0

data/.travis.yml

@@ -4,6 +4,7 @@ language:
 rvm:
   - "1.9.3"
   - "2.0.0"
+  - "2.1.0"
 
 script:
   - bundle exec rake test

data/lib/dynflow.rb

@@ -19,6 +19,7 @@ module Dynflow
   require 'dynflow/stateful'
   require 'dynflow/transaction_adapters'
   require 'dynflow/persistence'
+  require 'dynflow/middleware'
   require 'dynflow/action'
   require 'dynflow/flows'
   require 'dynflow/execution_plan'

data/lib/dynflow/action.rb

@@ -10,6 +10,8 @@ module Dynflow
     require 'dynflow/action/format'
     extend Format
 
+    extend Middleware::Action
+
     require 'dynflow/action/progress'
     include Progress
 
@@ -188,19 +190,19 @@ module Dynflow
 
     private
 
-    ERRORING = Object.new
+    ERROR = Object.new
 
     # DSL to terminate action execution and set it to error
     def error!(error)
       set_error(error)
-      throw ERRORING
+      throw ERROR
     end
 
     def with_error_handling(&block)
       raise "wrong state #{self.state}" unless self.state == :running
 
       begin
-        catch(ERRORING) { block.call }
+        catch(ERROR) { block.call }
       rescue Exception => error
         set_error(error)
         # reraise low-level exceptions

data/lib/dynflow/action/finalize_phase.rb

@@ -10,7 +10,9 @@ module Dynflow
       self.state = :running
       save_state
       with_error_handling do
-        finalize
+        world.middleware.execute(:finalize, self) do
+          finalize
+        end
       end
     end
 

data/lib/dynflow/action/plan_phase.rb

@@ -20,8 +20,10 @@ module Dynflow
       self.state = :running
       save_state
       with_error_handling do
-        execution_plan.switch_flow(Flows::Concurrence.new([])) do
-          plan(*args)
+        concurrence do
+          world.middleware.execute(:plan, self, *args) do |*new_args|
+            plan(*new_args)
+          end
         end
 
         subscribed_actions = world.subscribed_actions(self.action_class)

data/lib/dynflow/action/run_phase.rb

@@ -22,7 +22,9 @@ module Dynflow
         self.state = :running
         save_state
         with_error_handling do
-          result = catch(SUSPEND) { event ? run(event) : run }
+          result = catch(SUSPEND) do
+            world.middleware.execute(:run, self, *Array(event)) { |*args| run(*args) }
+          end
           if result == SUSPEND
             self.state = :suspended
           end

data/lib/dynflow/daemon.rb

@@ -13,6 +13,7 @@ module Dynflow
         terminated = Future.new
         trap('SIGINT') { @world.terminate terminated }
         terminated.wait
+        @listener.terminate.wait
       end
     end
 

data/lib/dynflow/execution_plan.rb

@@ -114,11 +114,13 @@ module Dynflow
     def plan(*args)
       update_state(:planning)
       world.transaction_adapter.transaction do
-        with_planning_scope do
-          root_plan_step.execute(self, nil, *args)
+        world.middleware.execute(:plan_phase, root_plan_step.action_class) do
+          with_planning_scope do
+            root_plan_step.execute(self, nil, *args)
 
-          if @dependency_graph.unresolved?
-            raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
+            if @dependency_graph.unresolved?
+              raise "Some dependencies were not resolved: #{@dependency_graph.inspect}"
+            end
           end
         end
 

data/lib/dynflow/executors/abstract.rb

@@ -1,6 +1,18 @@
 module Dynflow
   module Executors
     class Abstract
+      Event = Algebrick.type do
+        fields! execution_plan_id: String,
+                step_id:           Fixnum,
+                event:             Object,
+                result:            Future
+      end
+
+      Execution = Algebrick.type do
+        fields! execution_plan_id: String,
+                finished:          Future
+      end
+
       include Algebrick::TypeCheck
       attr_reader :world, :logger
 

data/lib/dynflow/executors/parallel.rb

@@ -14,50 +14,31 @@ module Dynflow
 
       UnprocessableEvent = Class.new(Dynflow::Error)
 
-      # actor messages
-      Algebrick.types do
-        Boolean = type { variants TrueClass, FalseClass }
+      Algebrick.type do |work|
+        Work = work
 
-        Execution = type do
-          fields! execution_plan_id: String,
-                  finished:          Future
+        Work::Finalize = type do
+          fields! sequential_manager: SequentialManager,
+                  execution_plan_id:  String
         end
 
-        Event = type do
-          fields! execution_plan_id: String,
-                  step_id:           Fixnum,
-                  event:             Object,
-                  result:            Future
+        Work::Step = type do
+          fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
+                  execution_plan_id: String
         end
 
-        Work = type do |work|
-          work::Finalize = type do
-            fields! sequential_manager: SequentialManager,
-                    execution_plan_id:  String
-          end
-
-          work::Step = type do
-            fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
-                    execution_plan_id: String
-          end
-
-          work::Event = type do
-            fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
-                    execution_plan_id: String,
-                    event:             Event
-          end
-
-          variants work::Step, work::Event, work::Finalize
+        Work::Event = type do
+          fields! step:              ExecutionPlan::Steps::AbstractFlowStep,
+                  execution_plan_id: String,
+                  event:             Event
         end
 
-        PoolDone   = type do
-          fields! work: Work
-        end
-        WorkerDone = type do
-          fields! work: Work, worker: Worker
-        end
+        variants Work::Step, Work::Event, Work::Finalize
       end
 
+      PoolDone   = Algebrick.type { fields! work: Work }
+      WorkerDone = Algebrick.type { fields! work: Work, worker: Worker }
+
       def initialize(world, pool_size = 10)
         super(world)
         @core = Core.new world, pool_size

data/lib/dynflow/executors/parallel/core.rb

@@ -18,11 +18,11 @@ module Dynflow
 
         def on_message(message)
           match message,
-                (on ~Execution do |(execution_plan_id, finished)|
+                (on ~Parallel::Execution do |(execution_plan_id, finished)|
                   start_executing track_execution_plan(execution_plan_id, finished)
                   true
                 end),
-                (on ~Event do |event|
+                (on ~Parallel::Event do |event|
                   event(event)
                 end),
                 (on PoolDone.(~any) do |step|
@@ -40,15 +40,18 @@ module Dynflow
           execution_plan = @world.persistence.load_execution_plan(execution_plan_id)
 
           if terminating?
-            raise Dynflow::Error, "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
+            raise Dynflow::Error,
+                  "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
           end
 
           if @execution_plan_managers[execution_plan_id]
-            raise Dynflow::Error, "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
+            raise Dynflow::Error,
+                  "cannot execute execution_plan_id:#{execution_plan_id} it's already running"
           end
 
           if execution_plan.state == :stopped
-            raise Dynflow::Error, "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
+            raise Dynflow::Error,
+                  "cannot execute execution_plan_id:#{execution_plan_id} it's stopped"
           end
 
           @execution_plan_managers[execution_plan_id] =
@@ -95,14 +98,16 @@ module Dynflow
         end
 
         def event(event)
-          Type! event, Event
+          Type! event, Parallel::Event
           execution_plan_manager = @execution_plan_managers[event.execution_plan_id]
           if execution_plan_manager
             feed_pool execution_plan_manager.event(event)
             true
           else
-            logger.warn "dropping event #{event} - no manager for #{event.execution_plan_id}:#{event.step_id}"
-            event.result.fail UnprocessableEvent.new("no manager for #{event.execution_plan_id}:#{event.step_id}")
+            logger.warn format('dropping event %s - no manager for %s:%s',
+                               event, event.execution_plan_id, event.step_id)
+            event.result.fail UnprocessableEvent.new(
+                                  "no manager for #{event.execution_plan_id}:#{event.step_id}")
           end
         end
 

data/lib/dynflow/executors/parallel/execution_plan_manager.rb

@@ -46,38 +46,30 @@ module Dynflow
             end
           end
 
-          match work,
-
-                Work::Step.(step: ~any) >-> step do
-                  suspended, work = @running_steps_manager.done(step)
-                  if suspended
-                    raise 'assert' unless compute_next_from_step.call(step).empty?
-                    work
-                  else
-                    execution_plan.update_execution_time step.execution_time
-                    compute_next_from_step.call step
-                  end
-                end,
-
-                Work::Event.(step: ~any) >-> step do
-                  suspended, work = @running_steps_manager.done(step)
-
-                  if suspended
-                    work
-                  else
-                    execution_plan.update_execution_time step.execution_time
-                    compute_next_from_step.call step
-                  end
-                end,
-
-                Work::Finalize >-> do
-                  raise unless @finalize_manager
-                  finish
-                end
+          match(work,
+                (on Work::Step.(step: ~any) | Work::Event.(step: ~any) do |step|
+                   execution_plan.steps[step.id] = step
+                   suspended, work = @running_steps_manager.done(step)
+                   unless suspended
+                     execution_plan.update_execution_time step.execution_time
+                     work = compute_next_from_step.call step
+                   end
+                   # TODO: can be probably disabled to improve
+                   # performance, execution time will not be updated,
+                   # maybe more - check on the other side, it allows
+                   # us to use persistence adapter for hooking into
+                   # the running process.
+                   execution_plan.save
+                   work
+                 end),
+                (on Work::Finalize do
+                   raise unless @finalize_manager
+                   finish
+                 end))
         end
 
         def event(event)
-          Type! event, Event
+          Type! event, Parallel::Event
           raise unless event.execution_plan_id == @execution_plan.id
           @running_steps_manager.event(event)
         end

data/lib/dynflow/executors/parallel/flow_manager.rb

@@ -19,9 +19,6 @@ module Dynflow
 
         # @return [Set] of steps to continue with
         def what_is_next(flow_step)
-          execution_plan.steps[flow_step.id] = flow_step
-          # TODO can be probably disabled to improve performance, execution time will not be updated, maybe more - check
-          execution_plan.save
           return [] if flow_step.state == :suspended
 
           success = flow_step.state != :error

data/lib/dynflow/executors/parallel/running_steps_manager.rb

@@ -34,7 +34,8 @@ module Dynflow
             while (event = @events.shift(step.id))
               message = "step #{step.execution_plan_id}:#{step.id} dropping event #{event.event}"
               @world.logger.warn message
-              event.event.result.fail UnprocessableEvent.new(message).tap { |e| e.set_backtrace(caller) }
+              event.event.result.fail UnprocessableEvent.new(message).
+                                          tap { |e| e.set_backtrace(caller) }
             end
             raise 'assert' unless @events.empty?(step.id)
             @running_steps.delete(step.id)
@@ -44,11 +45,12 @@ module Dynflow
 
         # @returns [Work, nil]
         def event(event)
-          Type! event, Event
+          Type! event, Parallel::Event
 
           step = @running_steps[event.step_id]
           unless step
-            event.result.fail UnprocessableEvent.new('step is not suspended, it cannot process events')
+            event.result.fail UnprocessableEvent.new(
+                                  'step is not suspended, it cannot process events')
             return nil
           end
 

data/lib/dynflow/executors/parallel/sequential_manager.rb

@@ -22,8 +22,12 @@ module Dynflow
         reset_finalize_steps
         unless execution_plan.error?
           world.transaction_adapter.transaction do
-            unless dispatch(execution_plan.finalize_flow)
-              world.transaction_adapter.rollback
+            step_id = execution_plan.finalize_flow.all_step_ids.first
+            action_class = execution_plan.steps[step_id].action_class
+            world.middleware.execute(:finalize_phase, action_class) do
+              unless dispatch(execution_plan.finalize_flow)
+                world.transaction_adapter.rollback
+              end
             end
           end
         end

data/lib/dynflow/executors/parallel/worker.rb

@@ -15,12 +15,13 @@ module Dynflow
 
         def on_message(message)
           match message,
-                Work::Step.(step: ~any) | Work::Event.(step: ~any, event: Event.(event: ~any)) >-> step, event do
+                (on Work::Step.(step: ~any) |
+                        Work::Event.(step: ~any, event: Parallel::Event.(event: ~any)) do |step, event|
                   step.execute event
-                end,
-                Work::Finalize.(~any, any) >-> sequential_manager do
+                end),
+                (on Work::Finalize.(~any, any) do |sequential_manager|
                   sequential_manager.finalize
-                end
+                end)
           @pool << WorkerDone[work: message, worker: self]
           @transaction_adapter.cleanup
         end

data/lib/dynflow/executors/remote_via_socket.rb

@@ -15,7 +15,7 @@ module Dynflow
       end
 
       def execute(execution_plan_id, finished = Future.new)
-        @core.ask(Core::Execute[execution_plan_id, finished]).value!.value!
+        @core.ask(Core::Execution[execution_plan_id, finished]).value!.value!
         finished
       rescue => e
         finished.fail e unless finished.ready?
@@ -23,7 +23,8 @@ module Dynflow
       end
 
       def event(execution_plan_id, step_id, event, future = Future)
-        raise 'events are handled in a process with real executor'
+        @core.ask(Core::Event[execution_plan_id, step_id, event, future]).value!
+        future
       end
 
       def terminate(future = Future.new)
@@ -33,6 +34,10 @@ module Dynflow
       def initialized
         @core.initialized
       end
+
+      def connected?
+        @core.ask(Core::Connect).value!
+      end
     end
   end
 end

data/lib/dynflow/executors/remote_via_socket/core.rb

@@ -5,16 +5,22 @@ module Dynflow
         include Listeners::Serialization
 
         Message = Algebrick.type do
+          Job = Algebrick.type do
+            variants Event     = Executors::Abstract::Event,
+                     Execution = Executors::Abstract::Execution
+          end
+
           variants Closed   = atom,
-                   Received = type { fields message: SocketMessage },
-                   Execute  = type { fields execution_plan_uuid: String, future: Future }
+                   Received = type { fields message: Protocol::Response },
+                   Connect  = atom,
+                   Job
         end
 
-        Execution = Algebrick.type do
-          fields! id: Integer, accepted: Future, finished: Future
+        TrackedJob = Algebrick.type do
+          fields! id: Integer, job: Protocol::Job, accepted: Future, finished: Future
         end
 
-        module Execution
+        module TrackedJob
           def accept!
             accepted.resolve true
             self
@@ -26,9 +32,16 @@ module Dynflow
             self
           end
 
-          def success!(value)
+          def success!(world)
             raise unless accepted.ready?
-            finished.resolve value
+            finished.resolve(
+                match job,
+                      (on Core::Protocol::Execution.(execution_plan_id: ~any) do |uuid|
+                        world.persistence.load_execution_plan(uuid)
+                      end),
+                      (on Core::Protocol::Event do
+                        true
+                      end))
             self
           end
 
@@ -49,26 +62,34 @@ module Dynflow
         private
 
         def delayed_initialize(world, socket_path)
-          @socket_path = Type! socket_path, String
-          @world       = Type! world, World
-          @socket      = nil
-          @last_id     = 0
-          @executions  = {}
+          @socket_path  = Type! socket_path, String
+          @world        = Type! world, World
+          @socket       = nil
+          @last_id      = 0
+          @tracked_jobs = {}
           connect
         end
 
         def termination
-          disconnect
+          terminate! if disconnect
         end
 
         def on_message(message)
+          Type! message, Message
           match message,
-
-                (on Core::Execute.(~any, ~any) do |execution_plan_uuid, future|
+                (on ~Job do |job|
                   raise 'terminating' if terminating?
-                  id, accepted = add_execution future
+                  job, future  =
+                      match job,
+                            (on ~Execution do |(execution_plan_uuid, future)|
+                              [Protocol::Execution[execution_plan_uuid], future]
+                            end),
+                            (on ~Event do |(execution_plan_id, step_id, event, future)|
+                              [Protocol::Event[execution_plan_id, step_id, event], future]
+                            end)
+                  id, accepted = add_tracked_job future, job
                   success      = connect && begin
-                    send_message @socket, RemoteViaSocket::Execute[id, execution_plan_uuid]
+                    send_message @socket, Protocol::Do[id, job]
                     true
                   rescue IOError => error
                     logger.warn error
@@ -76,36 +97,43 @@ module Dynflow
                   end
 
                   unless success
-                    @executions[id].reject! Dynflow::Error.new(
-                                                'No connection to RemoteViaSocket::Listener')
+                    @tracked_jobs[id].reject!(
+                        Dynflow::Error.new(
+                            "Cannot do #{message}, no connection to a Listener"))
                   end
 
                   return accepted
                 end),
 
-                (on Received.(Accepted.(~any)) do |id|
-                  @executions[id].accept!
+                (on Received.(~Protocol::Accepted) do |(id)|
+                  @tracked_jobs[id].accept!
                 end),
 
-                (on Received.(Failed.(~any, ~any)) do |id, error|
-                  @executions.delete(id).reject! Dynflow::Error.new(error)
+                (on Received.(~Protocol::Failed) do |(id, error)|
+                  @tracked_jobs.delete(id).reject! Dynflow::Error.new(error)
                 end),
 
-                (on Received.(Done.(~any, ~any)) do |id, uuid|
-                  @executions.delete(id).success! @world.persistence.load_execution_plan(uuid)
+                (on Received.(~Protocol::Done) do |(id)|
+                  @tracked_jobs.delete(id).success! @world
                 end),
 
                 (on Closed do
                   @socket = nil
                   logger.info 'Disconnected from server.'
-                  @executions.each { |_, c| c.fail! 'No connection to RemoteViaSocket::Listener' }
-                  @executions.clear
+                  @tracked_jobs.each do |_, c|
+                    c.fail! 'Connection to a Listener lost.'
+                  end
+                  @tracked_jobs.clear
                   terminate! if terminating?
+                end),
+
+                (on Connect do
+                  connect
                 end)
         end
 
-        def add_execution(finished)
-          @executions[id = (@last_id += 1)] = Execution[id, accepted = Future.new, finished]
+        def add_tracked_job(finished, job)
+          @tracked_jobs[id = (@last_id += 1)] = TrackedJob[id, job, accepted = Future.new, finished]
           return id, accepted
         end
 
@@ -115,14 +143,20 @@ module Dynflow
           logger.info 'Connected to server.'
           read_socket_until_closed
           true
-        rescue IOError => error
+        rescue SystemCallError, IOError => error
           logger.warn error
           false
+        rescue => error
+          logger.fatal error
+          raise error
         end
 
         def disconnect
           return true unless @socket
-          @socket.shutdown :RDWR
+
+          @socket.close
+          false
+        rescue Errno::ENOTCONN
           true
         end
 
@@ -136,7 +170,7 @@ module Dynflow
 
         def read_socket
           match message = receive_message(@socket),
-                SocketMessage >-> { self << Received[message] },
+                Protocol::Message >-> { self << Received[message] },
                 NilClass.to_m >-> do
                   self << Closed
                   throw :stop_reading