dynflow 0.7.9 → 0.8.0

data/lib/dynflow/coordinator_adapters.rb

@@ -0,0 +1,8 @@
+module Dynflow
+  module CoordinatorAdapters
+
+    require 'dynflow/coordinator_adapters/abstract'
+    require 'dynflow/coordinator_adapters/sequel'
+
+  end
+end

data/lib/dynflow/coordinator_adapters/abstract.rb

@@ -0,0 +1,28 @@
+module Dynflow
+  module CoordinatorAdapters
+    class Abstract
+      include Algebrick::TypeCheck
+
+      def initialize(world)
+        Type! world, World
+        @world = world
+      end
+
+      def create_record(record)
+        raise NotImplementedError
+      end
+
+      def update_record(record)
+        raise NotImplementedError
+      end
+
+      def delete_record(record)
+        raise NotImplementedError
+      end
+
+      def find_records(record)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/dynflow/coordinator_adapters/sequel.rb

@@ -0,0 +1,29 @@
+module Dynflow
+  module CoordinatorAdapters
+    class Sequel < Abstract
+      def initialize(world)
+        super
+        @sequel_adapter = world.persistence.adapter
+        Type! @sequel_adapter, PersistenceAdapters::Sequel
+      end
+
+      def create_record(record)
+        @sequel_adapter.insert_coordinator_record(record.to_hash)
+      rescue ::Sequel::UniqueConstraintViolation
+        raise Coordinator::DuplicateRecordError.new(record)
+      end
+
+      def update_record(record)
+        @sequel_adapter.update_coordinator_record(record.class.name, record.id, record.to_hash)
+      end
+
+      def delete_record(record)
+        @sequel_adapter.delete_coordinator_record(record.class.name, record.id)
+      end
+
+      def find_records(filter_options)
+        @sequel_adapter.find_coordinator_records(filters: filter_options)
+      end
+    end
+  end
+end

data/lib/dynflow/dispatcher.rb

@@ -0,0 +1,58 @@
+module Dynflow
+  module Dispatcher
+    Request = Algebrick.type do
+      Event = type do
+        fields! execution_plan_id: String,
+                step_id:           Fixnum,
+                event:             Object
+      end
+
+      Execution = type do
+        fields! execution_plan_id: String
+      end
+
+      Ping = type do
+        fields! receiver_id: String
+      end
+
+      variants Event, Execution, Ping
+    end
+
+    Response = Algebrick.type do
+      variants Accepted = atom,
+               Failed   = type { fields! error: String },
+               Done     = atom,
+               Pong     = atom
+    end
+
+    Envelope = Algebrick.type do
+      fields! request_id: Integer,
+              sender_id: String,
+              receiver_id: type { variants String, AnyExecutor = atom, UnknownWorld = atom },
+              message: type { variants Request, Response }
+    end
+
+    module Envelope
+      def build_response_envelope(response_message, sender)
+        Envelope[self.request_id,
+                 sender.id,
+                 self.sender_id,
+                 response_message]
+      end
+    end
+
+    module Event
+      def to_hash
+        super.update event: Base64.strict_encode64(Marshal.dump(event))
+      end
+
+      def self.product_from_hash(hash)
+        super(hash.merge 'event' => Marshal.load(Base64.strict_decode64(hash.fetch('event'))))
+      end
+    end
+  end
+end
+
+require 'dynflow/dispatcher/abstract'
+require 'dynflow/dispatcher/client_dispatcher'
+require 'dynflow/dispatcher/executor_dispatcher'

data/lib/dynflow/dispatcher/abstract.rb

@@ -0,0 +1,14 @@
+module Dynflow
+  module Dispatcher
+    class Abstract < Actor
+      def connector
+        @world.connector
+      end
+
+      def respond(request_envelope, response)
+        response_envelope = request_envelope.build_response_envelope(response, @world)
+        connector.send(response_envelope)
+      end
+    end
+  end
+end

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -0,0 +1,139 @@
+module Dynflow
+  module Dispatcher
+    class ClientDispatcher < Abstract
+
+      TrackedRequest = Algebrick.type do
+        fields! id: Integer, request: Request,
+                accepted: Concurrent::Edge::Future, finished: Concurrent::Edge::Future
+      end
+
+      module TrackedRequest
+        def accept!
+          accepted.success true unless accepted.completed?
+          self
+        end
+
+        def fail!(error)
+          accepted.fail error unless accepted.completed?
+          finished.fail error
+          self
+        end
+
+        def success!(resolve_to)
+          accepted.success true unless accepted.completed?
+          finished.success(resolve_to)
+          self
+        end
+      end
+
+      def initialize(world)
+        @world            = Type! world, World
+        @last_id          = 0
+        @tracked_requests = {}
+        @terminated       = nil
+      end
+
+      def publish_request(future, request, timeout)
+        track_request(future, request, timeout) do |tracked_request|
+          dispatch_request(request, @world.id, tracked_request.id)
+        end
+      end
+
+      def timeout(request_id)
+        resolve_tracked_request(request_id, Dynflow::Error.new("Request timeout"))
+      end
+
+      def start_termination(*args)
+        super
+        @tracked_requests.values.each { |tracked_request| tracked_request.fail!(Dynflow::Error.new('Dispatcher terminated')) }
+        @tracked_requests.clear
+        finish_termination
+      end
+
+      def dispatch_request(request, client_world_id, request_id)
+        executor_id = match request,
+            (on ~Execution do |execution|
+               AnyExecutor
+             end),
+            (on ~Event do |event|
+               find_executor(event.execution_plan_id)
+             end),
+            (on Ping.(~any) do |receiver_id|
+               receiver_id
+             end)
+        request = Envelope[request_id, client_world_id, executor_id, request]
+        if Dispatcher::UnknownWorld === request.receiver_id
+          raise Dynflow::Error, "Could not find an executor for #{request}"
+        end
+        connector.send(request).value!
+      rescue => e
+        respond(request, Failed[e.message])
+      end
+
+      def dispatch_response(envelope)
+        return unless @tracked_requests.key?(envelope.request_id)
+        match envelope.message,
+            (on ~Accepted do
+               @tracked_requests[envelope.request_id].accept!
+             end),
+            (on ~Failed do |msg|
+               resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
+             end),
+            (on Done | Pong do
+               resolve_tracked_request(envelope.request_id)
+             end)
+      end
+
+      private
+
+      def find_executor(execution_plan_id)
+        execution_lock = @world.coordinator.find_locks(class: Coordinator::ExecutionLock.name,
+                                                       id: "execution-plan:#{execution_plan_id}").first
+        if execution_lock
+          execution_lock.world_id
+        else
+          Dispatcher::UnknownWorld
+        end
+      end
+
+      def track_request(finished, request, timeout)
+        id = @last_id += 1
+        tracked_request = TrackedRequest[id, request, Concurrent.future, finished]
+        @tracked_requests[id] = tracked_request
+        @world.clock.ping(self, timeout, [:timeout, id]) if timeout
+        yield tracked_request
+      rescue Dynflow::Error => e
+        resolve_tracked_request(tracked_request.id, e)
+        log(Logger::ERROR, e)
+      end
+
+      def reset_tracked_request(tracked_request)
+        if tracked_request.finished.completed?
+          raise Dynflow::Error.new('Can not reset resolved tracked request')
+        end
+        unless tracked_request.accepted.completed?
+          tracked_request.accept! # otherwise nobody would set the accept future
+        end
+        @tracked_requests[tracked_request.id] = TrackedRequest[tracked_request.id, tracked_request.request, Concurrent.future, tracked_request.finished]
+      end
+
+      def resolve_tracked_request(id, error = nil)
+        return unless @tracked_requests.key?(id)
+        if error
+          @tracked_requests.delete(id).fail! error
+        else
+          tracked_request = @tracked_requests[id]
+          resolve_to = match tracked_request.request,
+              (on Execution.(execution_plan_id: ~any) do |uuid|
+                 @world.persistence.load_execution_plan(uuid)
+               end),
+              (on Event | Ping do
+                 true
+               end)
+          @tracked_requests.delete(id).success! resolve_to
+        end
+      end
+
+    end
+  end
+end

data/lib/dynflow/dispatcher/executor_dispatcher.rb

@@ -0,0 +1,86 @@
+module Dynflow
+  module Dispatcher
+    class ExecutorDispatcher < Abstract
+      def initialize(world)
+        @world        = Type! world, World
+        @current_futures = Set.new
+      end
+
+      def handle_request(envelope)
+        match(envelope.message,
+              on(Execution) { perform_execution(envelope, envelope.message) },
+              on(Event)     { perform_event(envelope, envelope.message) })
+      end
+
+      protected
+
+      def perform_execution(envelope, execution)
+        allocate_executor(execution.execution_plan_id, envelope.sender_id, envelope.request_id)
+        execution_lock = Coordinator::ExecutionLock.new(@world, execution.execution_plan_id, envelope.sender_id, envelope.request_id)
+        future = on_finish do |f|
+          f.then do |plan|
+            if plan.state == :running
+              @world.invalidate_execution_lock(execution_lock)
+            else
+              @world.coordinator.release(execution_lock)
+              respond(envelope, Done)
+            end
+          end.rescue do |reason|
+            @world.coordinator.release(execution_lock)
+            respond(envelope, Failed[reason.to_s])
+          end
+        end
+        @world.executor.execute(execution.execution_plan_id, future)
+        respond(envelope, Accepted)
+      rescue Dynflow::Error => e
+        future.fail(e) if future && !future.completed?
+        respond(envelope, Failed[e.message])
+      end
+
+      def perform_event(envelope, event_request)
+        future = on_finish do |f|
+          f.then do
+            respond(envelope, Done)
+          end.rescue do |reason|
+            respond(envelope, Failed[reason.to_s])
+          end
+        end
+        @world.executor.event(event_request.execution_plan_id, event_request.step_id, event_request.event, future)
+      rescue Dynflow::Error => e
+        future.fail(e) if future && !future.completed?
+      end
+
+      def start_termination(*args)
+        super
+        if @current_futures.empty?
+          reference.tell(:finish_termination)
+        else
+          Concurrent.zip(*@current_futures).then { reference.tell(:finish_termination) }
+        end
+      end
+
+      private
+
+      def allocate_executor(execution_plan_id, client_world_id, request_id)
+        execution_lock = Coordinator::ExecutionLock.new(@world, execution_plan_id, client_world_id, request_id)
+        @world.coordinator.acquire(execution_lock)
+      end
+
+      def on_finish
+        raise "Dispatcher terminating: no new work can be started" if terminating?
+        future = Concurrent.future
+        callbacks_future = (yield future).rescue { |reason| @world.logger.error("Unexpected fail on future #{reason}") }
+        # we track currently running futures to make sure to not
+        # terminate until the execution is finished (including
+        # cleaning of locks etc)
+        @current_futures << callbacks_future
+        callbacks_future.on_completion! { reference.tell([:finish_execution, callbacks_future]) }
+        return future
+      end
+
+      def finish_execution(future)
+        @current_futures.delete(future)
+      end
+    end
+  end
+end

data/lib/dynflow/errors.rb

@@ -25,7 +25,13 @@ module Dynflow
       end
     end
 
-    class PersistenceError < StandardError
+    class InactiveWorldError < Dynflow::Error
+      def initialize(world)
+        super("The world #{world.id} is not active (terminating or terminated)")
+      end
+    end
+
+    class PersistenceError < Dynflow::Error
       def self.delegate(original_exception)
         self.new("caused by #{original_exception.class}: #{original_exception.message}").tap do |e|
           e.set_backtrace original_exception.backtrace

data/lib/dynflow/execution_history.rb

@@ -0,0 +1,46 @@
+module Dynflow
+  class ExecutionHistory
+    include Algebrick::TypeCheck
+    include Enumerable
+
+    Event = Algebrick.type do
+      fields! time:     Integer,
+              name:     String,
+              world_id: type { variants String, NilClass }
+    end
+
+    module Event
+      def inspect
+        "#{Time.at(time).utc}: #{name}".tap { |s| s << " @ #{world_id}" if world_id }
+      end
+    end
+
+    attr_reader :events
+
+    def initialize(events = [])
+      @events = (events || []).each { |e| Type! e, Event }
+    end
+
+    def each(&block)
+      @events.each(&block)
+    end
+
+    def add(name, world_id = nil)
+      @events << Event[Time.now.to_i, name, world_id]
+    end
+
+    def to_hash
+      @events.map(&:to_hash)
+    end
+
+    def inspect
+      "ExecutionHistory: #{ @events.inspect }"
+    end
+
+    def self.new_from_hash(value)
+      value ||= [] # for compatibility with tasks before the
+      # introduction of execution history
+      self.new(value.map { |hash| Event[hash] })
+    end
+  end
+end

data/lib/dynflow/execution_plan.rb

@@ -13,7 +13,7 @@ module Dynflow
     require 'dynflow/execution_plan/dependency_graph'
 
     attr_reader :id, :world, :root_plan_step, :steps, :run_flow, :finalize_flow,
-                :started_at, :ended_at, :execution_time, :real_time
+                :started_at, :ended_at, :execution_time, :real_time, :execution_history
 
     def self.states
       @states ||= [:pending, :planning, :planned, :running, :paused, :stopped]
@@ -39,18 +39,20 @@ module Dynflow
         started_at = nil,
         ended_at = nil,
         execution_time = nil,
-        real_time = 0.0)
-
-      @id             = Type! id, String
-      @world          = Type! world, World
-      self.state      = state
-      @run_flow       = Type! run_flow, Flows::Abstract
-      @finalize_flow  = Type! finalize_flow, Flows::Abstract
-      @root_plan_step = root_plan_step
-      @started_at     = Type! started_at, Time, NilClass
-      @ended_at       = Type! ended_at, Time, NilClass
-      @execution_time = Type! execution_time, Numeric, NilClass
-      @real_time      = Type! real_time, Numeric
+        real_time = 0.0,
+        execution_history = ExecutionHistory.new)
+
+      @id                = Type! id, String
+      @world             = Type! world, World
+      self.state         = state
+      @run_flow          = Type! run_flow, Flows::Abstract
+      @finalize_flow     = Type! finalize_flow, Flows::Abstract
+      @root_plan_step    = root_plan_step
+      @started_at        = Type! started_at, Time, NilClass
+      @ended_at          = Type! ended_at, Time, NilClass
+      @execution_time    = Type! execution_time, Numeric, NilClass
+      @real_time         = Type! real_time, Numeric
+      @execution_history = Type! execution_history, ExecutionHistory
 
       steps.all? do |k, v|
         Type! k, Integer
@@ -274,7 +276,8 @@ module Dynflow
                         started_at:        time_to_str(started_at),
                         ended_at:          time_to_str(ended_at),
                         execution_time:    execution_time,
-                        real_time:         real_time
+                        real_time:         real_time,
+                        execution_history: execution_history.to_hash
     end
 
     def save
@@ -295,7 +298,8 @@ module Dynflow
                string_to_time(hash[:started_at]),
                string_to_time(hash[:ended_at]),
                hash[:execution_time].to_f,
-               hash[:real_time].to_f)
+               hash[:real_time].to_f,
+               ExecutionHistory.new_from_hash(hash[:execution_history]))
     end
 
     def compute_execution_time