dynflow 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e36392c5ebe5f3a20b879eca68d46ff0ee715985d7107af64f69b4ef256bf19e
-  data.tar.gz: 3177acf84d594013c47f5a6413fc56f47dbb121727c5549948ef2f30ec2099de
+  metadata.gz: 6ee35eec200e14b25add8941b4b4637e994012053a271898c4abc4a70234942e
+  data.tar.gz: 720fc9161e5aadff8f165c12f6bb278cfa65435ef7a1d96d93df8614d21a1de8
 SHA512:
-  metadata.gz: 41e469d615fb041a1323b6d512f31b57aafe55de76b35212b1d615e249822214aeda51f5710221932d4d7e78aae9515f500f80278b7d056f42403c24a28cdd31
-  data.tar.gz: b7a202cf192a9d7af21e556971876867f180eb2a6319cd624075af854bb2a64ed0cd12435076261dc71aad4439ff01ebc3c2216799f85d0dbff30e0d495658d7
+  metadata.gz: 1753d21be5307643a16704a27b0e343fe55a7e74330469d1eece503d58cfa9fd04be1da57918e6e09e73f7e2822f2be2bd517e9eea58b89432bee7226d51ab93
+  data.tar.gz: ac12ce027be3e289227db4a2b5311328342de364c0f5d51f5badc27e7ba484a6f137488ce681d319e050b1a04ef55ac4821d0d87a06d6d70019f3b2c6e889fca

data/lib/dynflow/config.rb

@@ -98,13 +98,21 @@ module Dynflow
     end
 
     config_attr :executor, Executors::Abstract, FalseClass do |world, config|
-      Executors::Parallel.new(world, config.queues)
+      Executors::Parallel.new(world, config.executor_heartbeat_interval, config.queues)
     end
 
     config_attr :executor_semaphore, Semaphores::Abstract, FalseClass do |world, config|
       Semaphores::Dummy.new
     end
 
+    config_attr :executor_heartbeat_interval, Integer do
+      15
+    end
+
+    config_attr :ping_cache_age, Integer do
+      60
+    end
+
     config_attr :connector, Connectors::Abstract do |world|
       Connectors::Direct.new(world)
     end
@@ -118,7 +126,7 @@ module Dynflow
     end
 
     config_attr :validity_check_timeout, Numeric do
-      5
+      30
     end
 
     config_attr :exit_on_terminate, Algebrick::Types::Boolean do

data/lib/dynflow/delayed_executors/abstract.rb

@@ -7,11 +7,18 @@ module Dynflow
       def initialize(world, options = {})
         @world = world
         @options = options
+        @started = false
         spawn
       end
 
+      def started?
+        @started
+      end
+
       def start
-        @core.ask(:start)
+        @core.ask(:start).tap do
+          @started = true
+        end
       end
 
       def terminate

data/lib/dynflow/delayed_executors/abstract_core.rb

@@ -46,6 +46,7 @@ module Dynflow
       def process(delayed_plans, check_time)
         processed_plan_uuids = []
         delayed_plans.each do |plan|
+          next if plan.frozen
           fix_plan_state(plan)
           with_error_handling do
             if plan.execution_plan.state != :scheduled

data/lib/dynflow/delayed_plan.rb

@@ -3,14 +3,16 @@ module Dynflow
 
     include Algebrick::TypeCheck
 
-    attr_reader :execution_plan_uuid, :start_at, :start_before
+    attr_reader :execution_plan_uuid, :start_before
+    attr_accessor :frozen, :start_at
 
-    def initialize(world, execution_plan_uuid, start_at, start_before, args_serializer)
+    def initialize(world, execution_plan_uuid, start_at, start_before, args_serializer, frozen)
       @world               = Type! world, World
       @execution_plan_uuid = Type! execution_plan_uuid, String
       @start_at            = Type! start_at, Time, NilClass
       @start_before        = Type! start_before, Time, NilClass
       @args_serializer     = Type! args_serializer, Serializers::Abstract
+      @frozen              = Type! frozen, Algebrick::Types::Boolean
     end
 
     def execution_plan
@@ -55,7 +57,8 @@ module Dynflow
                         :start_at            => @start_at,
                         :start_before        => @start_before,
                         :serialized_args     => @args_serializer.serialized_args,
-                        :args_serializer     => @args_serializer.class.name
+                        :args_serializer     => @args_serializer.class.name,
+                        :frozen              => @frozen
     end
 
     # Retrieves arguments from the serializer
@@ -73,7 +76,8 @@ module Dynflow
                hash[:execution_plan_uuid],
                string_to_time(hash[:start_at]),
                string_to_time(hash[:start_before]),
-               serializer)
+               serializer,
+               hash[:frozen] || false)
     rescue NameError => e
       error(e.message)
     end

data/lib/dynflow/dispatcher.rb

@@ -12,7 +12,8 @@ module Dynflow
       end
 
       Ping = type do
-        fields! receiver_id: String
+        fields! receiver_id: String,
+                use_cache: type { variants TrueClass, FalseClass }
       end
 
       Status = type do

data/lib/dynflow/dispatcher/client_dispatcher.rb

@@ -26,16 +26,96 @@ module Dynflow
         end
       end
 
-      def initialize(world)
+      # Class used for reducing the number of sent Pings among worlds.
+      # World's coordinator record include the time when was the world
+      # seen for the last time. This class can be used to query this
+      # information and determine whether the record is "fresh enough"
+      # or whether the Ping really needs to be sent.
+      class PingCache
+        # Format string used for formating and parsing times
+        TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%L'.freeze
+        DEFAULT_MAX_AGE = 60
+
+        # Formats time into a string
+        #
+        # @param time [Time] the time to format
+        # @return [String] the formatted time
+        def self.format_time(time = Time.now)
+          time.strftime(TIME_FORMAT)
+        end
+
+        # Parses time from a string
+        #
+        # @param time [String] the time string to parse
+        # @return [Time] the parsed time
+        def self.load_time(time)
+          Time.strptime(time, TIME_FORMAT)
+        end
+
+        # @param world [World] the world to which the PingCache belongs
+        def initialize(world, max_age = DEFAULT_MAX_AGE)
+          @world = world
+          @max_age = max_age
+          @executor = {}
+        end
+
+        # Records when was the world seen into the world's coordinator record
+        #
+        # @param id [String] Id of the world to be added to the cache
+        # @param time [Time] Time when was the world last seen
+        def add_record(id, time = Time.now)
+          record = find_world id
+          @executor[id] ||= record.data[:class] == 'Dynflow::Coordinator::ExecutorWorld'
+          record.data[:meta].update(:last_seen => self.class.format_time(time))
+          @world.coordinator.update_record(record)
+        end
+
+        # Looks into the cache whether the world has an executor
+        #
+        # @param id [String] Id of the world
+        # @return [TrueClass] if the world has an executor
+        # @return [FalseClass] if the world is a client world
+        # @return [NilClass] if unknown
+        def executor?(id)
+          @executor[id]
+        end
+
+        # Loads the coordinator record from the database and checks whether the world
+        # was last seen within the time limit
+        #
+        # @param id [String] Id of the world to be checked
+        # @return [TrueClass] if the world was last seen within the limit
+        # @return [FalseClass] if the world was last seen after the limit passed
+        def fresh_record?(id)
+          record = find_world(id)
+          return false if record.nil?
+          @executor[id] = record.data[:class] == 'Dynflow::Coordinator::ExecutorWorld'
+          time = self.class.load_time(record.data[:meta][:last_seen])
+          time >= Time.now - @max_age
+        end
+
+        private
+
+        def find_world(id)
+          @world.coordinator.find_records(:id => id,
+                                          :class => ['Dynflow::Coordinator::ExecutorWorld', 'Dynflow::Coordinator::ClientWorld']).first
+        end
+      end
+
+      attr_reader :ping_cache
+      def initialize(world, ping_cache_age)
         @world            = Type! world, World
         @last_id          = 0
         @tracked_requests = {}
         @terminated       = nil
+        @ping_cache       = PingCache.new world, ping_cache_age
       end
 
       def publish_request(future, request, timeout)
-        track_request(future, request, timeout) do |tracked_request|
-          dispatch_request(request, @world.id, tracked_request.id)
+        with_ping_request_caching(request, future) do
+          track_request(future, request, timeout) do |tracked_request|
+            dispatch_request(request, @world.id, tracked_request.id)
+          end
         end
       end
 
@@ -58,7 +138,7 @@ module Dynflow
                             (on ~Event do |event|
                                find_executor(event.execution_plan_id)
                              end),
-                            (on Ping.(~any) | Status.(~any, ~any) do |receiver_id, _|
+                            (on Ping.(~any, ~any) | Status.(~any, ~any) do |receiver_id, _|
                                receiver_id
                              end)
         envelope = Envelope[request_id, client_world_id, executor_id, request]
@@ -80,7 +160,11 @@ module Dynflow
               (on ~Failed do |msg|
                  resolve_tracked_request(envelope.request_id, Dynflow::Error.new(msg.error))
                end),
-              (on Done | Pong do
+              (on Done do
+                 resolve_tracked_request(envelope.request_id)
+               end),
+              (on Pong do
+                 add_ping_cache_record(envelope.sender_id)
                  resolve_tracked_request(envelope.request_id)
                end),
               (on ExecutionStatus.(~any) do |steps|
@@ -88,6 +172,15 @@ module Dynflow
                end)
       end
 
+      # Records when was the world with provided id last seen using a PingCache
+      #
+      # @param id [String] Id of the world
+      # @see PingCache#add_record
+      def add_ping_cache_record(id)
+        log Logger::DEBUG, "adding ping cache record for #{id}"
+        @ping_cache.add_record id
+      end
+
       private
 
       def find_executor(execution_plan_id)
@@ -141,6 +234,27 @@ module Dynflow
         end
       end
 
+      # Tries to reduce the number of sent Ping requests by first looking into a cache. If the
+      # destination world is an executor world, the result is resolved solely from the cache.
+      # For client worlds the Ping might be sent if the cache record is stale.
+      #
+      # @param request [Dynflow::Dispatcher::Request] the request to send
+      # @param future [Concurrent::Future] the future to fulfill if the world was seen recently
+      # @return [Concurrent::Future] the future tracking the request
+      def with_ping_request_caching(request, future)
+        return yield unless request.is_a?(Dynflow::Dispatcher::Ping)
+        return yield unless request.use_cache
+
+        if @ping_cache.fresh_record?(request.receiver_id)
+          future.success(true)
+        else
+          if @ping_cache.executor?(request.receiver_id)
+            future.fail
+          else
+            yield
+          end
+        end
+      end
     end
   end
 end

data/lib/dynflow/execution_plan.rb

@@ -252,7 +252,8 @@ module Dynflow
                                      id,
                                      delay_options[:start_at],
                                      delay_options.fetch(:start_before, nil),
-                                     serializer)
+                                     serializer,
+                                     delay_options[:frozen] || false)
       persistence.save_delayed_plan(delayed_plan)
     ensure
       update_state(error? ? :stopped : :scheduled)

data/lib/dynflow/executors/parallel.rb

@@ -5,10 +5,10 @@ module Dynflow
       require 'dynflow/executors/parallel/pool'
       require 'dynflow/executors/parallel/worker'
 
-      def initialize(world, queues_options = { :default => { :pool_size => 5 }})
+      def initialize(world, heartbeat_interval, queues_options = { :default => { :pool_size => 5 }})
         super(world)
         @core = Core.spawn name:        'parallel-executor-core',
-                           args:        [world, queues_options],
+                           args:        [world, heartbeat_interval, queues_options],
                            initialized: @core_initialized = Concurrent.future
       end
 

data/lib/dynflow/executors/parallel/core.rb

@@ -4,15 +4,17 @@ module Dynflow
       class Core < Actor
         attr_reader :logger
 
-        def initialize(world, queues_options)
+        def initialize(world, heartbeat_interval, queues_options)
           @logger         = world.logger
           @world          = Type! world, World
           @queues_options = queues_options
           @pools          = {}
           @terminated     = nil
           @director       = Director.new(@world)
+          @heartbeat_interval = heartbeat_interval
 
           initialize_queues
+          schedule_heartbeat
         end
 
         def initialize_queues
@@ -78,8 +80,27 @@ module Dynflow
           end
         end
 
+        def heartbeat
+          @logger.debug('Executor heartbeat')
+          record = @world.coordinator.find_records(:id => @world.id,
+                                                   :class => ['Dynflow::Coordinator::ExecutorWorld', 'Dynflow::Coordinator::ClientWorld']).first
+          unless record
+            logger.error(%{Executor's world record for #{@world.id} missing: terminating})
+            @world.terminate
+            return
+          end
+
+          record.data[:meta].update(:last_seen => Dynflow::Dispatcher::ClientDispatcher::PingCache.format_time)
+          @world.coordinator.update_record(record)
+          schedule_heartbeat
+        end
+
         private
 
+        def schedule_heartbeat
+          @world.clock.ping(self, @heartbeat_interval, :heartbeat)
+        end
+
         def on_message(message)
           super
         rescue Errors::PersistenceError => e

data/lib/dynflow/persistence.rb

@@ -93,6 +93,13 @@ module Dynflow
       adapter.save_delayed_plan(delayed_plan.execution_plan_uuid, delayed_plan.to_hash)
     end
 
+    def set_delayed_plan_frozen(execution_plan_id, frozen = true, new_start_at = nil)
+      plan = load_delayed_plan(execution_plan_id)
+      plan.frozen = frozen
+      plan.start_at = new_start_at if new_start_at
+      save_delayed_plan(plan)
+    end
+
     def load_delayed_plan(execution_plan_id)
       hash = adapter.load_delayed_plan(execution_plan_id)
       return nil unless hash

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -36,7 +36,7 @@ module Dynflow
                                             class action_class execution_plan_uuid queue),
                     envelope:            %w(receiver_id),
                     coordinator_record:  %w(id owner_id class),
-                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer)}
+                    delayed:             %w(execution_plan_uuid start_at start_before args_serializer frozen)}
 
       SERIALIZABLE_COLUMNS = { action:  %w(input output),
                                delayed: %w(serialized_args),
@@ -125,6 +125,7 @@ module Dynflow
         table_name = :delayed
         table(table_name)
           .where(::Sequel.lit('start_at <= ? OR (start_before IS NOT NULL AND start_before <= ?)', time, time))
+          .where(:frozen => false)
           .order_by(:start_at)
           .all
           .map { |plan| load_data(plan, table_name) }

data/lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:dynflow_delayed_plans) do
+      add_column :frozen, :boolean
+    end
+    self[:dynflow_delayed_plans].update(:frozen => false)
+  end
+end

data/lib/dynflow/rails.rb

@@ -42,8 +42,9 @@ module Dynflow
           config.run_on_init_hooks(world)
           # leave this just for long-running executors
           unless config.rake_task_with_executor?
-            world.perform_validity_checks
+            invalidated_worlds = world.perform_validity_checks
             world.auto_execute
+            world.post_initialization if invalidated_worlds > 0
           end
         end
       end

data/lib/dynflow/rails/configuration.rb

@@ -155,7 +155,9 @@ module Dynflow
         if remote?
           false
         else
-          ::Dynflow::Executors::Parallel.new(world, world.config.queues)
+          ::Dynflow::Executors::Parallel.new(world,
+                                             world.config.executor_heartbeat_interval,
+                                             world.config.queues)
         end
       end
 

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '1.0.5'.freeze
+  VERSION = '1.1.0'.freeze
 end

data/lib/dynflow/web/console.rb

@@ -50,14 +50,14 @@ module Dynflow
       end
 
       post('/worlds/check') do
-        load_worlds
         @validation_results = world.worlds_validity_check(params[:invalidate])
+        load_worlds
         erb :worlds
       end
 
       post('/worlds/:id/check') do |id|
-        load_worlds
         @validation_results = world.worlds_validity_check(params[:invalidate], id: params[:id])
+        load_worlds
         erb :worlds
       end
 

data/lib/dynflow/world.rb

@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
+require 'dynflow/world/invalidation'
+
 module Dynflow
   class World
     include Algebrick::TypeCheck
     include Algebrick::Matching
+    include Invalidation
 
     attr_reader :id, :config, :client_dispatcher, :executor_dispatcher, :executor, :connector,
                 :transaction_adapter, :logger_adapter, :coordinator,
@@ -28,7 +31,7 @@ module Dynflow
       @connector              = @config.connector
       @middleware             = Middleware::World.new
       @middleware.use Middleware::Common::Transaction if @transaction_adapter
-      @client_dispatcher      = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self)
+      @client_dispatcher      = spawn_and_wait(Dispatcher::ClientDispatcher, "client-dispatcher", self, @config.ping_cache_age)
       @dead_letter_handler    = spawn_and_wait(DeadLetterSilencer, 'default_dead_letter_handler', @config.silent_dead_letter_matchers)
       @auto_validity_check    = @config.auto_validity_check
       @validity_check_timeout = @config.validity_check_timeout
@@ -41,15 +44,9 @@ module Dynflow
         @executor_dispatcher = spawn_and_wait(Dispatcher::ExecutorDispatcher, "executor-dispatcher", self, @config.executor_semaphore)
         executor.initialized.wait
       end
+      update_register
       perform_validity_checks if auto_validity_check
 
-      @delayed_executor         = try_spawn(:delayed_executor, Coordinator::DelayedExecutorLock)
-      @execution_plan_cleaner   = try_spawn(:execution_plan_cleaner, Coordinator::ExecutionPlanCleanerLock)
-      @meta                     = @config.meta
-      @meta['queues']           = @config.queues if @executor
-      @meta['delayed_executor'] = true if @delayed_executor
-      @meta['execution_plan_cleaner'] = true if @execution_plan_cleaner
-      coordinator.register_world(registered_world)
       @termination_barrier = Mutex.new
       @before_termination_hooks = Queue.new
 
@@ -59,14 +56,38 @@ module Dynflow
           self.terminate.wait
         end
       end
+      post_initialization
+    end
+
+    # performs steps once the executor is ready and invalidation of previous worls is finished.
+    # Needs to be indempotent, as it can be called several times (expecially when auto_validity_check
+    # if false, as it should be called after `perform_validity_checks` method)
+    def post_initialization
+      @delayed_executor ||= try_spawn(:delayed_executor, Coordinator::DelayedExecutorLock)
+      @execution_plan_cleaner ||= try_spawn(:execution_plan_cleaner, Coordinator::ExecutionPlanCleanerLock)
+      update_register
+      @delayed_executor.start if @delayed_executor && !@delayed_executor.started?
       self.auto_execute if @config.auto_execute
-      @delayed_executor.start if @delayed_executor
     end
 
     def before_termination(&block)
       @before_termination_hooks << block
     end
 
+    def update_register
+      @meta                     ||= @config.meta
+      @meta['queues']           = @config.queues if @executor
+      @meta['delayed_executor'] = true if @delayed_executor
+      @meta['execution_plan_cleaner'] = true if @execution_plan_cleaner
+      @meta['last_seen'] = Dynflow::Dispatcher::ClientDispatcher::PingCache.format_time
+      if @already_registered
+        coordinator.update_record(registered_world)
+      else
+        coordinator.register_world(registered_world)
+        @already_registered = true
+      end
+    end
+
     def registered_world
       if executor
         Coordinator::ExecutorWorld.new(self)
@@ -193,7 +214,11 @@ module Dynflow
     end
 
     def ping(world_id, timeout, done = Concurrent.future)
-      publish_request(Dispatcher::Ping[world_id], done, false, timeout)
+      publish_request(Dispatcher::Ping[world_id, true], done, false, timeout)
+    end
+
+    def ping_without_cache(world_id, timeout, done = Concurrent.future)
+      publish_request(Dispatcher::Ping[world_id, false], done, false, timeout)
     end
 
     def get_execution_status(world_id, execution_plan_id, timeout, done = Concurrent.future)
@@ -270,121 +295,7 @@ module Dynflow
       defined?(@terminating)
     end
 
-    # Invalidate another world, that left some data in the runtime,
-    # but it's not really running
-    def invalidate(world)
-      Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
-      coordinator.acquire(Coordinator::WorldInvalidationLock.new(self, world)) do
-        if world.is_a? Coordinator::ExecutorWorld
-          old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
-                                                       owner_id: "world:#{world.id}")
-
-          coordinator.deactivate_world(world)
-
-          old_execution_locks.each do |execution_lock|
-            invalidate_execution_lock(execution_lock)
-          end
-        end
-
-        coordinator.delete_world(world)
-      end
-    end
-
-    def invalidate_execution_lock(execution_lock)
-      begin
-        plan = persistence.load_execution_plan(execution_lock.execution_plan_id)
-      rescue => e
-        if e.is_a?(KeyError)
-          logger.error "invalidated execution plan #{execution_lock.execution_plan_id} missing, skipping"
-        else
-          logger.error e
-          logger.error "unexpected error when invalidating execution plan #{execution_lock.execution_plan_id}, skipping"
-        end
-        coordinator.release(execution_lock)
-        coordinator.release_by_owner(execution_lock.execution_plan_id)
-        return
-      end
-      unless plan.valid?
-        logger.error "invalid plan #{plan.id}, skipping"
-        coordinator.release(execution_lock)
-        coordinator.release_by_owner(execution_lock.execution_plan_id)
-        return
-      end
-      plan.execution_history.add('terminate execution', execution_lock.world_id)
-
-      plan.steps.values.each do |step|
-        if step.state == :running
-          step.error = ExecutionPlan::Steps::Error.new("Abnormal termination (previous state: #{step.state})")
-          step.state = :error
-          step.save
-        end
-      end
-
-      plan.update_state(:paused) if plan.state == :running
-      plan.save
-      coordinator.release(execution_lock)
-
-      available_executors = coordinator.find_worlds(true)
-      if available_executors.any? && !plan.error?
-        client_dispatcher.tell([:dispatch_request,
-                                Dispatcher::Execution[execution_lock.execution_plan_id],
-                                execution_lock.client_world_id,
-                                execution_lock.request_id])
-      end
-    rescue Errors::PersistenceError
-      logger.error "failed to write data while invalidating execution lock #{execution_lock}"
-    end
-
-    def perform_validity_checks
-      worlds_validity_check
-      locks_validity_check
-    end
-
-    def worlds_validity_check(auto_invalidate = true, worlds_filter = {})
-      worlds = coordinator.find_worlds(false, worlds_filter)
-
-      world_checks = worlds.reduce({}) do |hash, world|
-        hash.update(world => ping(world.id, self.validity_check_timeout))
-      end
-      world_checks.values.each(&:wait)
-
-      results = {}
-      world_checks.each do |world, check|
-        if check.success?
-          result = :valid
-        else
-          if auto_invalidate
-            begin
-              invalidate(world)
-              result = :invalidated
-            rescue => e
-              logger.error e
-              result = e.message
-            end
-          else
-            result = :invalid
-          end
-        end
-        results[world.id] = result
-      end
-
-      unless results.values.all? { |result| result == :valid }
-        logger.error "invalid worlds found #{results.inspect}"
-      end
-
-      return results
-    end
-
-    def locks_validity_check
-      orphaned_locks = coordinator.clean_orphaned_locks
-
-      unless orphaned_locks.empty?
-        logger.error "invalid coordinator locks found and invalidated: #{orphaned_locks.inspect}"
-      end
-
-      return orphaned_locks
-    end
-
+    # 24119 - ensure delayed executor is preserved after invalidation
     # executes plans that are planned/paused and haven't reported any error yet (usually when no executor
     # was available by the time of planning or terminating)
     def auto_execute

data/lib/dynflow/world/invalidation.rb

@@ -0,0 +1,160 @@
+module Dynflow
+  class World
+    module Invalidation
+      # Invalidate another world, that left some data in the runtime,
+      # but it's not really running
+      #
+      # @param world [Coordinator::ClientWorld, Coordinator::ExecutorWorld] coordinator record
+      #   left behind by the world we're trying to invalidate
+      # @return [void]
+      def invalidate(world)
+        Type! world, Coordinator::ClientWorld, Coordinator::ExecutorWorld
+        coordinator.acquire(Coordinator::WorldInvalidationLock.new(self, world)) do
+          if world.is_a? Coordinator::ExecutorWorld
+            old_execution_locks = coordinator.find_locks(class: Coordinator::ExecutionLock.name,
+                                                         owner_id: "world:#{world.id}")
+
+            coordinator.deactivate_world(world)
+
+            old_execution_locks.each do |execution_lock|
+              invalidate_execution_lock(execution_lock)
+            end
+          end
+
+          coordinator.delete_world(world)
+        end
+      end
+
+      # Invalidate an execution lock, left behind by a executor that
+      # was executing an execution plan when it was terminated.
+      #
+      # @param execution_lock [Coordinator::ExecutionLock] the lock to invalidate
+      # @return [void]
+      def invalidate_execution_lock(execution_lock)
+        with_valid_execution_plan_for_lock(execution_lock) do |plan|
+          plan.execution_history.add('terminate execution', execution_lock.world_id)
+
+          plan.steps.values.each do |step|
+            if step.state == :running
+              step.error = ExecutionPlan::Steps::Error.new("Abnormal termination (previous state: #{step.state})")
+              step.state = :error
+              step.save
+            end
+          end
+
+          plan.update_state(:paused) if plan.state == :running
+          plan.save
+          coordinator.release(execution_lock)
+
+          if plan.error?
+            rescue_id = plan.rescue_plan_id
+            execute(rescue_id) if rescue_id
+          else
+            if coordinator.find_worlds(true).any? # Check if there are any executors
+              client_dispatcher.tell([:dispatch_request,
+                                      Dispatcher::Execution[execution_lock.execution_plan_id],
+                                      execution_lock.client_world_id,
+                                      execution_lock.request_id])
+            end
+          end
+        end
+      rescue Errors::PersistenceError
+        logger.error "failed to write data while invalidating execution lock #{execution_lock}"
+      end
+
+      # Tries to load an execution plan using id stored in the
+      # lock. If the execution plan cannot be loaded or is invalid,
+      # the lock is released. If the plan gets loaded successfully, it
+      # is yielded to a given block.
+      #
+      # @param execution_lock [Coordinator::ExecutionLock] the lock for which we're trying
+      #   to load the execution plan
+      # @yieldparam [ExecutionPlan] execution_plan the successfully loaded execution plan
+      # @return [void]
+      def with_valid_execution_plan_for_lock(execution_lock)
+        begin
+          plan = persistence.load_execution_plan(execution_lock.execution_plan_id)
+        rescue => e
+          if e.is_a?(KeyError)
+            logger.error "invalidated execution plan #{execution_lock.execution_plan_id} missing, skipping"
+          else
+            logger.error e
+            logger.error "unexpected error when invalidating execution plan #{execution_lock.execution_plan_id}, skipping"
+          end
+          coordinator.release(execution_lock)
+          coordinator.release_by_owner(execution_lock.execution_plan_id)
+          return
+        end
+        unless plan.valid?
+          logger.error "invalid plan #{plan.id}, skipping"
+          coordinator.release(execution_lock)
+          coordinator.release_by_owner(execution_lock.execution_plan_id)
+          return
+        end
+        yield plan
+      end
+
+      # Performs world validity checks
+      #
+      # @return [Integer] number of invalidated worlds
+      def perform_validity_checks
+        world_invalidation_result = worlds_validity_check
+        locks_validity_check
+        world_invalidation_result.values.select { |result| result == :invalidated }.size
+      end
+
+      # Checks if all worlds are valid and optionally invalidates them
+      #
+      # @param auto_invalidate [Boolean] whether automatic invalidation should be performed
+      # @param worlds_filter [Hash] hash of filters to select only matching worlds
+      # @return [Hash{String=>Symbol}] hash containg validation results, mapping world id to a result
+      def worlds_validity_check(auto_invalidate = true, worlds_filter = {})
+        worlds = coordinator.find_worlds(false, worlds_filter)
+
+        world_checks = worlds.reduce({}) do |hash, world|
+          hash.update(world => ping_without_cache(world.id, self.validity_check_timeout))
+        end
+        world_checks.values.each(&:wait)
+
+        results = {}
+        world_checks.each do |world, check|
+          if check.success?
+            result = :valid
+          else
+            if auto_invalidate
+              begin
+                invalidate(world)
+                result = :invalidated
+              rescue => e
+                logger.error e
+                result = e.message
+              end
+            else
+              result = :invalid
+            end
+          end
+          results[world.id] = result
+        end
+
+        unless results.values.all? { |result| result == :valid }
+          logger.error "invalid worlds found #{results.inspect}"
+        end
+
+        return results
+      end
+
+      # Cleans up locks which don't have a resource
+      #
+      # @return [Array<Coordinator::Lock>] the removed locks
+      def locks_validity_check
+        orphaned_locks = coordinator.clean_orphaned_locks
+
+        unless orphaned_locks.empty?
+          logger.error "invalid coordinator locks found and invalidated: #{orphaned_locks.inspect}"
+        end
+
+        return orphaned_locks
+      end
+    end
+  end
+end

data/test/abnormal_states_recovery_test.rb

@@ -72,6 +72,33 @@ module Dynflow
             end
           end
 
+          it "honors rescue strategy when invalidating execution locks" do
+            coordinator = executor_world_2.coordinator
+            # Plan and action
+            plan = client_world.plan(Support::DummyExample::SkippableDummy)
+            plan.update_state :running
+            plan.save
+
+            # Simulate leaving behind an execution lock for it
+            lock = Coordinator::ExecutionLock.new(executor_world, plan.id, client_world.id, 0)
+            coordinator.acquire(lock)
+
+            # Simulate abnormal termination
+            step = plan.steps.values.last
+            step.state = :error
+            step.save
+
+            # Invalidate the world's lock
+            world_lock = coordinator.find_worlds(false, :id => executor_world.id).first
+            executor_world_2.invalidate(world_lock)
+
+            wait_for do
+              plan = executor_world_2.persistence.load_execution_plan(plan.id)
+              step = plan.steps.values.last
+              plan.state == :stopped && step.state == :skipped
+            end
+          end
+
           it "prevents from running the invalidation twice on the same world" do
             client_world.invalidate(executor_world.registered_world)
             expected_locks = ["lock world-invalidation:#{executor_world.id}",
@@ -194,7 +221,7 @@ module Dynflow
             client_world_config = Config::ForWorld.new(Config.new.tap { |c| c.executor = false }, create_world )
             client_world_config.auto_validity_check.must_equal false
 
-            executor_world_config = Config::ForWorld.new(Config.new.tap { |c| c.executor = lambda { |w, _| Executors::Parallel.new(w) } }, create_world )
+            executor_world_config = Config::ForWorld.new(Config.new.tap { |c| c.executor = lambda { |w, _| Executors::Parallel.new(w, 15) } }, create_world )
             executor_world_config.auto_validity_check.must_equal true
           end
 

data/test/daemon_test.rb

@@ -17,7 +17,7 @@ class DaemonTest < ActiveSupport::TestCase
     @dummy_world = ::Dynflow::Testing::DummyWorld.new
     @dummy_world.stubs(:id => '123')
     @dummy_world.stubs(:auto_execute)
-    @dummy_world.stubs(:perform_validity_checks)
+    @dummy_world.stubs(:perform_validity_checks => 0)
     @event = Concurrent.event
     @dummy_world.stubs(:terminated).returns(@event)
     @world_class.stubs(:new).returns(@dummy_world)
@@ -42,7 +42,14 @@ class DaemonTest < ActiveSupport::TestCase
     @event.wait
   end
 
-  test 'run command works withou memory_limit option specified' do
+  test 'run command works without memory_limit option specified' do
+    @daemon.run(@current_folder)
+    @dynflow.initialize!
+  end
+
+  test 'runs post_initialization when there are invalid worlds detected' do
+    @dummy_world.stubs(:perform_validity_checks => 1)
+    @dummy_world.expects(:post_initialization)
     @daemon.run(@current_folder)
     @dynflow.initialize!
   end

data/test/dispatcher_test.rb

@@ -76,12 +76,41 @@ module Dynflow
             assert ping_response.success?
           end
 
+          it 'succeeds when the world is available without cache' do
+            ping_response = client_world.ping_without_cache(executor_world.id, 0.5)
+            ping_response.wait
+            assert ping_response.success?
+          end
+
           it 'time-outs when the world is not responding' do
             executor_world.terminate.wait
             ping_response = client_world.ping(executor_world.id, 0.5)
             ping_response.wait
             assert ping_response.failed?
           end
+
+          it 'time-outs when the world is not responding without cache' do
+            executor_world.terminate.wait
+            ping_response = client_world.ping_without_cache(executor_world.id, 0.5)
+            ping_response.wait
+            assert ping_response.failed?
+          end
+
+          it 'caches the pings and pongs' do
+            # Spawn the worlds
+            client_world
+            executor_world
+
+            ping_cache = Dynflow::Dispatcher::ClientDispatcher::PingCache.new(executor_world)
+
+            # Records are fresh because of the heartbeat
+            assert ping_cache.fresh_record?(client_world.id)
+            assert ping_cache.fresh_record?(executor_world.id)
+
+            # Expire the record
+            ping_cache.add_record(executor_world.id, Time.now - 1000)
+            refute ping_cache.fresh_record?(executor_world.id)
+          end
         end
       end
 

data/test/future_execution_test.rb

@@ -177,7 +177,7 @@ module Dynflow
         let(:args) { %w(arg1 arg2) }
         let(:serializer) { Dynflow::Serializers::Noop.new(nil, args) }
         let(:delayed_plan) do
-          Dynflow::DelayedPlan.new(Dynflow::World.allocate, 'an uuid', nil, nil, serializer)
+          Dynflow::DelayedPlan.new(Dynflow::World.allocate, 'an uuid', nil, nil, serializer, false)
         end
 
         it "allows access to serializer's args" do

data/test/persistence_test.rb

@@ -302,16 +302,30 @@ module Dynflow
           it 'finds plans with start_before in past' do
             start_time = Time.now.utc
             prepare_and_save_plans
-            adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :start_at => format_time(start_time + 60),
+            adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :frozen => false, :start_at => format_time(start_time + 60),
                                       :start_before => format_time(start_time - 60))
-            adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :start_at => format_time(start_time - 60))
-            adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :start_at => format_time(start_time + 60))
-            adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :start_at => format_time(start_time - 60),
+            adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :frozen => false, :start_at => format_time(start_time - 60))
+            adapter.save_delayed_plan('plan3', :execution_plan_uuid => 'plan3', :frozen => false, :start_at => format_time(start_time + 60))
+            adapter.save_delayed_plan('plan4', :execution_plan_uuid => 'plan4', :frozen => false, :start_at => format_time(start_time - 60),
                                       :start_before => format_time(start_time - 60))
             plans = adapter.find_past_delayed_plans(start_time)
             plans.length.must_equal 3
             plans.map { |plan| plan[:execution_plan_uuid] }.must_equal %w(plan2 plan4 plan1)
           end
+
+          it 'does not find plans that are frozen' do
+            start_time = Time.now.utc
+            prepare_and_save_plans
+
+            adapter.save_delayed_plan('plan1', :execution_plan_uuid => 'plan1', :frozen => false, :start_at => format_time(start_time + 60),
+                                      :start_before => format_time(start_time - 60))
+            adapter.save_delayed_plan('plan2', :execution_plan_uuid => 'plan2', :frozen => true, :start_at => format_time(start_time + 60),
+                                      :start_before => format_time(start_time - 60))
+
+            plans = adapter.find_past_delayed_plans(start_time)
+            plans.length.must_equal 1
+            plans.first[:execution_plan_uuid].must_equal 'plan1'
+          end
         end
       end
 

data/test/support/dummy_example.rb

@@ -6,6 +6,12 @@ module Support
       def run; end
     end
 
+    class SkippableDummy < Dummy
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Skip
+      end
+    end
+
     class MySerializer < Dynflow::Serializers::Noop
       def serialize(arg)
         raise 'Enforced serializer failure' if arg == :fail

data/test/world_test.rb

@@ -10,6 +10,7 @@ module Dynflow
       describe '#meta' do
         it 'by default informs about the hostname and the pid running the world' do
           registered_world = world.coordinator.find_worlds(false, id: world.id).first
+          registered_world.meta.delete('last_seen')
           registered_world.meta.must_equal('hostname' => Socket.gethostname, 'pid' => Process.pid,
                                            'queues' => { 'default' => { 'pool_size' => 5 },
                                                          'slow' => { 'pool_size' => 1 }})

data/web/views/worlds.erb

@@ -9,7 +9,8 @@
 <h3>Executors</h3>
 <% @executors.each do |world| %>
   <%= value_field('Id', world.id) %>
-  <%= value_field('Metadata', world.meta) %>
+  <%= value_field('Metadata', world.meta.reject { |key, _| key == 'last_seen' }) %>
+  <%= value_field('Last seen', world.meta['last_seen']) %>
   <p>
     <b>Status:</b>
     <%= erb :world_validation_result, locals: { world: world } %>
@@ -39,13 +40,15 @@
     <tr>
       <th>Id</th>
       <th>Meta</th>
+      <th>Last seen</th>
       <th></th>
     </tr>
   </thead>
 <% @clients.each do |world| %>
   <tr>
     <td><%= h(world.id) %></td>
-    <td><%= h(world.meta) %></td>
+    <td><%= h(world.meta.reject { |key, _| key == 'last_seen' }) %></td>
+    <td><%= h(world.meta[:last_seen]) %></td>
     <td>
       <%= erb :world_validation_result, locals: { world: world } %>
     </td>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-13 00:00:00.000000000 Z
+date: 2018-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -483,6 +483,7 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/014_add_step_columns.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/015_add_execution_plan_columns.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/016_add_step_queue.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/017_add_delayed_plan_frozen.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb
@@ -498,6 +499,7 @@ files:
 - lib/dynflow/serializers/abstract.rb
 - lib/dynflow/serializers/noop.rb
 - lib/dynflow/stateful.rb
+- lib/dynflow/telemetry.rb
 - lib/dynflow/testing.rb
 - lib/dynflow/testing/assertions.rb
 - lib/dynflow/testing/dummy_execution_plan.rb
@@ -525,6 +527,7 @@ files:
 - lib/dynflow/web/world_helpers.rb
 - lib/dynflow/web_console.rb
 - lib/dynflow/world.rb
+- lib/dynflow/world/invalidation.rb
 - test/abnormal_states_recovery_test.rb
 - test/action_test.rb
 - test/activejob_adapter_test.rb