dynflow 1.3.0 → 1.4.0

data/lib/dynflow/executors/parallel/core.rb

@@ -1,20 +1,17 @@
+# frozen_string_literal: true
+require 'dynflow/executors/parallel/pool'
+require 'dynflow/executors/parallel/worker'
+
 module Dynflow
   module Executors
-    class Parallel < Abstract
-      class Core < Actor
+    class Parallel
+      class Core < Abstract::Core
         attr_reader :logger
 
         def initialize(world, heartbeat_interval, queues_options)
-          @logger         = world.logger
-          @world          = Type! world, World
-          @queues_options = queues_options
-          @pools          = {}
-          @terminated     = nil
-          @director       = Director.new(@world)
-          @heartbeat_interval = heartbeat_interval
-
+          super
+          @pools = {}
           initialize_queues
-          schedule_heartbeat
         end
 
         def initialize_queues
@@ -27,41 +24,8 @@ module Dynflow
           end
         end
 
-        def handle_execution(execution_plan_id, finished)
-          if terminating?
-            raise Dynflow::Error,
-                  "cannot accept execution_plan_id:#{execution_plan_id} core is terminating"
-          end
-
-          feed_pool(@director.start_execution(execution_plan_id, finished))
-        end
-
-        def handle_event(event)
-          Type! event, Director::Event
-          if terminating?
-            raise Dynflow::Error,
-                  "cannot accept event: #{event} core is terminating"
-          end
-          feed_pool(@director.handle_event(event))
-        end
-
-        def work_finished(work)
-          feed_pool(@director.work_finished(work))
-        end
-
-        def handle_persistence_error(error, work = nil)
-          logger.error "PersistenceError in executor"
-          logger.error error
-          @director.work_failed(work) if work
-          if error.is_a? Errors::FatalPersistenceError
-            logger.fatal "Terminating"
-            @world.terminate
-          end
-        end
-
         def start_termination(*args)
           super
-          logger.info 'shutting down Core ...'
           @pools.values.each { |pool| pool.tell([:start_termination, Concurrent::Promises.resolvable_future]) }
         end
 
@@ -69,66 +33,21 @@ module Dynflow
           @pools.delete(pool_name)
           # we expect this message from all worker pools
           return unless @pools.empty?
-          @director.terminate
-          logger.info '... Dynflow core terminated.'
           super()
         end
 
-        def dead_letter_routing
-          @world.dead_letter_handler
-        end
-
         def execution_status(execution_plan_id = nil)
           @pools.each_with_object({}) do |(pool_name, pool), hash|
             hash[pool_name] = pool.ask!([:execution_status, execution_plan_id])
           end
         end
 
-        def heartbeat
-          @logger.debug('Executor heartbeat')
-          record = @world.coordinator.find_records(:id => @world.id,
-                                                   :class => ['Dynflow::Coordinator::ExecutorWorld', 'Dynflow::Coordinator::ClientWorld']).first
-          unless record
-            logger.error(%{Executor's world record for #{@world.id} missing: terminating})
-            @world.terminate
-            return
-          end
-
-          record.data[:meta].update(:last_seen => Dynflow::Dispatcher::ClientDispatcher::PingCache.format_time)
-          @world.coordinator.update_record(record)
-          schedule_heartbeat
-        end
-
-        private
-
-        def schedule_heartbeat
-          @world.clock.ping(self, @heartbeat_interval, :heartbeat)
-        end
-
-        def on_message(message)
-          super
-        rescue Errors::PersistenceError => e
-          self.tell([:handle_persistence_error, e])
-        end
-
         def feed_pool(work_items)
-          return if terminating?
-          return if work_items.nil?
-          work_items = [work_items] if work_items.is_a? Director::WorkItem
-          work_items.all? { |i| Type! i, Director::WorkItem }
           work_items.each do |new_work|
-            pool = @pools[new_work.queue]
-            unless pool
-              logger.error("Pool is not available for queue #{new_work.queue}, falling back to #{fallback_queue}")
-              pool = @pools[fallback_queue]
-            end
-            pool.tell([:schedule_work, new_work])
+            new_work.world = @world
+            @pools.fetch(suggest_queue(new_work)).tell([:schedule_work, new_work])
           end
         end
-
-        def fallback_queue
-          :default
-        end
       end
     end
   end

data/lib/dynflow/executors/parallel/pool.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Dynflow
   module Executors
-    class Parallel < Abstract
+    class Parallel
       class Pool < Actor
         class JobStorage
           def initialize
@@ -53,7 +54,8 @@ module Dynflow
         end
 
         def worker_done(worker, work)
-          @executor_core.tell([:work_finished, work])
+          step = work.step if work.is_a?(Director::StepWorkItem)
+          @executor_core.tell([:work_finished, work, step && step.delayed_events])
           @free_workers << worker
           Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_active_workers, -1, telemetry_options) }
           distribute_jobs

data/lib/dynflow/executors/parallel/worker.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
 module Dynflow
   module Executors
-    class Parallel < Abstract
+    class Parallel
       class Worker < Actor
         def initialize(pool, transaction_adapter, telemetry_options = {})
           @pool                = Type! pool, Concurrent::Actor::Reference

data/lib/dynflow/executors/sidekiq/core.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+require 'dynflow/executors/sidekiq/serialization'
+require 'dynflow/executors/sidekiq/internal_job_base'
+require 'dynflow/executors/sidekiq/orchestrator_jobs'
+require 'dynflow/executors/sidekiq/worker_jobs'
+require 'dynflow/executors/sidekiq/redis_locking'
+
+require 'sidekiq-reliable-fetch'
+Sidekiq.configure_server do |config|
+  # Use semi-reliable fetch
+  # for details see https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/blob/master/README.md
+  config.options[:semi_reliable_fetch] = true
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+end
+
+module Dynflow
+  module Executors
+    module Sidekiq
+      class Core < Abstract::Core
+        include RedisLocking
+
+        TELEMETRY_UPDATE_INTERVAL = 30 # update telemetry every 30s
+
+        attr_reader :logger
+
+        def initialize(world, *_args)
+          @world = world
+          @logger = world.logger
+          wait_for_orchestrator_lock
+          super
+          schedule_update_telemetry
+          begin_startup!
+        end
+
+        def heartbeat
+          super
+          reacquire_orchestrator_lock
+        end
+
+        def start_termination(*args)
+          super
+          release_orchestrator_lock
+          finish_termination
+        end
+
+        # TODO: needs thoughs on how to implement it
+        def execution_status(execution_plan_id = nil)
+          {}
+        end
+
+        def feed_pool(work_items)
+          work_items.each do |new_work|
+            WorkerJobs::PerformWork.set(queue: suggest_queue(new_work)).perform_async(new_work)
+          end
+        end
+
+        def update_telemetry
+          sidekiq_queues = ::Sidekiq::Stats.new.queues
+          @queues_options.keys.each do |queue|
+            queue_size = sidekiq_queues[queue.to_s]
+            if queue_size
+              Dynflow::Telemetry.with_instance { |t| t.set_gauge(:dynflow_queue_size, queue_size, telemetry_options(queue)) }
+            end
+          end
+          schedule_update_telemetry
+        end
+
+        def work_finished(work, delayed_events = nil)
+          # If the work item is sent in reply to a request from the current orchestrator, proceed
+          if work.sender_orchestrator_id == @world.id
+            super
+          else
+            # If we're in recovery, we can drop the work as the execution plan will be resumed during validity checks performed when leaving recovery
+            # If we're not in recovery and receive an event from another orchestrator, it means it survived the queue draining.
+            handle_unknown_work_item(work) unless @recovery
+          end
+        end
+
+        def begin_startup!
+          WorkerJobs::DrainMarker.perform_async(@world.id)
+          @recovery = true
+        end
+
+        def startup_complete
+          logger.info('Performing validity checks')
+          @world.perform_validity_checks
+          logger.info('Finished performing validity checks')
+          @recovery = false
+        end
+
+        private
+
+        def fallback_queue
+          :default
+        end
+
+        def schedule_update_telemetry
+          @world.clock.ping(reference, TELEMETRY_UPDATE_INTERVAL, [:update_telemetry])
+        end
+
+        def telemetry_options(queue)
+          { queue: queue.to_s, world: @world.id }
+        end
+
+        # We take a look if an execution lock is already being held by an orchestrator (it should be the current one). If no lock is held
+        # we try to resume the execution plan if possible
+        def handle_unknown_work_item(work)
+          # We are past recovery now, if we receive an event here, the execution plan will be most likely paused
+          # We can either try to rescue it or turn it over to stopped
+          execution_lock = @world.coordinator.find_locks(class: Coordinator::ExecutionLock.name,
+                                                         id: "execution-plan:#{work.execution_plan_id}").first
+          if execution_lock.nil?
+            plan = @world.persistence.load_execution_plan(work.execution_plan_id)
+            should_resume = !plan.error? || plan.prepare_for_rescue == :running
+            @world.execute(plan.id) if should_resume
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/sidekiq/internal_job_base.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+module Dynflow
+  module Executors
+    module Sidekiq
+      class InternalJobBase
+        include ::Sidekiq::Worker
+        extend ::Dynflow::Executors::Sidekiq::Serialization::WorkerExtension::ClassMethods
+        sidekiq_options retry: false, backtrace: true
+
+        def self.inherited(klass)
+          klass.prepend(::Dynflow::Executors::Sidekiq::Serialization::WorkerExtension)
+        end
+
+        def worker_id
+          ::Sidekiq::Logging.tid
+        end
+
+        def telemetry_options(work_item)
+          { queue: work_item.queue.to_s, world: Dynflow.process_world.id, worker: worker_id }
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/sidekiq/orchestrator_jobs.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+module Dynflow
+  module Executors
+    module Sidekiq
+      module OrchestratorJobs
+        # handles resposnes about finished work form the workers
+        # or some event to handle on orchestrator side
+        class WorkerDone < InternalJobBase
+          sidekiq_options queue: :dynflow_orchestrator
+
+          # @param request_envelope [Dispatcher::Request] - request to handle on orchestrator side
+          #   usually to start new execution or to pass some event
+          def perform(work_item, delayed_events = nil)
+            # Usually the step is saved on the worker's side. However if sidekiq is shut down,
+            #   then the step may not have been saved so we save it just to be sure
+            if work_item.is_a?(Director::StepWorkItem) && work_item.step&.error&.exception.is_a?(::Sidekiq::Shutdown)
+              work_item.step.save
+            end
+            Dynflow.process_world.executor.core.tell([:work_finished, work_item, delayed_events])
+          end
+        end
+
+        # handles setting up an event on orchestrator
+        class PlanEvent < InternalJobBase
+          sidekiq_options queue: :dynflow_orchestrator
+
+          # @param event_envelope [Dispatcher::Event] - request to handle on orchestrator side
+          #   usually to start new execution or to pass some event
+          def perform(execution_plan_id, step_id, event, time)
+            Dynflow.process_world.plan_event(execution_plan_id, step_id, event, time)
+          end
+        end
+
+        class HandlePersistenceError < InternalJobBase
+          sidekiq_options queue: :dynflow_orchestrator
+
+          # @param request_envelope [Dispatcher::Request] - request to handle on orchestrator side
+          #   usually to start new execution or to pass some event
+          def perform(error, work_item)
+            Dynflow.process_world.executor.core.tell([:handle_persistence_error, error, work_item])
+          end
+        end
+
+        class StartupComplete < InternalJobBase
+          sidekiq_options queue: :dynflow_orchestrator
+
+          # @param request_envelope [Dispatcher::Request] - request to handle on orchestrator side
+          #   usually to start new execution or to pass some event
+          def perform(world_id)
+            if Dynflow.process_world.id == world_id
+              Dynflow.process_world.executor.core.tell([:startup_complete])
+            else
+              logger.warn("Received startup complete for a different world #{world_id}, discarding.")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/sidekiq/redis_locking.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+module Dynflow
+  module Executors
+    module Sidekiq
+      module RedisLocking
+        REDIS_LOCK_KEY = 'dynflow_orchestrator_uuid'
+        REDIS_LOCK_TTL = 60
+        REDIS_LOCK_POLL_INTERVAL = 15
+
+        ACQUIRE_OK = 0
+        ACQUIRE_MISSING = 1
+        ACQUIRE_TAKEN = 2
+
+        RELEASE_SCRIPT = <<~LUA
+          if redis.call("get", KEYS[1]) == ARGV[1] then
+            redis.call("del", KEYS[1])
+          end
+          return #{ACQUIRE_OK}
+        LUA
+
+        REACQUIRE_SCRIPT = <<~LUA
+          if redis.call("exists", KEYS[1]) == 1 then
+            local owner = redis.call("get", KEYS[1])
+            if owner == ARGV[1] then
+              redis.call("set", KEYS[1], ARGV[1], "XX", "EX", #{REDIS_LOCK_TTL})
+              return #{ACQUIRE_OK}
+            else
+              return #{ACQUIRE_TAKEN}
+            end
+          else
+            redis.call("set", KEYS[1], ARGV[1], "NX", "EX", #{REDIS_LOCK_TTL})
+            return #{ACQUIRE_MISSING}
+          end
+        LUA
+
+        def release_orchestrator_lock
+          ::Sidekiq.redis { |conn| conn.eval RELEASE_SCRIPT, [REDIS_LOCK_KEY], [@world.id] }
+        end
+
+        def wait_for_orchestrator_lock
+          mode = nil
+          loop do
+            active = ::Sidekiq.redis do |conn|
+              conn.set(REDIS_LOCK_KEY, @world.id, :ex => REDIS_LOCK_TTL, :nx => true)
+            end
+            break if active
+            if mode.nil?
+              mode = :passive
+              @logger.info('Orchestrator lock already taken, entering passive mode.')
+            end
+            sleep REDIS_LOCK_POLL_INTERVAL
+          end
+          @logger.info('Acquired orchestrator lock, entering active mode.')
+        end
+
+        def reacquire_orchestrator_lock
+          case ::Sidekiq.redis { |conn| conn.eval REACQUIRE_SCRIPT, [REDIS_LOCK_KEY], [@world.id] }
+          when ACQUIRE_MISSING
+            @logger.error('The orchestrator lock was lost, reacquired')
+          when ACQUIRE_TAKEN
+            owner = ::Sidekiq.redis { |conn| conn.get REDIS_LOCK_KEY }
+            @logger.fatal("The orchestrator lock was stolen by #{owner}, aborting.")
+            Process.kill('INT', Process.pid)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/executors/sidekiq/serialization.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Dynflow
+  module Executors
+    module Sidekiq
+      # Module to prepend the Sidekiq job to handle the serialization
+      module Serialization
+        def self.serialize(value)
+          Dynflow.serializer.dump(value)
+        end
+
+        def self.deserialize(value)
+          value = Utils::IndifferentHash.new(value) if value.is_a? Hash
+          Dynflow.serializer.load(value)
+        end
+
+        module WorkerExtension
+          # Overriding the Sidekiq entry method to perform additional serialization preparation
+          module ClassMethods
+            def client_push(opts)
+              opts['args'] = opts['args'].map { |a| Serialization.serialize(a) }
+              super(opts)
+            end
+          end
+
+          def perform(*args)
+            args = args.map { |a| Serialization.deserialize(a) }
+            super(*args)
+          end
+        end
+      end
+    end
+  end
+end