delayed_job_master 2.0.3 → 3.0.0

data/lib/delayed/master/postgresql/job_listener.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    module Postgresql
+      class JobListener < Delayed::Master::JobListener
+        def initialize(master)
+          @master = master
+          @config = master.config
+          @databases = master.databases
+          @threads = []
+        end
+
+        def start
+          @threads = @databases.map do |database|
+            Thread.new(database) do |database|
+              loop do
+                if @master.stop?
+                  break
+                else
+                  listen(database)
+                end
+              end
+            end
+          end
+        end
+
+        def wait
+          @threads.each(&:join)
+        end
+
+        def shutdown
+          @threads.each(&:kill)
+        end
+
+        private
+
+        def listen(database)
+          database.with_connection do |connection|
+            listen_connection(database, connection) do
+              loop do
+                if @master.stop?
+                  break
+                else
+                  wait_for_notify(database, connection)
+                end
+              end
+            end
+          end
+        end
+
+        def listen_connection(database, connection)
+          @master.logger.info { "listening @#{database.spec_name}..." }
+          connection.execute("LISTEN delayed_job_master")
+          yield
+        rescue => e
+          @master.logger.warn { "#{e.class}: #{e.message}" }
+          @master.logger.debug { e.backtrace.join("\n") }
+        ensure
+          @master.logger.info { "unlisten @#{database.spec_name}" }
+          connection.execute("UNLISTEN delayed_job_master")
+        end
+
+        def wait_for_notify(database, connection)
+          connection.raw_connection.wait_for_notify(1) do |_event, _pid, _payload|
+            @master.logger.info { "received notification @#{database.spec_name}" }
+            @master.job_checker.schedule(database)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/postgresql/job_notifier.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    module Postgresql
+      class << self
+        def notify(model)
+          model.connection.execute "NOTIFY delayed_job_master"
+        end
+      end
+
+      module JobNotifier
+        extend ActiveSupport::Concern
+
+        included do
+          after_create :notify_to_delayed_job_master
+        end
+
+        private
+
+        def notify_to_delayed_job_master
+          if run_at && run_at < Time.zone.now
+            Delayed::Master::Postgresql.notify(self.class)
+          end
+        end
+      end
+
+      module BulkJobNotifier
+        extend ActiveSupport::Concern
+
+        included do
+          after_enqueue :notify_to_delayed_job_master
+        end
+
+        private
+
+        def notify_to_delayed_job_master
+          if @jobs.any? { |job| job.run_at && job.run_at < Time.zone.now }
+            Delayed::Master::Postgresql.notify(@jobs.first.class)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/safe_array.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class SafeArray < Array
+      def initialize(*args)
+        @mon = Monitor.new
+        super
+      end
+
+      def <<(*args)
+        @mon.synchronize do
+          super
+        end
+      end
+
+      def delete(*args)
+        @mon.synchronize do
+          super
+        end
+      end
+
+      def clear
+        @mon.synchronize do
+          super
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/signaler.rb

@@ -1,13 +1,23 @@
+# frozen_string_literal: true
+
 module Delayed
-  class Master
+  module Master
     class Signaler
+      SIGNAL_HANDLERS = [
+        [:TERM, :stop],
+        [:INT, :stop],
+        [:QUIT, :quit],
+        [:WINCH, :graceful_stop],
+        [:USR1, :reopen_files],
+        [:USR2, :restart]
+      ]
+
       def initialize(master)
         @master = master
       end
 
       def register
-        signals = [[:TERM, :stop], [:INT, :stop], [:QUIT, :quit], [:USR1, :reopen_files], [:USR2, :restart]]
-        signals.each do |signal, method|
+        SIGNAL_HANDLERS.each do |signal, method|
           register_signal(signal, method)
         end
       end
@@ -22,9 +32,9 @@ module Delayed
       private
 
       def register_signal(signal, method)
-        trap(signal) do
+        Signal.trap(signal) do
           Thread.new do
-            @master.logger.info "received #{signal} signal"
+            @master.logger.info { "received #{signal} signal" }
             @master.public_send(method)
           end
         end
@@ -32,9 +42,9 @@ module Delayed
 
       def dispatch_to(signal, pid)
         Process.kill(signal, pid)
-        @master.logger.info "sent #{signal} signal to worker #{pid}"
+        @master.logger.info { "sent #{signal} signal to worker #{pid}" }
       rescue
-        @master.logger.error "failed to send #{signal} signal to worker #{pid}"
+        @master.logger.error { "failed to send #{signal} signal to worker #{pid}" }
       end
     end
   end

data/lib/delayed/master/sleep.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    module Sleep
+      def loop_with_sleep(sec)
+        count = [sec.to_i, 1].max
+        div = sec.to_f / count
+        loop do
+          count.times do |i|
+            yield i
+            sleep div
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/backend/active_record.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Backend
+    module ActiveRecord
+      class Job < ::ActiveRecord::Base
+        # Remove locked_by from query because all jobs reserved by current process have same locked_by.
+        def self.ready_to_run(worker_name, max_run_time)
+          where(
+            "(run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) AND failed_at IS NULL",
+            db_time_now,
+            db_time_now - max_run_time
+          )
+        end
+
+        # Patch for postgresql query.
+        def self.reserve_with_scope_using_optimized_postgres(ready_scope, worker, now)
+          quoted_name = connection.quote_table_name(table_name)
+          subquery    = ready_scope.limit(1).lock(true).select("id").to_sql
+          sql         = <<~SQL.squish
+            WITH job AS (#{subquery} SKIP LOCKED)
+              UPDATE #{quoted_name} AS jobs SET locked_at = ?, locked_by = ? FROM job
+              WHERE jobs.id = job.id RETURNING *
+          SQL
+          reserved    = find_by_sql([sql, now, worker.name])
+          reserved[0]
+        end
+
+        # Patch for mysql query.
+        def self.reserve_with_scope_using_optimized_mysql(ready_scope, worker, now)
+          transaction do
+            ready_scope.limit(worker.read_ahead).select(:id).lock.detect do |job|
+              count = where(id: job.id).update_all(locked_at: now, locked_by: worker.name)
+              count == 1 && job.reload
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/extension.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative 'lifecycle'
+require_relative 'thread_pool'
+require_relative 'thread_worker'
+require_relative 'plugins/all'
+require_relative 'backend/active_record' if defined?(Delayed::Backend::ActiveRecord)
+
+module Delayed
+  class Worker
+    attr_accessor :master_logger
+    attr_accessor :max_threads, :max_memory
+  end
+end

data/lib/delayed/master/worker/lifecycle.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+events = Delayed::Lifecycle::EVENTS.dup.merge(
+  thread: [:worker],
+  scheduler_thread: [:worker],
+  worker_thread: [:worker, :job]
+)
+
+Delayed::Lifecycle.send(:remove_const, :EVENTS)
+Delayed::Lifecycle.const_set(:EVENTS, events)

data/lib/delayed/master/worker/plugins/all.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative 'executor_wrapper'
+require_relative 'memory_checker'
+require_relative 'signal_handler'
+require_relative 'status_notifier'
+
+[
+  Delayed::Master::Worker::Plugins::ExecutorWrapper,
+  Delayed::Master::Worker::Plugins::MemoryChecker,
+  Delayed::Master::Worker::Plugins::SignalHandler,
+  Delayed::Master::Worker::Plugins::StatusNotifier
+].each do |plugin|
+  unless Delayed::Worker.plugins.include?(plugin)
+    Delayed::Worker.plugins << plugin
+  end
+end

data/lib/delayed/master/worker/plugins/executor_wrapper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Worker
+      module Plugins
+        class ExecutorWrapper < Delayed::Plugin
+          callbacks do |lifecycle|
+            lifecycle.around(:thread) do |worker, &block|
+              if defined?(Rails)
+                Rails.application.executor.wrap do
+                  block.call
+                end
+              else
+                block.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/plugins/memory_checker.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'get_process_mem'
+
+module Delayed
+  module Master
+    class Worker
+      module Plugins
+        class MemoryChecker < Delayed::Plugin
+          callbacks do |lifecycle|
+            lifecycle.before(:perform) do |worker, job|
+              mem = GetProcessMem.new
+              worker.master_logger.info { "performing #{job.name}, memory: #{mem.mb.to_i} MB" }
+            end
+            lifecycle.after(:perform) do |worker, job|
+              mem = GetProcessMem.new
+              worker.master_logger.info { "performed #{job.name}, memory: #{mem.mb.to_i} MB" }
+              if worker.max_memory && mem.mb > worker.max_memory
+                worker.master_logger.info { "shutting down worker #{Process.pid} because it consumes large memory..." }
+                worker.stop
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/plugins/signal_handler.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Worker
+      module Plugins
+        class SignalHandler < Delayed::Plugin
+          callbacks do |lifecycle|
+            lifecycle.before(:execute) do |worker|
+              worker.instance_eval do
+                Signal.trap(:USR1) do
+                  Thread.new do
+                    master_logger.info { "reopening files..." }
+                    Delayed::Master::FileReopener.reopen
+                    master_logger.info { "reopened" }
+                  end
+                end
+                Signal.trap(:USR2) do
+                  Thread.new do
+                    $0 = "#{$0} [OLD]"
+                    master_logger.info { "shutting down worker #{Process.pid}..." }
+                    stop
+                  end
+                end
+              end
+            end
+            lifecycle.after(:execute) do |worker|
+              worker.master_logger.info { "shut down worker #{Process.pid}" }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/plugins/status_notifier.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Worker
+      module Plugins
+        class StatusNotifier < Delayed::Plugin
+          callbacks do |lifecycle|
+            lifecycle.around(:execute) do |worker, job, &block|
+              title = $0
+              $0 = "#{title} [BUSY]"
+              ret = block.call
+              $0 = title
+              ret
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/thread_pool.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Worker
+      class ThreadPool
+        def initialize(worker, size)
+          @worker = worker
+          @size = size
+          @queue = SizedQueue.new(@size)
+          @queue_delay = 0.5
+        end
+
+        def schedule
+          @scheduler = Thread.new do
+            Delayed::Worker.lifecycle.run_callbacks(:thread, @worker) do
+              loop do
+                while @queue.num_waiting == 0
+                  sleep @queue_delay
+                end
+
+                if item = yield
+                  @queue.push(item)
+                  Thread.pass
+                else
+                  @size.times { @queue.push(:exit) }
+                  break
+                end
+              end
+            end
+          end
+        end
+
+        def work
+          @threads = @size.times.map do
+            Thread.new do
+              Delayed::Worker.lifecycle.run_callbacks(:thread, @worker) do
+                loop do
+                  item = @queue.pop
+                  if item == :exit
+                    break
+                  else
+                    yield item
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def wait
+          @scheduler.join
+          @threads.each(&:join)
+        end
+
+        def shutdown
+          @scheduler.kill
+          @threads.each(&:kill)
+          @queue.close
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/worker/thread_worker.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Overrides Delayed::Worker to support multithread.
+# See original code at https://github.com/collectiveidea/delayed_job/blob/master/lib/delayed/worker.rb
+module Delayed
+  module Master
+    class Worker
+      module ThreadWorker
+        def work_off(num = 100)
+          if multithread?
+            work_off_for_multithread
+          else
+            super
+          end
+        end
+
+        def multithread?
+          @max_threads.to_i > 1
+        end
+
+        def work_off_for_multithread
+          success = 0
+          failure = 0
+
+          monitor = Monitor.new
+          thread_pool = ThreadPool.new(self, @max_threads)
+
+          thread_pool.schedule do
+            self.class.lifecycle.run_callbacks(:scheduler_thread, self) do
+              if stop?
+                next nil
+              else
+                next reserve_job
+              end
+            end
+          end
+
+          thread_pool.work do |job|
+            @master_logger.debug { "start worker thread #{Thread.current.object_id}" }
+            self.class.lifecycle.run_callbacks(:worker_thread, self, job) do
+              case run_one_job(job)
+              when true
+                monitor.synchronize { success += 1 }
+              when false
+                monitor.synchronize { failure += 1 }
+              end
+            end
+            @master_logger.debug { "stop worker thread #{Thread.current.object_id}" }
+          end
+
+          thread_pool.wait
+          thread_pool.shutdown
+
+          [success, failure]
+        end
+
+        def run_one_job(job)
+          self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) }
+        end
+      end
+    end
+  end
+end
+
+Delayed::Worker.prepend Delayed::Master::Worker::ThreadWorker

data/lib/delayed/master/worker.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Delayed
-  class Master
+  module Master
     class Worker
-      attr_accessor :index, :setting, :database
+      attr_accessor :setting, :database
       attr_accessor :pid, :instance
 
       def initialize(attrs = {})
@@ -15,14 +17,13 @@ module Delayed
       end
 
       def info
-        str = name
-        str << " @#{@database}" if @database
-        str << " (#{@setting.queues.join(', ')})" if @setting.queues.respond_to?(:join)
-        str
+        strs = [@setting.worker_info]
+        strs << "@#{@database.spec_name}" if @database
+        strs.join(' ')
       end
 
       def process_title
-        "delayed_job.#{@index}: #{info}"
+        "delayed_job: #{info}"
       end
     end
   end

data/lib/delayed/master/worker_setting.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class WorkerSetting
+      SIMPLE_CONFIGS = [:id, :max_processes, :max_threads, :max_memory,
+                        :min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete,
+                        :max_attempts, :max_run_time, :destroy_failed_jobs]
+      ARRAY_CONFIGS  = [:queues]
+
+      attr_accessor *SIMPLE_CONFIGS
+      attr_accessor *ARRAY_CONFIGS
+
+      def initialize(attrs = {})
+        @queues = []
+        @max_processes = 1
+        @max_threads = 1
+        @exit_on_complete = true
+        self.attributes = attrs
+      end
+
+      def attributes=(attrs = {})
+        attrs.each do |key, value|
+          send("#{key}=", value)
+        end
+      end
+
+      def worker_name
+        "worker[#{id}]"
+      end
+
+      def worker_info
+        strs = [worker_name]
+        strs << "(#{@queues.join(', ')})" if @queues.present?
+        strs.join(' ')
+      end
+
+      SIMPLE_CONFIGS.each do |key|
+        define_method(key) do |*args|
+          if args.size > 0
+            instance_variable_set("@#{key}", args[0])
+          else
+            instance_variable_get("@#{key}")
+          end
+        end
+      end
+
+      ARRAY_CONFIGS.each do |key|
+        define_method(key) do |*args|
+          if args.size > 0
+            instance_variable_set("@#{key}", Array(args[0]))
+          else
+            instance_variable_get("@#{key}")
+          end
+        end
+      end
+
+      def control(value = nil)
+        ActiveSupport::Deprecation.warn <<-TEXT.squish
+          deprecated 'control' setting was used. Remove it from your config file.
+        TEXT
+      end
+
+      def count(value = nil)
+        ActiveSupport::Deprecation.warn <<-TEXT.squish
+          deprecated 'count' setting was used. Use 'max_processes' instead.
+        TEXT
+        max_processes value
+      end
+    end
+  end
+end