delayed_job_master 2.0.3 → 3.0.0

data/lib/delayed/master.rb

@@ -1,105 +1,12 @@
-require 'fileutils'
-require 'logger'
-require_relative 'master/version'
-require_relative 'master/command'
-require_relative 'master/worker'
-require_relative 'master/monitoring'
-require_relative 'master/signaler'
-require_relative 'master/util/file_reopener'
+# frozen_string_literal: true
 
-module Delayed
-  class Master
-    attr_reader :config, :logger, :workers
-
-    def initialize(argv)
-      @config = Command.new(argv).config
-      @logger = setup_logger(@config.log_file, @config.log_level)
-      @workers = []
-
-      @signaler = Signaler.new(self)
-      @monitoring = Monitoring.new(self)
-    end
-
-    def run
-      print_config
-      daemonize if @config.daemon
+require_relative 'master/core'
 
-      @logger.info "started master #{Process.pid}".tap { |msg| puts msg }
-
-      handle_pid_file do
-        @signaler.register
-        @prepared = true
-        @monitoring.monitor_while { stop? }
-      end
-
-      @logger.info "shut down master"
-    end
-
-    def prepared?
-      @prepared
-    end
-
-    def quit
-      @signaler.dispatch(:KILL)
-      @stop = true
-    end
-
-    def stop
-      @signaler.dispatch(:TERM)
-      @stop = true
-    end
-
-    def stop?
-      @stop == true
-    end
-
-    def reopen_files
-      @signaler.dispatch(:USR1)
-      @logger.info "reopening files..."
-      Util::FileReopener.reopen
-      @logger.info "reopened"
-    end
-
-    def restart
-      @signaler.dispatch(:USR2)
-      @logger.info "restarting master..."
-      exec(*([$0] + ARGV))
-    end
-
-    private
-
-    def setup_logger(log_file, log_level)
-      FileUtils.mkdir_p(File.dirname(log_file)) if log_file.is_a?(String)
-      logger = Logger.new(log_file)
-      logger.level = log_level
-      logger
-    end
-  
-    def daemonize
-      Process.daemon(true)
-    end
-
-    def handle_pid_file
-      create_pid_file
-      yield
-      remove_pid_file
-    end
-
-    def create_pid_file
-      FileUtils.mkdir_p(File.dirname(@config.pid_file))
-      File.write(@config.pid_file, Process.pid)
-    end
-
-    def remove_pid_file
-      File.delete(@config.pid_file) if File.exist?(@config.pid_file)
-    end
-
-    def print_config
-      @logger.info "databases: #{@config.databases.join(', ')}" if @config.databases
-      @config.worker_settings.each do |setting|
-        message = "worker[#{setting.id}]: #{setting.count} processes"
-        message << " (#{setting.queues.join(', ')})" if setting.queues.respond_to?(:join)
-        @logger.info message
+module Delayed
+  module Master
+    class << self
+      def new(argv)
+        Core.new(argv)
       end
     end
   end

data/lib/delayed_job_master/railtie.rb

@@ -0,0 +1,16 @@
+module DelayedJobMaster
+  class Railtie < Rails::Railtie
+    config.after_initialize do
+      case DelayedJobMaster.config.listener
+      when :postgresql
+        require_relative '../delayed/master/postgresql/job_notifier'
+        if defined?(Delayed::Backend::ActiveRecord)
+          Delayed::Backend::ActiveRecord::Job.include Delayed::Master::Postgresql::JobNotifier
+        end
+        if defined?(Delayed::Backend::Bulk)
+          Delayed::Backend::Bulk.include Delayed::Master::Postgresql::BulkJobNotifier
+        end
+      end
+    end
+  end
+end

data/lib/delayed_job_master/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DelayedJobMaster
+  VERSION = "3.0.0"
+end

data/lib/delayed_job_master.rb

@@ -1 +1,15 @@
-require 'delayed/master'
+require 'active_support'
+
+require_relative 'delayed_job_master/version'
+require_relative 'delayed_job_master/railtie' if defined?(Rails)
+
+module DelayedJobMaster
+  mattr_accessor :config, default: ActiveSupport::InheritableOptions.new
+  config.listener = :postgresql if defined?(PG)
+
+  class << self
+    def configure
+      yield config
+    end
+  end
+end

data/lib/generators/delayed_job_master/templates/config.rb

@@ -1,8 +1,11 @@
 # working directory
 working_directory Dir.pwd
 
-# monitor wait time in second
-monitor_wait 5
+# monitor interval for events (in seconds)
+monitor_interval 5
+
+# polling interval for new jobs (in seconds)
+polling_interval 5
 
 # path to pid file
 pid_file "#{Dir.pwd}/tmp/pids/delayed_job_master.pid"
@@ -18,9 +21,12 @@ add_worker do |worker|
   # queue name for the worker
   worker.queues %w(queue1)
 
-  # worker count
-  worker.count 1
+  # max process count
+  worker.max_processes 1
 
+  # max thread count for each worker
+  worker.max_threads 1
+  
   # max memory in MB
   worker.max_memory 300
 
@@ -37,20 +43,14 @@ end
 # worker2
 add_worker do |worker|
   worker.queues %w(queue2)
-  worker.count 2
+  worker.max_processes 2
+  worker.max_threads 2
 end
 
 before_fork do |master, worker|
-  Delayed::Worker.before_fork if defined?(Delayed::Worker)
+  ActiveRecord::Base.connection.disconnect!
 end
 
 after_fork do |master, worker|
-  Delayed::Worker.after_fork if defined?(Delayed::Worker)
-end
-
-before_monitor do |master|
-  ActiveRecord::Base.connection.verify! if defined?(ActiveRecord::Base)
-end
-
-after_monitor do |master|
+  ActiveRecord::Base.establish_connection
 end

data/lib/generators/delayed_job_master/templates/script

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
-require 'delayed_job_master'
+require 'delayed/master'
 
 Delayed::Master.new(ARGV).run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delayed_job_master
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 3.0.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-04 00:00:00.000000000 Z
+date: 2023-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: delayed_job
@@ -38,20 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -122,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: delayed_job_bulk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: A simple delayed_job master process to control multiple workers
 email:
 - kaneta@sitebridge.co.jp
@@ -139,29 +181,41 @@ files:
 - README.md
 - Rakefile
 - delayed_job_master.gemspec
-- gemfiles/rails50.gemfile
-- gemfiles/rails51.gemfile
-- gemfiles/rails52.gemfile
 - gemfiles/rails60.gemfile
 - gemfiles/rails61.gemfile
 - gemfiles/rails70.gemfile
 - lib/delayed/master.rb
+- lib/delayed/master/callbacks.rb
 - lib/delayed/master/command.rb
 - lib/delayed/master/config.rb
-- lib/delayed/master/database_detector.rb
+- lib/delayed/master/core.rb
+- lib/delayed/master/database.rb
+- lib/delayed/master/file_reopener.rb
 - lib/delayed/master/forker.rb
 - lib/delayed/master/job_checker.rb
-- lib/delayed/master/job_counter.rb
+- lib/delayed/master/job_finder.rb
+- lib/delayed/master/job_listener.rb
 - lib/delayed/master/monitoring.rb
-- lib/delayed/master/plugins/memory_checker.rb
-- lib/delayed/master/plugins/signal_handler.rb
-- lib/delayed/master/plugins/status_notifier.rb
+- lib/delayed/master/postgresql/job_listener.rb
+- lib/delayed/master/postgresql/job_notifier.rb
+- lib/delayed/master/safe_array.rb
 - lib/delayed/master/signaler.rb
-- lib/delayed/master/util/file_reopener.rb
-- lib/delayed/master/version.rb
+- lib/delayed/master/sleep.rb
 - lib/delayed/master/worker.rb
-- lib/delayed/master/worker_extension.rb
+- lib/delayed/master/worker/backend/active_record.rb
+- lib/delayed/master/worker/extension.rb
+- lib/delayed/master/worker/lifecycle.rb
+- lib/delayed/master/worker/plugins/all.rb
+- lib/delayed/master/worker/plugins/executor_wrapper.rb
+- lib/delayed/master/worker/plugins/memory_checker.rb
+- lib/delayed/master/worker/plugins/signal_handler.rb
+- lib/delayed/master/worker/plugins/status_notifier.rb
+- lib/delayed/master/worker/thread_pool.rb
+- lib/delayed/master/worker/thread_worker.rb
+- lib/delayed/master/worker_setting.rb
 - lib/delayed_job_master.rb
+- lib/delayed_job_master/railtie.rb
+- lib/delayed_job_master/version.rb
 - lib/generators/delayed_job_master/config_generator.rb
 - lib/generators/delayed_job_master/templates/config.rb
 - lib/generators/delayed_job_master/templates/script
@@ -177,7 +231,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/gemfiles/rails50.gemfile

@@ -1,7 +0,0 @@
-source 'https://rubygems.org'
-
-gem "delayed_job_active_record", "~> 4.1.0"
-gem "rails", "~> 5.0.0"
-gem "sqlite3", "~> 1.3.6"
-
-gemspec path: "../"

data/gemfiles/rails51.gemfile

@@ -1,7 +0,0 @@
-source 'https://rubygems.org'
-
-gem "delayed_job_active_record", "~> 4.1.0"
-gem "rails", "~> 5.1.0"
-gem "sqlite3", "~> 1.3.6"
-
-gemspec path: "../"

data/gemfiles/rails52.gemfile

@@ -1,7 +0,0 @@
-source 'https://rubygems.org'
-
-gem "delayed_job_active_record", "~> 4.1.0"
-gem "rails", "~> 5.2.0"
-gem "sqlite3", "~> 1.3.6"
-
-gemspec path: "../"

data/lib/delayed/master/database_detector.rb

@@ -1,37 +0,0 @@
-module Delayed
-  class Master
-    class DatabaseDetector
-      def initialize
-      end
-
-      def call
-        load_spec_names.select { |spec_name| has_delayed_job_table?(spec_name) }
-      end
-
-      private
-
-      def load_spec_names
-        if Rails::VERSION::MAJOR >= 6
-          load_spec_names_from_multi_db_config
-        else
-          [Rails.env.to_sym]
-        end
-      end
-
-      def load_spec_names_from_multi_db_config
-        configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env)
-        configs.reject!(&:replica?)
-        if Rails::VERSION::MAJOR == 6
-          configs.map { |c| c.spec_name.to_sym }
-        else
-          configs.map { |c| c.name.to_sym }
-        end
-      end
-
-      def has_delayed_job_table?(spec_name)
-        ActiveRecord::Base.establish_connection(spec_name)
-        ActiveRecord::Base.connection.tables.include?('delayed_jobs')
-      end
-    end
-  end
-end

data/lib/delayed/master/job_counter.rb

@@ -1,26 +0,0 @@
-# JobCounter depends on delayed_job_active_record.
-# See https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/delayed/backend/active_record.rb
-module Delayed
-  class Master
-    class JobCounter
-      def initialize(klass)
-        @klass = klass
-      end
-
-      def count(setting)
-        jobs = ready_to_run(setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
-        jobs.where!("priority >= ?", setting.min_priority) if setting.min_priority
-        jobs.where!("priority <= ?", setting.max_priority) if setting.max_priority
-        jobs.where!(queue: setting.queues) if setting.queues.any?
-        jobs.count
-      end
-
-      private
-
-      def ready_to_run(max_run_time)
-        db_time_now = @klass.db_time_now
-        @klass.where("(run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) AND failed_at IS NULL", db_time_now, db_time_now - max_run_time)
-      end
-    end
-  end
-end

data/lib/delayed/master/plugins/memory_checker.rb

@@ -1,24 +0,0 @@
-require 'get_process_mem'
-
-module Delayed
-  class Master
-    module Plugins
-      class MemoryChecker < Delayed::Plugin
-        callbacks do |lifecycle|
-          lifecycle.before(:perform) do |worker, job|
-            mem = GetProcessMem.new
-            worker.master_logger.info "performing #{job.name}, memory: #{mem.mb.to_i} MB"
-          end
-          lifecycle.after(:perform) do |worker, job|
-            mem = GetProcessMem.new
-            worker.master_logger.info "performed #{job.name}, memory: #{mem.mb.to_i} MB"
-            if worker.max_memory && mem.mb > worker.max_memory
-              worker.master_logger.info "shutting down worker #{Process.pid} because it consumes large memory..."
-              worker.stop
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/delayed/master/plugins/signal_handler.rb

@@ -1,31 +0,0 @@
-module Delayed
-  class Master
-    module Plugins
-      class SignalHandler < Delayed::Plugin
-        callbacks do |lifecycle|
-          lifecycle.before(:execute) do |worker|
-            worker.instance_eval do
-              trap(:USR1) do
-                Thread.new do
-                  master_logger.info "reopening files..."
-                  Delayed::Master::Util::FileReopener.reopen
-                  master_logger.info "reopened"
-                end
-              end
-              trap(:USR2) do
-                Thread.new do
-                  $0 = "#{$0} [OLD]"
-                  master_logger.info "shutting down worker #{Process.pid}..."
-                  stop
-                end
-              end
-            end
-          end
-          lifecycle.after(:execute) do |worker|
-            worker.master_logger.info "shut down worker #{Process.pid}"
-          end
-        end
-      end
-    end
-  end
-end

data/lib/delayed/master/plugins/status_notifier.rb

@@ -1,17 +0,0 @@
-module Delayed
-  class Master
-    module Plugins
-      class StatusNotifier < Delayed::Plugin
-        callbacks do |lifecycle|
-          lifecycle.around(:perform) do |worker, job, &block|
-            title = $0
-            $0 = "#{title} [BUSY]"
-            ret = block.call
-            $0 = title
-            ret
-          end
-        end
-      end
-    end
-  end
-end

data/lib/delayed/master/util/file_reopener.rb

@@ -1,18 +0,0 @@
-module Delayed
-  class Master
-    class Util
-      class FileReopener
-        class << self
-          def reopen
-            ObjectSpace.each_object(File) do |file|
-              next if file.closed? || !file.sync
-              file.reopen file.path, 'a+'
-              file.sync = true
-              file.flush
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/delayed/master/version.rb

@@ -1,5 +0,0 @@
-module Delayed
-  class Master
-    VERSION = "2.0.3"
-  end
-end

data/lib/delayed/master/worker_extension.rb

@@ -1,19 +0,0 @@
-module Delayed
-  class Worker
-    attr_accessor :master_logger, :max_memory
-  end
-end
-
-require_relative 'plugins/memory_checker'
-require_relative 'plugins/signal_handler'
-require_relative 'plugins/status_notifier'
-
-[
-  Delayed::Master::Plugins::MemoryChecker,
-  Delayed::Master::Plugins::SignalHandler,
-  Delayed::Master::Plugins::StatusNotifier
-].each do |plugin|
-  unless Delayed::Worker.plugins.include?(plugin)
-    Delayed::Worker.plugins << plugin
-  end
-end