inst-jobs 2.0.0 → 3.1.0

data/lib/delayed/server.rb

@@ -1,34 +1,32 @@
 # frozen_string_literal: true
 
-require 'sinatra/base'
-require 'sinatra/json'
-require 'json'
-require 'delayed_job'
+require "sinatra/base"
+require "sinatra/json"
+require "json"
+require "delayed_job"
 
 module Delayed
   class Server < Sinatra::Base
     APP_DIR = File.dirname(File.expand_path(__FILE__))
-    set :views,         File.join(APP_DIR, 'server', 'views')
-    set :public_folder, File.join(APP_DIR, 'server', 'public')
+    set :views,         File.join(APP_DIR, "server", "views")
+    set :public_folder, File.join(APP_DIR, "server", "public")
 
-    def initialize(*args, &block)
+    def initialize(*args)
       super()
       # Rails will take care of establishing the DB connection for us if there is
       # an application present
       if using_active_record? && !ActiveRecord::Base.connected?
-        ActiveRecord::Base.establish_connection(ENV['DATABASE_URL'])
+        ActiveRecord::Base.establish_connection(ENV.fetch("DATABASE_URL", nil))
       end
 
-      @allow_update = args.length > 0 && args[0][:update]
+      @allow_update = args.length.positive? && args[0][:update]
     end
 
     def using_active_record?
       Delayed::Job == Delayed::Backend::ActiveRecord::Job
     end
 
-    def allow_update
-      @allow_update
-    end
+    attr_reader :allow_update
 
     # Ensure we're connected to the DB before processing the request
     before do
@@ -43,72 +41,72 @@ module Delayed
     end
 
     configure :development do
-      require 'sinatra/reloader'
+      require "sinatra/reloader"
       register Sinatra::Reloader
     end
 
     helpers do
       # this can't get required until the class has been opened for the first time
-      require 'delayed/server/helpers'
+      require "delayed/server/helpers"
       include Delayed::Server::Helpers
     end
 
-    get '/' do
+    get "/" do
       erb :index
     end
 
-    get '/running' do
+    get "/running" do
       content_type :json
       json({
-        draw: params['draw'].to_i,
-        recordsTotal: Delayed::Job.running.count,
-        recordsFiltered: Delayed::Job.running.count,
-        data: Delayed::Job.running_jobs.map{ |j|
-          j.as_json(include_root: false, except: [:handler, :last_error])
-        },
-      })
+             draw: params["draw"].to_i,
+             recordsTotal: Delayed::Job.running.count,
+             recordsFiltered: Delayed::Job.running.count,
+             data: Delayed::Job.running_jobs.map do |j|
+                     j.as_json(include_root: false, except: %i[handler last_error])
+                   end
+           })
     end
 
-    get '/tags' do
+    get "/tags" do
       content_type :json
       json({
-        draw: params['draw'].to_i,
-        data: Delayed::Job.tag_counts('current', 10)
-      })
+             draw: params["draw"].to_i,
+             data: Delayed::Job.tag_counts("current", 10)
+           })
     end
 
     DEFAULT_PAGE_SIZE = 10
     MAX_PAGE_SIZE = 100
-    get '/jobs' do
+    get "/jobs" do
       content_type :json
-      flavor = params['flavor'] || 'current'
+      flavor = params["flavor"] || "current"
       page_size = extract_page_size
-      offset = Integer(params['start'] || 0)
+      offset = Integer(params["start"] || 0)
       case flavor
-      when 'id'
-        jobs = Delayed::Job.where(id: params['search_term'])
+      when "id"
+        jobs = Delayed::Job.where(id: params["search_term"])
         total_records = 1
-      when 'future', 'current', 'failed'
+      when "future", "current", "failed"
         jobs = Delayed::Job.list_jobs(flavor, page_size, offset)
-        total_records =  Delayed::Job.jobs_count(flavor)
+        total_records = Delayed::Job.jobs_count(flavor)
       else
-        query = params['search_term']
-        if query.present?
-          jobs = Delayed::Job.list_jobs(flavor, page_size, offset, query)
-        else
-          jobs = []
-        end
-        total_records =  Delayed::Job.jobs_count(flavor, query)
+        query = params["search_term"]
+        jobs = if query.present?
+                 Delayed::Job.list_jobs(flavor, page_size, offset, query)
+               else
+                 []
+               end
+        total_records = Delayed::Job.jobs_count(flavor, query)
       end
       json({
-        draw: params['draw'].to_i,
-        recordsTotal: total_records,
-        recordsFiltered: jobs.size,
-        data: build_jobs_json(jobs),
-      })
+             draw: params["draw"].to_i,
+             recordsTotal: total_records,
+             recordsFiltered: jobs.size,
+             data: build_jobs_json(jobs)
+           })
     end
 
-    post '/bulk_update' do
+    post "/bulk_update" do
       content_type :json
 
       halt 403 unless @allow_update
@@ -117,25 +115,24 @@ module Delayed
       Delayed::Job.bulk_update(payload[:action], { ids: payload[:ids] })
 
       json({
-        success: true
-      })
+             success: true
+           })
     end
 
     private
 
     def extract_page_size
-      page_size = Integer(params['length'] || DEFAULT_PAGE_SIZE)
+      page_size = Integer(params["length"] || DEFAULT_PAGE_SIZE)
       # if dataTables wants all of the records it will send us -1 but we don't
       # want the potential to kill our servers with this request so we'll limit it
       page_size = DEFAULT_PAGE_SIZE if page_size == -1
       [page_size, MAX_PAGE_SIZE].min
     end
 
-
     def build_jobs_json(jobs)
-      json = jobs.map{ |j|
-        j.as_json(root: false, except: [:handler, :last_error])
-      }
+      jobs.map do |j|
+        j.as_json(root: false, except: %i[handler last_error])
+      end
     end
   end
 end

data/lib/delayed/settings.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
-require 'yaml'
-require 'erb'
-require 'active_support/core_ext/hash/indifferent_access'
+require "yaml"
+require "erb"
+require "active_support/core_ext/hash/indifferent_access"
 
 module Delayed
   module Settings
     SETTINGS = [
       :default_job_options,
+      :disable_abandoned_job_cleanup,
       :disable_periodic_jobs,
       :disable_automatic_orphan_unlocking,
       :fetch_batch_size,
+      # this is a transitional setting, so that you don't have a time where a
+      # singleton switches to using the singleton column, but there are old
+      # jobs that only used strand
+      :infer_strand_from_singleton,
       :kill_workers_on_exit,
       :last_ditch_logfile,
       :max_attempts,
@@ -23,23 +28,14 @@ module Delayed
       :slow_exit_timeout,
       :worker_health_check_type,
       :worker_health_check_config,
-      :worker_procname_prefix,
-    ]
-    SETTINGS_WITH_ARGS = [
-      :job_detailed_log_format,
-      :num_strands
-    ]
-
-    SETTINGS.each do |setting|
-      mattr_writer(setting)
-      self.send("#{setting}=", nil)
-      define_singleton_method(setting) do
-        val = class_variable_get(:"@@#{setting}")
-        val.respond_to?(:call) ? val.call() : val
-      end
-    end
+      :worker_procname_prefix
+    ].freeze
 
-    mattr_accessor(*SETTINGS_WITH_ARGS)
+    SETTINGS_WITH_ARGS = %i[
+      job_detailed_log_format
+      job_short_log_format
+      num_strands
+    ].freeze
 
     PARENT_PROCESS_DEFAULTS = {
       server_socket_timeout: 10.0,
@@ -49,17 +45,86 @@ module Delayed
 
       # We'll accept a partial, relative path and assume we want it inside
       # Rails.root with inst-jobs.sock appended if provided a directory.
-      server_address: 'tmp',
+      server_address: "tmp"
     }.with_indifferent_access.freeze
 
-    mattr_reader(:parent_process)
-    @@parent_process = PARENT_PROCESS_DEFAULTS.dup
+    class << self
+      attr_accessor(*SETTINGS_WITH_ARGS)
+      attr_reader :parent_process
+
+      SETTINGS.each do |setting|
+        attr_writer setting
+
+        define_method(setting) do
+          val = instance_variable_get(:"@#{setting}")
+          val.respond_to?(:call) ? val.call : val
+        end
+      end
+
+      def queue=(queue_name)
+        raise ArgumentError, "queue_name must not be blank" if queue_name.blank?
+
+        @queue = queue_name
+      end
+
+      def worker_config(config_filename = nil)
+        config_filename ||= default_worker_config_name
+        config = YAML.load(ERB.new(File.read(config_filename)).result)
+        env = Rails.env || "development"
+        config = config[env] || config["default"]
+        # Backwards compatibility from when the config was just an array of queues
+        config = { workers: config } if config.is_a?(Array)
+        unless config.is_a?(Hash)
+          raise ArgumentError,
+                "Invalid config file #{config_filename}"
+        end
+        config = config.with_indifferent_access
+        config[:workers].map! do |worker_config|
+          config.except(:workers).merge(worker_config.with_indifferent_access)
+        end
+        config
+      end
+
+      def apply_worker_config!(config)
+        SETTINGS.each do |setting|
+          send("#{setting}=", config[setting.to_s]) if config.key?(setting.to_s)
+        end
+        if config.key?("parent_process_client_timeout")
+          parent_process.client_timeout = config["parent_process_client_timeout"]
+        end
+        self.parent_process = config["parent_process"] if config.key?("parent_process")
+      end
+
+      def default_worker_config_name
+        expand_rails_path("config/delayed_jobs.yml")
+      end
+
+      # Expands rails-relative paths, without depending on rails being loaded.
+      def expand_rails_path(path)
+        root = if defined?(Rails) && Rails.root
+                 Rails.root.join("Gemfile")
+               else
+                 ENV.fetch("BUNDLE_GEMFILE", "#{Dir.pwd}/Gemfile")
+               end
+        File.expand_path("../#{path}", root)
+      end
+
+      def parent_process_client_timeout=(val)
+        parent_process["server_socket_timeout"] = Integer(val)
+      end
+
+      def parent_process=(new_config)
+        raise "Parent process configurations must be a hash!" unless new_config.is_a?(Hash)
+
+        @parent_process = PARENT_PROCESS_DEFAULTS.merge(new_config)
+      end
 
-    def self.queue=(queue_name)
-      raise(ArgumentError, "queue_name must not be blank") if queue_name.blank?
-      @@queue = queue_name
+      def worker_health_check_config=(new_config)
+        @worker_health_check_config = (new_config || {}).with_indifferent_access
+      end
     end
 
+    self.parent_process = PARENT_PROCESS_DEFAULTS.dup
     self.queue = "queue"
     self.max_attempts = 1
     self.sleep_delay = 2.0
@@ -68,9 +133,12 @@ module Delayed
     self.select_random_from_batch = false
     self.silence_periodic_log = false
 
-    self.num_strands = ->(strand_name){ nil }
-    self.default_job_options = ->{ Hash.new }
-    self.job_detailed_log_format = ->(job){ job.to_json(include_root: false, only: %w(tag strand priority attempts created_at max_attempts source)) }
+    self.num_strands = ->(_strand_name) {}
+    self.default_job_options = -> { {} }
+    self.job_detailed_log_format = lambda { |job|
+      job.to_json(include_root: false, only: %w[tag strand singleton priority attempts created_at max_attempts source])
+    }
+    self.job_short_log_format = ->(_job) { "" }
 
     # Send workers KILL after QUIT if they haven't exited within the
     # slow_exit_timeout
@@ -79,58 +147,5 @@ module Delayed
 
     self.worker_health_check_type = :none
     self.worker_health_check_config = {}
-
-    def self.worker_config(config_filename = nil)
-      config_filename ||= default_worker_config_name
-      config = YAML.load(ERB.new(File.read(config_filename)).result)
-      env = defined?(RAILS_ENV) ? RAILS_ENV : ENV['RAILS_ENV'] || 'development'
-      config = config[env] || config['default']
-      # Backwards compatibility from when the config was just an array of queues
-      config = { :workers => config } if config.is_a?(Array)
-      unless config && config.is_a?(Hash)
-        raise ArgumentError,
-          "Invalid config file #{config_filename}"
-      end
-      config = config.with_indifferent_access
-      config[:workers].map! do |worker_config|
-        config.except(:workers).merge(worker_config.with_indifferent_access)
-      end
-      config
-    end
-
-    def self.apply_worker_config!(config)
-      SETTINGS.each do |setting|
-        self.send("#{setting}=", config[setting.to_s]) if config.key?(setting.to_s)
-      end
-      parent_process.client_timeout = config['parent_process_client_timeout'] if config.key?('parent_process_client_timeout')
-      self.parent_process = config['parent_process'] if config.key?('parent_process')
-    end
-
-    def self.default_worker_config_name
-      expand_rails_path("config/delayed_jobs.yml")
-    end
-
-    # Expands rails-relative paths, without depending on rails being loaded.
-    def self.expand_rails_path(path)
-      root = if defined?(Rails) && Rails.root
-        (Rails.root+"Gemfile").to_s
-      else
-        ENV.fetch('BUNDLE_GEMFILE', Dir.pwd+"/Gemfile")
-      end
-      File.expand_path("../#{path}", root)
-    end
-
-    def self.parent_process_client_timeout=(val)
-      parent_process['server_socket_timeout'] = Integer(val)
-    end
-
-    def self.parent_process=(new_config)
-      raise 'Parent process configurations must be a hash!' unless Hash === new_config
-      @@parent_process = PARENT_PROCESS_DEFAULTS.merge(new_config)
-    end
-
-    def self.worker_health_check_config=(new_config)
-      @@worker_health_check_config = (new_config || {}).with_indifferent_access
-    end
   end
 end

data/lib/delayed/testing.rb

@@ -1,34 +1,33 @@
 # frozen_string_literal: true
 
 module Delayed
-module Testing
-  def self.run_job(job)
-    Delayed::Worker.new.perform(job)
-  end
+  module Testing
+    def self.run_job(job)
+      Delayed::Worker.new.perform(job)
+    end
 
-  def self.drain
-    while job = Delayed::Job.get_and_lock_next_available(
-        'spec run_jobs',
+    def self.drain
+      while (job = Delayed::Job.get_and_lock_next_available(
+        "spec run_jobs",
         Delayed::Settings.queue,
         0,
-        Delayed::MAX_PRIORITY)
-      run_job(job)
+        Delayed::MAX_PRIORITY
+      ))
+        run_job(job)
+      end
     end
-  end
 
-  def self.track_created
-    job_tracking = JobTracking.track { yield }
-    job_tracking.created
-  end
+    def self.track_created(&block)
+      job_tracking = JobTracking.track(&block)
+      job_tracking.created
+    end
 
-  def self.clear_all!
-    case Delayed::Job.name
-    when /Redis/
-      Delayed::Job.redis.flushdb
-    when /ActiveRecord/
-      Delayed::Job.delete_all
-      Delayed::Job::Failed.delete_all
+    def self.clear_all!
+      case Delayed::Job.name
+      when /ActiveRecord/
+        Delayed::Job.delete_all
+        Delayed::Job::Failed.delete_all
+      end
     end
   end
 end
-end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.0.0"
+  VERSION = "3.1.0"
 end

data/lib/delayed/work_queue/in_process.rb

@@ -1,23 +1,27 @@
 # frozen_string_literal: true
 
 module Delayed
-module WorkQueue
-# The simplest possible implementation of a WorkQueue -- just turns around and
-# queries the queue inline.
-class InProcess
-  def get_and_lock_next_available(worker_name, worker_config)
-    Delayed::Worker.lifecycle.run_callbacks(:work_queue_pop, self, worker_config) do
-      Delayed::Job.get_and_lock_next_available(
-        worker_name,
-        worker_config[:queue],
-        worker_config[:min_priority],
-        worker_config[:max_priority])
+  module WorkQueue
+    # The simplest possible implementation of a WorkQueue -- just turns around and
+    # queries the queue inline.
+    class InProcess
+      def get_and_lock_next_available(worker_name, worker_config)
+        Delayed::Worker.lifecycle.run_callbacks(:work_queue_pop, self, worker_config) do
+          Delayed::Job.get_and_lock_next_available(
+            worker_name,
+            worker_config[:queue],
+            worker_config[:min_priority],
+            worker_config[:max_priority]
+          )
+        end
+      end
+
+      # intentional nops for compatibility w/ parent process
+      def init; end
+
+      def close; end
+
+      def wake_up; end
     end
   end
-
-  # intentional nops for compatibility w/ parent process
-  def close; end
-  def wake_up; end
-end
-end
 end

data/lib/delayed/work_queue/parent_process/client.rb

@@ -1,73 +1,75 @@
 # frozen_string_literal: true
 
 module Delayed
-module WorkQueue
-class ParentProcess
-  class Client
-    attr_reader :addrinfo
+  module WorkQueue
+    class ParentProcess
+      class Client
+        attr_reader :addrinfo
 
-    include Delayed::Logging
+        include Delayed::Logging
 
-    def initialize(addrinfo, config: Settings.parent_process)
-      @addrinfo = addrinfo
-      @connect_timeout = config['client_connect_timeout'] || 2
-      @self_pipe = IO.pipe
-    end
+        def initialize(addrinfo, config: Settings.parent_process)
+          @addrinfo = addrinfo
+          @connect_timeout = config["client_connect_timeout"] || 2
+        end
 
-    def close
-      reset_connection
-    end
+        def init
+          @self_pipe ||= IO.pipe # rubocop:disable  Naming/MemoizedInstanceVariableName
+        end
 
-    def get_and_lock_next_available(worker_name, worker_config)
-      Marshal.dump([worker_name, worker_config], socket)
+        def close
+          reset_connection
+        end
 
-      # We're assuming there won't ever be a partial write here so we only need
-      # to wait for anything to be available on the 'wire', this is a valid
-      # assumption because we control the server and it's a Unix domain socket,
-      # not TCP.
-      if socket.eof?
-        # Other end closed gracefully, so should we
-        logger.debug("server closed connection")
-        return reset_connection
-      end
+        def get_and_lock_next_available(worker_name, worker_config)
+          Marshal.dump([worker_name, worker_config], socket)
+
+          # We're assuming there won't ever be a partial write here so we only need
+          # to wait for anything to be available on the 'wire', this is a valid
+          # assumption because we control the server and it's a Unix domain socket,
+          # not TCP.
+          if socket.eof?
+            # Other end closed gracefully, so should we
+            logger.debug("server closed connection")
+            return reset_connection
+          end
+
+          readers, = IO.select([socket, @self_pipe[0]])
 
-      readers, _, _ = IO.select([socket, @self_pipe[0]])
+          if readers.include?(@self_pipe[0])
+            # we're probably exiting so we just want to break out of the blocking read
+            logger.debug("Broke out of select due to being awakened, exiting")
+          else
+            Marshal.load(socket).tap do |response|
+              unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == worker_name)
+                raise(ProtocolError, "response is not a locked job: #{response.inspect}")
+              end
 
-      if readers.include?(@self_pipe[0])
-        # we're probably exiting so we just want to break out of the blocking read
-        logger.debug("Broke out of select due to being awakened, exiting")
-      else
-        Marshal.load(socket).tap do |response|
-          unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == worker_name)
-            raise(ProtocolError, "response is not a locked job: #{response.inspect}")
+              logger.debug("Received job #{response.id}")
+            end
           end
-          logger.debug("Received job #{response.id}")
+        rescue SystemCallError, IOError => e
+          logger.error("Work queue connection lost, reestablishing on next poll. (#{e})")
+          # The work queue process died. Return nil to signal the worker
+          # process should sleep as if no job was found, and then retry.
+          reset_connection
         end
-      end
-    rescue SystemCallError, IOError => ex
-      logger.error("Work queue connection lost, reestablishing on next poll. (#{ex})")
-      # The work queue process died. Return nil to signal the worker
-      # process should sleep as if no job was found, and then retry.
-      reset_connection
-    end
 
-    def wake_up
-      @self_pipe[1].write_nonblock('.', exception: false)
-    end
+        def wake_up
+          @self_pipe[1].write_nonblock(".", exception: false)
+        end
 
-    private
+        private
 
-    def socket
-      @socket ||= @addrinfo.connect(timeout: @connect_timeout)
-    end
+        def socket
+          @socket ||= @addrinfo.connect(timeout: @connect_timeout)
+        end
 
-    def reset_connection
-      if @socket
-        @socket.close
-        @socket = nil
+        def reset_connection
+          @socket&.close
+          @socket = nil
+        end
       end
     end
   end
 end
-end
-end