inst-jobs 0.12.3 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb490099d8b25310bcfd0c7f419dd095277f9d11
-  data.tar.gz: 2be27c6ea2e6d37b6739d6ef19dcf971a0bcf81e
+  metadata.gz: 59c2643528ac5d09160a35e1d4fd338d058b593d
+  data.tar.gz: 69e5d68901df56c91fc10df1eaafb973877ef9fa
 SHA512:
-  metadata.gz: 935a70be1f89625da0a07d96a02cbc4b21bfcd414d93e260582c266a987be59f947ff7084fa723ff8edebbca6262a4619e41ca88cac95c125b4c81c289e8cf5c
-  data.tar.gz: 4930328d5b9c06f595425f11ae11c03113a4cfaa96327062e6c085e4da354d86c4bc1fcd4f54bbbe983e1213acf4d85fd14aaee5ce37f826ce7b22837f95b594
+  metadata.gz: e9b5d4be7a15c770a825beffc9d175a0f6b6318e6df899af14995f90f52dc24f23adc7b4f90321e8cb9eba2818ed0ac7889394d382ce86b18fe58d61aef91f60
+  data.tar.gz: 31d950b9b5bc5d843f9ed04f316f0ce076b3ba4e32f5284c499a1ed1cd4d4ab36d674246fdb31fdefde8d25b02a42e84e3568f5af567fb35637250a91e37f1c3

data/lib/delayed/backend/active_record.rb

@@ -16,6 +16,13 @@ module Delayed
       class Job < ::ActiveRecord::Base
         include Delayed::Backend::Base
         self.table_name = :delayed_jobs
+        # Rails hasn't completely loaded yet, and setting the table name will cache some stuff
+        # so reset that cache so that it will load correctly after Rails is all loaded
+        # It's fixed in Rails 5 to not cache anything when you set the table_name
+        if Rails.version < '5' && Rails.version >= '4.2'
+          @arel_engine = nil
+          @arel_table = nil
+        end
 
         def self.reconnect!
           clear_all_connections!
@@ -201,7 +208,9 @@ module Delayed
         def self.get_and_lock_next_available(worker_names,
                                              queue = Delayed::Settings.queue,
                                              min_priority = nil,
-                                             max_priority = nil)
+                                             max_priority = nil,
+                                             extra_jobs: 0,
+                                             extra_jobs_owner: nil)
 
           check_queue(queue)
           check_priorities(min_priority, max_priority)
@@ -216,7 +225,7 @@ module Delayed
                 effective_worker_names = Array(worker_names)
 
                 target_jobs = all_available(queue, min_priority, max_priority).
-                    limit(effective_worker_names.length).
+                    limit(effective_worker_names.length + extra_jobs).
                     lock
                 jobs_with_row_number = all.from(target_jobs).
                     select("id, ROW_NUMBER() OVER () AS row_number")
@@ -224,6 +233,9 @@ module Delayed
                 effective_worker_names.each_with_index do |worker, i|
                   updates << "WHEN #{i + 1} THEN #{connection.quote(worker)} "
                 end
+                if extra_jobs_owner
+                  updates << "ELSE #{connection.quote(extra_jobs_owner)} "
+                end
                 updates << "END, locked_at = #{connection.quote(db_time_now)}"
                 # joins and returning in an update! just bypass AR
                 query = "UPDATE #{quoted_table_name} SET #{updates} FROM (#{jobs_with_row_number.to_sql}) j2 WHERE j2.id=delayed_jobs.id RETURNING delayed_jobs.*"
@@ -231,7 +243,14 @@ module Delayed
                 # because this is an atomic query, we don't have to return more jobs than we needed
                 # to try and lock them, nor is there a possibility we need to try again because
                 # all of the jobs we tried to lock had already been locked by someone else
-                return worker_names.is_a?(Array) ? jobs.index_by(&:locked_by) : jobs.first
+                if worker_names.is_a?(Array)
+                  result = jobs.index_by(&:locked_by)
+                  # all of the extras can come back as an array
+                  result[extra_jobs_owner] = jobs.select { |j| j.locked_by == extra_jobs_owner } if extra_jobs_owner
+                  return result
+                else
+                  return jobs.first
+                end
               else
                 batch_size = Settings.fetch_batch_size
                 batch_size *= worker_names.length if worker_names.is_a?(Array)
@@ -314,6 +333,12 @@ module Delayed
           end
         end
 
+        def self.unlock(jobs)
+          unlocked = where(id: jobs).update_all(locked_at: nil, locked_by: nil)
+          jobs.each(&:unlock)
+          unlocked
+        end
+
         # Lock this job for this worker.
         # Returns true if we have the lock, false otherwise.
         #
@@ -332,6 +357,18 @@ module Delayed
           end
         end
 
+        def transfer_lock!(from:, to:)
+          now = self.class.db_time_now
+          # We don't own this job so we will update the locked_by name and the locked_at
+          affected_rows = self.class.where(id: self, locked_by: from).update_all(locked_at: now, locked_by: to)
+          if affected_rows == 1
+            mark_as_locked!(now, to)
+            return true
+          else
+            return false
+          end
+        end
+
         def mark_as_locked!(time, worker)
           self.locked_at    = time
           self.locked_by    = worker

data/lib/delayed/backend/base.rb

@@ -118,6 +118,13 @@ module Delayed
           Time.now.utc
         end
 
+        def unlock_orphaned_pending_jobs
+          horizon = db_time_now - Settings.parent_process[:pending_jobs_idle_timeout] * 4
+          orphaned_jobs = running_jobs.select { |job| job.locked_by.start_with?('work_queue:') && job.locked_at < horizon }
+          return 0 if orphaned_jobs.empty?
+          unlock(orphaned_jobs)
+        end
+
         def unlock_orphaned_jobs(pid = nil, name = nil)
           begin
             name ||= Socket.gethostname

data/lib/delayed/backend/redis/job.rb

@@ -221,7 +221,9 @@ class Job
   def self.get_and_lock_next_available(worker_name,
       queue = Delayed::Settings.queue,
       min_priority = Delayed::MIN_PRIORITY,
-      max_priority = Delayed::MAX_PRIORITY)
+      max_priority = Delayed::MAX_PRIORITY,
+      extra_jobs: nil,
+      extra_jobs_owner: nil)
 
     check_queue(queue)
     check_priorities(min_priority, max_priority)
@@ -352,9 +354,18 @@ class Job
     self.create!(options.merge(:singleton => true))
   end
 
+  def self.unlock(jobs)
+    jobs.each(&:unlock!)
+    jobs.length
+  end
+
   # not saved, just used as a marker when creating
   attr_accessor :singleton
 
+  def transfer_lock!(from:, to:)
+    lock_in_redis!(to)
+  end
+
   def lock_in_redis!(worker_name)
     self.locked_at = self.class.db_time_now
     self.locked_by = worker_name
@@ -362,8 +373,7 @@ class Job
   end
 
   def unlock!
-    self.locked_at = nil
-    self.locked_by = nil
+    unlock
     save!
   end
 

data/lib/delayed/logging.rb

@@ -0,0 +1,32 @@
+require 'date'
+
+module Delayed
+  module Logging
+    TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%S.%6N'.freeze
+    private_constant :TIMESTAMP_FORMAT
+
+    FORMAT = '%s - %s'
+    private_constant :FORMAT
+
+
+    def self.logger
+      return @logger if @logger
+      if defined?(Rails.logger) && Rails.logger
+        @logger = Rails.logger
+      else
+        @logger = ::Logger.new(STDOUT).tap do |logger|
+          logger.formatter = ->(_, time, _, msg) {
+            FORMAT % [
+              time.strftime(TIMESTAMP_FORMAT),
+              msg
+            ]
+          }
+        end
+      end
+    end
+
+    def logger
+      Delayed::Logging.logger
+    end
+  end
+end

data/lib/delayed/settings.rb

@@ -1,5 +1,6 @@
 require 'yaml'
 require 'erb'
+require 'active_support/core_ext/hash/indifferent_access'
 
 module Delayed
   module Settings
@@ -17,7 +18,6 @@ module Delayed
       :disable_periodic_jobs,
       :disable_automatic_orphan_unlocking,
       :last_ditch_logfile,
-      :parent_process_client_timeout,
     ]
     SETTINGS_WITH_ARGS = [ :num_strands ]
 
@@ -32,6 +32,22 @@ module Delayed
 
     mattr_accessor(*SETTINGS_WITH_ARGS)
 
+    PARENT_PROCESS_DEFAULTS = {
+      server_receive_timeout: 10.0,
+      server_socket_timeout: 10.0,
+      pending_jobs_idle_timeout: 30.0,
+
+      client_connect_timeout: 2.0,
+      client_receive_timeout: 10.0,
+
+      # We'll accept a partial, relative path and assume we want it inside
+      # Rails.root with inst-jobs.sock appended if provided a directory.
+      server_address: 'tmp',
+    }.with_indifferent_access.freeze
+
+    mattr_reader(:parent_process)
+    @@parent_process = PARENT_PROCESS_DEFAULTS.dup
+
     def self.queue=(queue_name)
       raise(ArgumentError, "queue_name must not be blank") if queue_name.blank?
       @@queue = queue_name
@@ -44,7 +60,6 @@ module Delayed
     self.fetch_batch_size = 5
     self.select_random_from_batch = false
     self.silence_periodic_log = false
-    self.parent_process_client_timeout = 10.0
 
     self.num_strands = ->(strand_name){ nil }
     self.default_job_options = ->{ Hash.new }
@@ -71,6 +86,8 @@ module Delayed
       SETTINGS.each do |setting|
         self.send("#{setting}=", config[setting.to_s]) if config.key?(setting.to_s)
       end
+      parent_process.client_timeout = config['parent_process_client_timeout'] if config.key?('parent_process_client_timeout')
+      parent_process = config['parent_process'] if config.key?('parent_process')
     end
 
     def self.default_worker_config_name
@@ -86,5 +103,14 @@ module Delayed
       end
       File.expand_path("../#{path}", root)
     end
+
+    def self.parent_process_client_timeout=(val)
+      parent_process['server_socket_timeout'] = Integer(val)
+    end
+
+    def self.parent_process=(new_config)
+      raise 'Parent process configurations must be a hash!' unless Hash === new_config
+      @@parent_process = PARENT_PROCESS_DEFAULTS.merge(new_config)
+    end
   end
 end

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.12.3"
+  VERSION = "0.13.0"
 end

data/lib/delayed/work_queue/parent_process.rb

@@ -1,7 +1,10 @@
+require 'pathname'
 require 'socket'
-require 'tempfile'
 require 'timeout'
 
+require_relative 'parent_process/client'
+require_relative 'parent_process/server'
+
 module Delayed
 module WorkQueue
 # ParentProcess is a WorkQueue implementation that spawns a separate worker
@@ -24,200 +27,41 @@ class ParentProcess
   class ProtocolError < RuntimeError
   end
 
-  def initialize
-    @path = self.class.generate_socket_path
-  end
+  attr_reader :server_address
+
+  DEFAULT_SOCKET_NAME = 'inst-jobs.sock'.freeze
+  private_constant :DEFAULT_SOCKET_NAME
 
-  def self.generate_socket_path
-    # We utilize Tempfile as a convenient way to get a socket filename in the
-    # writeable temp directory. However, since we destroy the normal file and
-    # write a unix socket file to the same location, we lose the hard uniqueness
-    # guarantees of Tempfile. This is OK for this use case, we only generate one
-    # Tempfile with this prefix.
-    tmp = Tempfile.new("inst-jobs-#{Process.pid}-")
-    path = tmp.path
-    tmp.close!
-    path
+  def initialize(config = Settings.parent_process)
+    @config = config
+    @server_address = generate_socket_path(config['server_address'])
   end
 
   def server(parent_pid: nil)
     # The unix_server_socket method takes care of cleaning up any existing
     # socket for us if the work queue process dies and is restarted.
-    listen_socket = Socket.unix_server_socket(@path)
-    Server.new(listen_socket, parent_pid: parent_pid)
+    listen_socket = Socket.unix_server_socket(@server_address)
+    Server.new(listen_socket, parent_pid: parent_pid, config: @config)
   end
 
   def client
-    Client.new(Addrinfo.unix(@path))
-  end
-
-  module SayUtil
-    def say(msg, level = :debug)
-      if defined?(Rails.logger) && Rails.logger
-        message = -> { "[#{Process.pid}]Q #{msg}" }
-        Rails.logger.send(level, self.class.name, &message)
-      else
-        puts(msg)
-      end
-    end
+    Client.new(Addrinfo.unix(@server_address), config: @config)
   end
 
-  class Client
-    attr_reader :addrinfo
+  private
 
-    include SayUtil
+  def generate_socket_path(supplied_path)
+    pathname = Pathname.new(supplied_path)
 
-    def initialize(addrinfo)
-      @addrinfo = addrinfo
-    end
-
-    def get_and_lock_next_available(worker_name, worker_config)
-      @socket ||= @addrinfo.connect
-      say("Requesting work using #{@socket.inspect}")
-      Marshal.dump([worker_name, worker_config], @socket)
-      response = Marshal.load(@socket)
-      unless response.nil? || (response.is_a?(Delayed::Job) && response.locked_by == worker_name)
-        say("Received invalid response from server: #{response.inspect}")
-        raise(ProtocolError, "response is not a locked job: #{response.inspect}")
-      end
-      say("Received work from server: #{response.inspect}")
-      response
-    rescue SystemCallError, IOError => ex
-      say("Work queue connection lost, reestablishing on next poll. (#{ex})", :error)
-      # The work queue process died. Return nil to signal the worker
-      # process should sleep as if no job was found, and then retry.
-      @socket = nil
-      nil
+    if pathname.absolute? && pathname.directory?
+      pathname.join(DEFAULT_SOCKET_NAME).to_s
+    elsif pathname.absolute?
+      supplied_path
+    else
+      generate_socket_path(Settings.expand_rails_path(supplied_path))
     end
   end
-
-  class Server
-    attr_reader :listen_socket
-
-    include SayUtil
-
-    def initialize(listen_socket, parent_pid: nil)
-      @listen_socket = listen_socket
-      @parent_pid = parent_pid
-      @clients = {}
-      @waiting_clients = {}
-    end
-
-    def connected_clients
-      @clients.size
-    end
-
-    def all_workers_idle?
-      !@clients.any? { |_, c| c.working }
-    end
-
-    # run the server queue worker
-    # this method does not return, only exits or raises an exception
-    def run
-      say "Starting work queue process"
-
-      while !exit?
-        run_once
-      end
-
-    rescue => e
-      say "WorkQueue Server died: #{e.inspect}", :error
-      raise
-    end
-
-    def run_once
-      handles = @clients.keys + [@listen_socket]
-      timeout = Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)
-      readable, _, _ = IO.select(handles, nil, nil, timeout)
-      if readable
-        readable.each { |s| handle_read(s) }
-      end
-      check_for_work
-    end
-
-    def handle_read(socket)
-      if socket == @listen_socket
-        handle_accept
-      else
-        handle_request(socket)
-      end
-    end
-
-    # Any error on the listen socket other than WaitReadable will bubble up
-    # and terminate the work queue process, to be restarted by the parent daemon.
-    def handle_accept
-      socket, _addr = @listen_socket.accept_nonblock
-      if socket
-        @clients[socket] = ClientState.new(false, socket)
-      end
-    rescue IO::WaitReadable
-      say("Server attempted to read listen_socket but failed with IO::WaitReadable", :error)
-      # ignore and just try accepting again next time through the loop
-    end
-
-    def handle_request(socket)
-      # There is an assumption here that the client will never send a partial
-      # request and then leave the socket open. Doing so would leave us hanging
-      # here forever. This is only a reasonable assumption because we control
-      # the client.
-      worker_name, worker_config = client_timeout { Marshal.load(socket) }
-      client = @clients[socket]
-      client.name = worker_name
-      client.working = false
-      (@waiting_clients[worker_config] ||= []) << client
-
-    rescue SystemCallError, IOError, Timeout::Error => ex
-      say("Receiving message from client (#{socket}) failed: #{ex.inspect}", :error)
-      drop_socket(socket)
-    end
-
-    def check_for_work
-      @waiting_clients.each do |(worker_config, workers)|
-        next if workers.empty?
-
-        Delayed::Worker.lifecycle.run_callbacks(:work_queue_pop, self, worker_config) do
-          response = Delayed::Job.get_and_lock_next_available(
-              workers.map(&:name),
-              worker_config[:queue],
-              worker_config[:min_priority],
-              worker_config[:max_priority])
-          response.each do |(worker_name, job)|
-            client = workers.find { |worker| worker.name == worker_name }
-            client.working = true
-            @waiting_clients[worker_config].delete(client)
-            begin
-              client_timeout { Marshal.dump(job, client.socket) }
-            rescue SystemCallError, IOError, Timeout::Error
-              drop_socket(client.socket)
-            end
-          end
-        end
-      end
-    end
-
-    def drop_socket(socket)
-      # this socket went away
-      begin
-        socket.close
-      rescue IOError
-      end
-      @clients.delete(socket)
-    end
-
-    def exit?
-      parent_exited?
-    end
-
-    def parent_exited?
-      @parent_pid && @parent_pid != Process.ppid
-    end
-
-    def client_timeout
-      Timeout.timeout(Settings.parent_process_client_timeout) { yield }
-    end
-
-    ClientState = Struct.new(:working, :socket, :name)
-  end
 end
 end
 end
+