resque_manager 3.3.0

data/lib/resque_manager/overrides/resque/worker.rb

@@ -0,0 +1,291 @@
+require 'socket'
+
+module Resque
+  class Worker
+    @@local_ip = nil
+
+    def local_ip
+      @@local_ip ||= begin
+        UDPSocket.open do |s|
+          s.connect 'google.com', 1
+          s.addr.last
+        end
+      end
+    end
+
+    # The string representation is the same as the id for this worker
+    # instance. Can be used with `Worker.find`.
+    def to_s
+      @to_s || "#{hostname}(#{local_ip}):#{Process.pid}:#{Thread.current.object_id}:#{Thread.current[:path]}:#{Thread.current[:queues]}"
+    end
+
+    alias_method :id, :to_s
+
+    # When the worker gets the -USR2 signal, to_s may give a different value for the thread and queue portion
+    def pause_key
+      key = to_s.split(':')
+      "worker:#{key.first}:#{key.second}:all_workers:paused"
+    end
+
+    def pid
+      to_s.split(':').second
+    end
+
+    def thread
+      to_s.split(':').third
+    end
+
+    def path
+      to_s.split(':').fourth
+    end
+
+    def queue
+      to_s.split(':').last
+    end
+
+    def workers_in_pid
+      Array(Resque.redis.smembers(:workers)).select { |id| id =~ /\(#{ip}\):#{pid}/ }.map { |id| Resque::Worker.find(id) }.compact
+    end
+
+    def ip
+      to_s.split(':').first[/\b(?:\d{1,3}\.){3}\d{1,3}\b/]
+    end
+
+    def queues_in_pid
+      workers_in_pid.collect { |w| w.queue }
+    end
+
+    #OVERRIDE for multithreaded workers
+    def queues
+      Thread.current[:queues] == "*" ? Resque.queues.sort : Thread.current[:queues].split(',')
+    end
+
+    # Runs all the methods needed when a worker begins its lifecycle.
+    #OVERRIDE for multithreaded workers
+    def startup
+      enable_gc_optimizations
+      if Thread.current == Thread.main
+        register_signal_handlers
+        prune_dead_workers
+      end
+      run_hook :before_first_fork
+      register_worker
+
+      # Fix buffering so we can `rake resque:work > resque.log` and
+      # get output from the child in there.
+      $stdout.sync = true
+    end
+
+    # Schedule this worker for shutdown. Will finish processing the
+    # current job.
+    #OVERRIDE for multithreaded workers
+    def shutdown
+      log 'Exiting...'
+      Thread.list.each { |t| t[:shutdown] = true }
+      @shutdown = true
+    end
+
+    def paused
+      Resque.redis.get pause_key
+    end
+
+    # are we paused?
+    # OVERRIDE so UI can tell if we're paused
+    def paused?
+      @paused || paused.present?
+    end
+
+    # Stop processing jobs after the current one has completed (if we're
+    # currently running one).
+    #OVERRIDE to set a redis key so UI knows it's paused too
+    # Would prefer to call super but get no superclass method error
+    def pause_processing
+      log "USR2 received; pausing job processing"
+      @paused = true
+      Resque.redis.set(pause_key, Time.now.to_s)
+    end
+
+    # Start processing jobs again after a pause
+    #OVERRIDE to set remove redis key so UI knows it's unpaused too
+    # Would prefer to call super but get no superclass method error
+    def unpause_processing
+      log "CONT received; resuming job processing"
+      @paused = false
+      Resque.redis.del(pause_key)
+    end
+
+    # Looks for any workers which should be running on this server
+    # and, if they're not, removes them from Redis.
+    #
+    # This is a form of garbage collection. If a server is killed by a
+    # hard shutdown, power failure, or something else beyond our
+    # control, the Resque workers will not die gracefully and therefor
+    # will leave stale state information in Redis.
+    #
+    # By checking the current Redis state against the actual
+    # environment, we can determine if Redis is old and clean it up a bit.
+    def prune_dead_workers
+      Worker.all.each do |worker|
+        host, pid, thread, path, queues = worker.id.split(':')
+        next unless host.include?(hostname)
+        next if worker_pids.include?(pid)
+        log! "Pruning dead worker: #{worker}"
+        worker.unregister_worker
+      end
+    end
+
+    # Unregisters ourself as a worker. Useful when shutting down.
+    # OVERRIDE to also remove the pause key
+    # Would prefer to call super but get no superclass method error
+    def unregister_worker_with_pause(exception = nil)
+      unregister_worker_without_pause(exception)
+
+      Resque.redis.del(pause_key)
+    end
+    alias_method_chain :unregister_worker, :pause
+
+    def all_workers_in_pid_working
+      workers_in_pid.select { |w| (hash = w.processing) && !hash.empty? }
+    end
+
+    # This is the main workhorse method. Called on a Worker instance,
+    # it begins the worker life cycle.
+    #
+    # The following events occur during a worker's life cycle:
+    #
+    # 1. Startup:   Signals are registered, dead workers are pruned,
+    #               and this worker is registered.
+    # 2. Work loop: Jobs are pulled from a queue and processed.
+    # 3. Teardown:  This worker is unregistered.
+    #
+    # Can be passed an integer representing the polling frequency.
+    # The default is 5 seconds, but for a semi-active site you may
+    # want to use a smaller value.
+    #
+    # Also accepts a block which will be passed the job as soon as it
+    # has completed processing. Useful for testing.
+    #OVERRIDE for multithreaded workers
+    def work(interval = 5.0, &block)
+      interval = Float(interval)
+      $0 = "resque: Starting"
+      startup
+
+      loop do
+        break if shutdown? || Thread.current[:shutdown]
+
+        if not paused? and job = reserve
+          log "got: #{job.inspect}"
+          job.worker = self
+          working_on job
+
+          procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class}]"
+          if @child = fork(job) do
+            unregister_signal_handlers if term_child
+            reconnect
+            perform(job, &block)
+            exit! unless run_at_exit_hooks
+          end
+
+            srand # Reseeding
+            procline "Forked #{@child} at #{Time.now.to_i}"
+            begin
+              Process.waitpid(@child)
+            rescue SystemCallError
+              nil
+            end
+            job.fail(DirtyExit.new($?.to_s)) if $?.signaled?
+          else
+            reconnect
+            perform(job, &block)
+          end
+          done_working
+          @child = nil
+        else
+          break if interval.zero?
+          log! "Sleeping for #{interval} seconds"
+          procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
+          sleep interval
+        end
+      end
+
+      unregister_worker
+      loop do
+        #hang onto the process until all threads are done
+        break if all_workers_in_pid_working.blank?
+        sleep interval.to_i
+      end
+    rescue Exception => exception
+      log "Failed to start worker : #{exception.inspect}"
+
+      unregister_worker(exception)
+    end
+
+    # logic for mappged_mget changed where it returns keys with nil values in latest redis gem.
+    def self.working
+      names = all
+      return [] unless names.any?
+      names.map! { |name| "worker:#{name}" }
+      Resque.redis.mapped_mget(*names).map do |key, value|
+        find key.sub("worker:", '') unless value.nil?
+      end.compact
+    end
+
+    def overview_message=(message)
+      data = encode(job.merge('overview_message' => message))
+      Resque.redis.set("worker:#{self}", data)
+    end
+
+    def overview_message
+      job['overview_message']
+    end
+
+    def self.start(options)
+      ips = options[:hosts]
+      application_path = options[:application_path]
+      queues = options[:queues]
+      if Rails.env =~ /development|test/
+        Thread.new(application_path, queues) { |application_path, queue| system("cd #{application_path || '.'}; bundle exec #{ResqueManager.resque_worker_rake || 'rake'} RAILS_ENV=#{Rails.env} QUEUE=#{queue} resque:work") }
+      else
+        Thread.new(ips, application_path, queues) { |ip_list, application_path, queue| system("cd #{Rails.root}; bundle exec cap #{Rails.env} resque:work host=#{ip_list} application_path=#{application_path} queue=#{queue}") }
+      end
+    end
+
+    def quit
+      if Rails.env =~ /development|test/
+        if RUBY_PLATFORM =~ /java/
+          #jruby doesn't trap the -QUIT signal
+          #-TERM gracefully kills the main pid and does a -9 on the child if there is one.
+          #Since jruby doesn't fork a child, the main worker is gracefully killed.
+          system("kill -TERM  #{self.pid}")
+        else
+          system("kill -QUIT  #{self.pid}")
+        end
+      else
+        system("cd #{Rails.root}; bundle exec cap #{Rails.env} resque:quit_worker pid=#{self.pid} host=#{self.ip} application_path=#{self.path}")
+      end
+    end
+
+    def pause
+      if Rails.env =~ /development|test/
+        system("kill -USR2  #{self.pid}")
+      else
+        system("cd #{Rails.root}; bundle exec cap #{Rails.env} resque:pause_worker pid=#{self.pid} host=#{self.ip}")
+      end
+    end
+
+    def continue
+      if Rails.env =~ /development|test/
+        system("kill -CONT  #{self.pid}")
+      else
+        system("cd #{Rails.root}; bundle exec cap #{Rails.env} resque:continue_worker pid=#{self.pid} host=#{self.ip}")
+      end
+    end
+
+    def restart
+      queues = self.queues_in_pid.join('#')
+      quit
+      self.class.start(hosts: self.ip, queues: queues, application_path: self.path)
+    end
+
+  end
+end

data/lib/resque_manager/overrides/resque_scheduler/resque_scheduler.rb

@@ -0,0 +1,58 @@
+module ResqueScheduler
+  def schedule=(schedule_hash)
+    raise 'not implemented'
+  end
+
+  # Returns the schedule hash
+  def schedule
+    #the scheduler gem expects a hash, but it's now stored in
+    #redis as an array.
+    hash = {}
+    Resque.list_range(:scheduled, 0, -0).each do |job|
+      hash.merge! job
+    end
+    hash
+  end
+
+  def self.start(ips)
+    if Rails.env =~ /development|test/
+      Thread.new{system("rake resque:scheduler")}
+    else
+      Thread.new(ips){|ip_list|system("cd #{Rails.root}; #{ResqueManager::Cap.path} #{Rails.env} resque:scheduler host=#{ip_list}")}
+    end
+  end
+
+  def self.quit(ips)
+    if Rails.env =~ /development|test/
+      system("rake resque:quit_scheduler")
+    else
+      system("cd #{Rails.root}; #{ResqueManager::Cap.path} #{Rails.env} resque:quit_scheduler host=#{ips}")
+    end
+  end
+
+  def self.restart(ips)
+    quit(ips)
+    start(ips)
+  end
+
+  def self.farm_status
+    status = {}
+    if Rails.env =~ /development|test/
+      status['localhost'] = pids.present? ? 'Running' : 'Stopped'
+    else
+      Resque.schedule.values.collect{|job| job['ip']}.each do |ip|
+        cap = `cd #{Rails.root}; #{ResqueManager::Cap.path} #{Rails.env} resque:scheduler_status hosts=#{ip}`
+        status[ip] = cap =~ /resque:scheduler is up/ ? 'Running' : 'Stopped'
+      end
+    end
+    status
+  end
+
+  # Returns an array of string pids of all the other workers on this
+  # machine. Useful when pruning dead workers on startup.
+  def self.pids
+    `ps -A -o pid,command | grep [r]esque:scheduler`.split("\n").map do |line|
+      line.split(' ')[0]
+    end
+  end
+end

data/lib/resque_manager/overrides/resque_status/chained_status.rb

@@ -0,0 +1,46 @@
+module Resque
+  module Plugins
+    module ChainedStatus
+
+      def self.included(base)
+        base.class_eval do
+          include Resque::Plugins::Status
+          extend ClassOverrides
+          include InstanceOverrides
+        end
+      end
+
+      module InstanceOverrides
+        # OVERRIDE to just use the name of it's parent job.
+        def name
+          status.name rescue nil
+        end
+
+        def completed(*messages)
+          super(*messages)
+          # "You must override this method to provide your own logic of when to actually call complete."
+          #      if counter(:processed) >= options['total']
+          #        super
+          #      end
+        end
+      end
+
+      module ClassOverrides
+        # OVERRIDE to grab the uuid out of options so it can be chained to the calling worker
+        # instead of creating a new uuid.
+        def enqueue_to(queue, klass, options = {})
+          #tie this job to the status of the calling job
+          opts = HashWithIndifferentAccess.new(options)
+          raise ArgumentError, "You must supply a :uuid attribute in your call to create." unless opts['uuid']
+          uuid = opts['uuid']
+          if Resque.enqueue_to(queue, klass, uuid, options)
+            uuid
+          else
+            Resque::Plugins::Status::Hash.remove(uuid)
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque_manager/overrides/resque_status/hash.rb

@@ -0,0 +1,12 @@
+module Resque
+  module Plugins
+    module Status
+      class Hash < ::Hash
+        # The STATUSES constant is frozen, so we'll just manually add the paused? method here
+        def paused?
+          self['status'] === 'paused'
+        end
+      end
+    end
+  end
+end

data/lib/resque_manager/overrides/resque_status/status.rb

@@ -0,0 +1,161 @@
+module Resque
+  module Plugins
+    module Status
+
+
+      #OVERRIDE so we can add OverridesAndExtensionsClassMethods
+      def self.included(base)
+        attr_reader :worker
+
+        # can't call super, so add ClassMethods here that resque-status was doing
+        base.extend(ClassMethods) #add the methods in the resque-status gem
+        base.extend(ClassOverridesAndExtensions)
+      end
+
+      module ClassOverridesAndExtensions
+
+        #OVERRIDE to set the name that will be displayed on the status page for this job when it is first queued.
+        #The name will be changed when set_status is called, which is called on #tick, to the value set in your name method,
+        #but the UI name field is blank when it is first queued, so setting it here so we have something.
+        def enqueue_to(queue, klass, options = {})
+          uuid = Resque::Plugins::Status::Hash.generate_uuid
+          Resque::Plugins::Status::Hash.create uuid, {name: "#{self.name}: #{options.inspect}"}.merge(options)
+
+          if Resque.enqueue_to(queue, klass, uuid, options)
+            uuid
+          else
+            Resque::Plugins::Status::Hash.remove(uuid)
+            nil
+          end
+        end
+
+        # This is the method called by Resque::Worker when processing jobs. It
+        # creates a new instance of the job class and populates it with the uuid and
+        # options.
+        #
+        # You should not override this method, rather the <tt>perform</tt> instance method.
+        # OVERRIDE to get the worker and set when initializing the class
+        def perform(uuid=nil, options = {})
+          uuid ||= Resque::Plugins::Status::Hash.generate_uuid
+          worker = yield if block_given?
+          instance = new(uuid, worker, options)
+          instance.safe_perform!
+          instance
+        end
+
+        # OVERRIDE to clear all the keys that have the UUI. status, counters, etc.
+        def remove(uuid)
+          Resque.redis.zrem(set_key, uuid)
+          Resque.redis.keys("*#{uuid}").each do |key|
+            Resque.redis.del(key)
+          end
+        end
+
+        #If multiple workers are running at once and you need an incrementer, you can't use the status' num attribute because of race conditions.
+        #You can use a counter and call incr on it instead
+        def counter_key(counter, uuid)
+          "#{counter}:#{uuid}"
+        end
+
+        def counter(counter, uuid)
+          Resque.redis[counter_key(counter, uuid)].to_i
+        end
+
+        def incr_counter(counter, uuid)
+          key = counter_key(counter, uuid)
+          n = Resque.redis.incr(key)
+          if Resque::Plugins::Status::Hash.expire_in
+            Resque.redis.expire(key, Resque::Plugins::Status::Hash.expire_in)
+          end
+          n
+        end
+      end
+
+      # sets the status of the job for the current iteration. You should use
+      # the <tt>at</tt> method if you have actual numbers to track the iteration count.
+      # This will kill the job if it has been added to the kill list with
+      # <tt>Resque::Status.kill()</tt>
+      def tick(*messages)
+        kill! if should_kill? || status.killed?
+        set_status({'status' => 'working'}, *messages)
+        # check to see if the worker doing the job has been paused, pause the job if so
+        if self.worker && self.worker.paused?
+          loop do
+            # Set the status to paused.
+            # May need to do this repeatedly because there could be workers in a chained job still doing work.
+            pause! unless status.paused?
+            break unless self.worker.paused?
+            sleep 60
+          end
+          set_status({'status' => 'working'}, *messages) unless status && (status.completed? || status.paused? || status.killed?)
+        end
+      end
+
+      # Pause the current job, setting the status to 'paused'
+      def pause!
+        set_status({
+                       'status' => 'paused',
+                       'message' => "#{worker} paused at #{Time.now}"
+                   })
+      end
+
+      # Create a new instance with <tt>uuid</tt> and <tt>options</tt>
+      # OVERRIDE to add the worker attr
+      def initialize(uuid, worker = nil, options = {})
+        @uuid = uuid
+        @options = options
+        @worker = worker
+      end
+
+      # Run by the Resque::Worker when processing this job. It wraps the <tt>perform</tt>
+      # method ensuring that the final status of the job is set regardless of error.
+      # If an error occurs within the job's work, it will set the status as failed and
+      # re-raise the error.
+      #OVERRIDE to kill it.  The parent job may have been killed, so all child jobs should die as well.
+      def safe_perform!
+        k = should_kill?
+        kill! if k
+        unless k || (status && status.killed?)
+          set_status({'status' => 'working'})
+          perform
+          if status && status.failed?
+            on_failure(status.message) if respond_to?(:on_failure)
+            return
+          elsif status && !status.completed?
+            completed
+          end
+          on_success if respond_to?(:on_success)
+        end
+      rescue Killed
+        Rails.logger.info "Job #{self} Killed at #{Time.now}"
+        Resque::Plugins::Status::Hash.killed(uuid)
+        on_killed if respond_to?(:on_killed)
+      rescue => e
+        Rails.logger.error e
+        failed("The task failed because of an error: #{e}")
+        if respond_to?(:on_failure)
+          on_failure(e)
+        else
+          raise e
+        end
+      end
+
+      # sets a message for the job on the overview page
+      # it can be set repeatedly durring the job's processing to
+      # indicate the status of the job.
+      def overview_message=(message)
+        # there is no worker when run inline
+        self.worker.overview_message = message if self.worker
+      end
+
+      def incr_counter(counter)
+        self.class.incr_counter(counter, uuid)
+      end
+
+      def counter(counter)
+        self.class.counter(counter, uuid)
+      end
+
+    end
+  end
+end