resque 1.27.4 → 2.6.0

data/lib/resque/web_runner.rb

@@ -0,0 +1,374 @@
+require 'open-uri'
+require 'logger'
+require 'optparse'
+require 'fileutils'
+require 'rack'
+require 'resque/server'
+
+# only used with `bin/resque-web`
+# https://github.com/resque/resque/pull/1780
+
+module Resque
+  WINDOWS = !!(RUBY_PLATFORM =~ /(mingw|bccwin|wince|mswin32)/i)
+  JRUBY = !!(RbConfig::CONFIG["RUBY_INSTALL_NAME"] =~ /^jruby/i)
+
+  class WebRunner
+    attr_reader :app, :app_name, :filesystem_friendly_app_name,
+      :rack_handler, :port, :options, :args
+
+    PORT       = 5678
+    HOST       = WINDOWS ? 'localhost' : '0.0.0.0'
+
+    def initialize(*runtime_args)
+      @options = runtime_args.last.is_a?(Hash) ? runtime_args.pop : {}
+
+      self.class.logger.level = options[:debug] ? Logger::DEBUG : Logger::INFO
+
+      @app      = Resque::Server
+      @app_name = 'resque-web'
+      @filesystem_friendly_app_name = @app_name.gsub(/\W+/, "_")
+
+      @args = load_options(runtime_args)
+
+      @rack_handler = (s = options[:rack_handler]) ? Rack::Handler.get(s) : setup_rack_handler
+
+      case option_parser.command
+      when :help
+        puts option_parser
+      when :kill
+        kill!
+      when :status
+        status
+      when :version
+        puts "resque #{Resque::VERSION}"
+        puts "rack #{Rack::VERSION.join('.')}"
+        puts "sinatra #{Sinatra::VERSION}" if defined?(Sinatra)
+      else
+        before_run
+        start unless options[:start] == false
+      end
+    end
+
+    def launch_path
+      if options[:launch_path].respond_to?(:call)
+        options[:launch_path].call(self)
+      else
+        options[:launch_path]
+      end
+    end
+
+    def app_dir
+      if !options[:app_dir] && !ENV['HOME']
+        raise ArgumentError.new("nor --app-dir neither ENV['HOME'] defined")
+      end
+      options[:app_dir] || File.join(ENV['HOME'], filesystem_friendly_app_name)
+    end
+
+    def pid_file
+      options[:pid_file] || File.join(app_dir, "#{filesystem_friendly_app_name}.pid")
+    end
+
+    def url_file
+      options[:url_file] || File.join(app_dir, "#{filesystem_friendly_app_name}.url")
+    end
+
+    def log_file
+      options[:log_file] || File.join(app_dir, "#{filesystem_friendly_app_name}.log")
+    end
+
+    def host
+      options.fetch(:host) { HOST }
+    end
+
+    def url
+      "http://#{host}:#{port}"
+    end
+
+    def before_run
+      if (redis_conf = options[:redis_conf])
+        logger.info "Using Redis connection '#{redis_conf}'"
+        Resque.redis = redis_conf
+      end
+      if (namespace = options[:redis_namespace])
+        logger.info "Using Redis namespace '#{namespace}'"
+        Resque.redis.namespace = namespace
+      end
+      if (url_prefix = options[:url_prefix])
+        logger.info "Using URL Prefix '#{url_prefix}'"
+        Resque::Server.url_prefix = url_prefix
+      end
+      app.set(options.merge web_runner: self)
+      path = (ENV['RESQUECONFIG'] || args.first)
+      load_config_file(path.to_s.strip) if path
+    end
+
+    def start(path = launch_path)
+      logger.info "Running with Windows Settings" if WINDOWS
+      logger.info "Running with JRuby" if JRUBY
+      logger.info "Starting '#{app_name}'..."
+
+      check_for_running(path)
+      find_port
+      write_url
+      launch!(url, path)
+      daemonize! unless options[:foreground]
+      run!
+    rescue RuntimeError => e
+      logger.warn "There was an error starting '#{app_name}': #{e}"
+      exit
+    end
+
+    def find_port
+      if @port = options[:port]
+        announce_port_attempted
+
+        unless port_open?
+          logger.warn "Port #{port} is already in use. Please try another. " +
+                      "You can also omit the port flag, and we'll find one for you."
+        end
+      else
+        @port = PORT
+        announce_port_attempted
+
+        until port_open?
+          @port += 1
+          announce_port_attempted
+        end
+      end
+    end
+
+    def announce_port_attempted
+      logger.info "trying port #{port}..."
+    end
+
+    def port_open?(check_url = nil)
+      begin
+        check_url ||= url
+        options[:no_proxy] ? uri_open(check_url, :proxy => nil) : uri_open(check_url)
+        false
+      rescue Errno::ECONNREFUSED, Errno::EPERM, Errno::ETIMEDOUT
+        true
+      end
+    end
+
+    def uri_open(*args)
+      (RbConfig::CONFIG['ruby_version'] < '2.7') ? open(*args) : URI.open(*args)
+    end
+
+    def write_url
+      # Make sure app dir is setup
+      FileUtils.mkdir_p(app_dir)
+      File.open(url_file, 'w') {|f| f << url }
+    end
+
+    def check_for_running(path = nil)
+      if File.exist?(pid_file) && File.exist?(url_file)
+        running_url = File.read(url_file)
+        if !port_open?(running_url)
+          logger.warn "'#{app_name}' is already running at #{running_url}"
+          launch!(running_url, path)
+          exit!(1)
+        end
+      end
+    end
+
+    def run!
+      logger.info "Running with Rack handler: #{@rack_handler.inspect}"
+
+      rack_handler.run app, :Host => host, :Port => port do |server|
+        kill_commands.each do |command|
+          trap(command) do
+            ## Use thins' hard #stop! if available, otherwise just #stop
+            server.respond_to?(:stop!) ? server.stop! : server.stop
+            logger.info "'#{app_name}' received INT ... stopping"
+            delete_pid!
+          end
+        end
+      end
+    end
+
+    # Adapted from Rackup
+    def daemonize!
+      if JRUBY
+        # It's not a true daemon but when executed with & works like one
+        thread = Thread.new {daemon_execute}
+        thread.join
+
+      elsif RUBY_VERSION < "1.9"
+        logger.debug "Parent Process: #{Process.pid}"
+        exit!(0) if fork
+        logger.debug "Child Process: #{Process.pid}"
+        daemon_execute
+
+      else
+        Process.daemon(true, true)
+        daemon_execute
+      end
+    end
+
+    def daemon_execute
+      File.umask 0000
+      FileUtils.touch log_file
+      STDIN.reopen    log_file
+      STDOUT.reopen   log_file, "a"
+      STDERR.reopen   log_file, "a"
+
+      logger.debug "Child Process: #{Process.pid}"
+
+      File.open(pid_file, 'w') {|f| f.write("#{Process.pid}") }
+      at_exit { delete_pid! }
+    end
+
+    def launch!(specific_url = nil, path = nil)
+      return if options[:skip_launch]
+      cmd = WINDOWS ? "start" : "open"
+      system "#{cmd} #{specific_url || url}#{path}"
+    end
+
+    def kill!
+      pid = File.read(pid_file)
+      logger.warn "Sending #{kill_command} to #{pid.to_i}"
+      Process.kill(kill_command, pid.to_i)
+    rescue => e
+      logger.warn "pid not found at #{pid_file} : #{e}"
+    end
+
+    def status
+      if File.exist?(pid_file)
+        logger.info "'#{app_name}' running"
+        logger.info "PID #{File.read(pid_file)}"
+        logger.info "URL #{File.read(url_file)}" if File.exist?(url_file)
+      else
+        logger.info "'#{app_name}' not running!"
+      end
+    end
+
+    # Loads a config file at config_path and evals it in the context of the @app.
+    def load_config_file(config_path)
+      abort "Can not find config file at #{config_path}" if !File.readable?(config_path)
+      config = File.read(config_path)
+      # trim off anything after __END__
+      config.sub!(/^__END__\n.*/, '')
+      @app.module_eval(config)
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def self.logger
+      @logger ||= LOGGER if defined?(LOGGER)
+      if !@logger
+        @logger           = Logger.new(STDOUT)
+        @logger.formatter = Proc.new {|s, t, n, msg| "[#{t}] #{msg}\n"}
+        @logger
+      end
+      @logger
+    end
+
+    def logger
+      self.class.logger
+    end
+
+  private
+    def setup_rack_handler
+      # First try to set Rack handler via a special hook we honor
+      @rack_handler = if @app.respond_to?(:detect_rack_handler)
+        @app.detect_rack_handler
+
+      # If they aren't using our hook, try to use their @app.server settings
+      elsif @app.respond_to?(:server) and @app.server
+        # If :server isn't set, it returns an array of possibilities,
+        # sorted from most to least preferable.
+        if @app.server.is_a?(Array)
+          handler = nil
+          @app.server.each do |server|
+            begin
+              handler = Rack::Handler.get(server)
+              break
+            rescue LoadError, NameError
+              next
+            end
+          end
+          raise 'No available Rack handler (e.g. WEBrick, Thin, Puma, etc.) was found.' if handler.nil?
+
+          handler
+
+        # :server might be set explicitly to a single option like "mongrel"
+        else
+          Rack::Handler.get(@app.server)
+        end
+
+      # If all else fails, we'll use Thin
+      else
+        JRUBY ? Rack::Handler::WEBrick : Rack::Handler::Thin
+      end
+    end
+
+    def load_options(runtime_args)
+      @args = option_parser.parse!(runtime_args)
+      options.merge!(option_parser.options)
+      args
+    rescue OptionParser::MissingArgument => e
+      logger.warn "#{e}, run -h for options"
+      exit
+    end
+
+    def option_parser
+      @option_parser ||= Parser.new(app_name)
+    end
+
+    class Parser < OptionParser
+      attr_reader :command, :options
+
+      def initialize(app_name)
+        super("", 24, '  ')
+        self.banner = "Usage: #{app_name} [options]"
+
+        @options = {}
+        basename = app_name.gsub(/\W+/, "_")
+        on('-K', "--kill", "kill the running process and exit") { @command = :kill }
+        on('-S', "--status", "display the current running PID and URL then quit") { @command = :status }
+        string_option("-s", "--server SERVER", "serve using SERVER (thin/mongrel/webrick)", :rack_handler)
+        string_option("-o", "--host HOST", "listen on HOST (default: #{HOST})", :host)
+        string_option("-p", "--port PORT", "use PORT (default: #{PORT})", :port)
+        on("-x", "--no-proxy", "ignore env proxy settings (e.g. http_proxy)") { opts[:no_proxy] = true }
+        boolean_option("-F", "--foreground", "don't daemonize, run in the foreground", :foreground)
+        boolean_option("-L", "--no-launch", "don't launch the browser", :skip_launch)
+        boolean_option('-d', "--debug", "raise the log level to :debug (default: :info)", :debug)
+        string_option("--app-dir APP_DIR", "set the app dir where files are stored (default: ~/#{basename}/)", :app_dir)
+        string_option("-P", "--pid-file PID_FILE", "set the path to the pid file (default: app_dir/#{basename}.pid)", :pid_file)
+        string_option("--log-file LOG_FILE", "set the path to the log file (default: app_dir/#{basename}.log)", :log_file)
+        string_option("--url-file URL_FILE", "set the path to the URL file (default: app_dir/#{basename}.url)", :url_file)
+        string_option('-N NAMESPACE', "--namespace NAMESPACE", "set the Redis namespace", :redis_namespace)
+        string_option('-r redis-connection', "--redis redis-connection", "set the Redis connection string", :redis_conf)
+        string_option('-a url-prefix', "--append url-prefix", "set reverse_proxy friendly prefix to links", :url_prefix)
+        separator ""
+        separator "Common options:"
+        on_tail("-h", "--help", "Show this message") { @command = :help }
+        on_tail("--version", "Show version") { @command = :version }
+      end
+
+      def boolean_option(*argv)
+        k = argv.pop; on(*argv) { options[k] = true }
+      end
+
+      def string_option(*argv)
+        k = argv.pop; on(*argv) { |value| options[k] = value }
+      end
+    end
+
+    def kill_commands
+      WINDOWS ? [1] : [:INT, :TERM]
+    end
+
+    def kill_command
+      kill_commands[0]
+    end
+
+    def delete_pid!
+      File.delete(pid_file) if File.exist?(pid_file)
+    end
+  end
+
+end

data/lib/resque/worker.rb

@@ -103,7 +103,7 @@ module Resque
       skip_exists = options[:skip_exists]
 
       if skip_exists || exists?(worker_id)
-        host, pid, queues_raw = worker_id.split(':')
+        host, pid, queues_raw = worker_id.split(':', 3)
         queues = queues_raw.split(',')
         worker = new(*queues)
         worker.hostname = host
@@ -145,6 +145,11 @@ module Resque
       @paused = nil
       @before_first_fork_hook_ran = false
 
+      @heartbeat_thread = nil
+      @heartbeat_thread_signal = nil
+
+      @last_state = :idle
+
       verbose_value = ENV['LOGGING'] || ENV['VERBOSE']
       self.verbose = verbose_value if verbose_value
       self.very_verbose = ENV['VVERBOSE'] if ENV['VVERBOSE']
@@ -162,9 +167,6 @@ module Resque
     # once per worker.
     def prepare
       if ENV['BACKGROUND']
-        unless Process.respond_to?('daemon')
-            abort "env var BACKGROUND is set, which requires ruby >= 1.9"
-        end
         Process.daemon(true)
       end
 
@@ -175,12 +177,20 @@ module Resque
       self.reconnect if ENV['BACKGROUND']
     end
 
+    WILDCARDS = ['*', '?', '{', '}', '[', ']'].freeze
+
     def queues=(queues)
-      queues = queues.empty? ? (ENV["QUEUES"] || ENV['QUEUE']).to_s.split(',') : queues
-      @queues = queues.map { |queue| queue.to_s.strip }
-      unless ['*', '?', '{', '}', '[', ']'].any? {|char| @queues.join.include?(char) }
-        @static_queues = @queues.flatten.uniq
-      end
+      queues = (ENV["QUEUES"] || ENV['QUEUE']).to_s.split(',') if queues.empty?
+      queues = queues.map { |queue| queue.to_s.strip }
+
+      @skip_queues, @queues = queues.partition { |queue| queue.start_with?('!') }
+      @skip_queues.map! { |queue| queue[1..-1] }
+
+      # The behavior of `queues` is dependent on the value of `@has_dynamic_queues: if it's true, the method returns the result of filtering @queues with `glob_match`
+      # if it's false, the method returns @queues directly. Since `glob_match` will cause skipped queues to be filtered out, we want to make sure it's called if we have @skip_queues.any?
+      @has_dynamic_queues =
+        @skip_queues.any? || WILDCARDS.any? { |char| @queues.join.include?(char) }
+
       validate_queues
     end
 
@@ -198,13 +208,18 @@ module Resque
     # A splat ("*") means you want every queue (in alpha order) - this
     # can be useful for dynamically adding new queues.
     def queues
-      return @static_queues if @static_queues
-      @queues.map { |queue| glob_match(queue) }.flatten.uniq
+      if @has_dynamic_queues
+        current_queues = Resque.queues
+        @queues.map { |queue| glob_match(current_queues, queue) }.flatten.uniq
+      else
+        @queues
+      end
     end
 
-    def glob_match(pattern)
-      Resque.queues.select do |queue|
-        File.fnmatch?(pattern, queue)
+    def glob_match(list, pattern)
+      list.select do |queue|
+        File.fnmatch?(pattern, queue) &&
+          @skip_queues.none? { |skip_pattern| File.fnmatch?(skip_pattern, queue) }
       end.sort
     end
 
@@ -232,6 +247,7 @@ module Resque
         break if shutdown?
 
         unless work_one_job(&block)
+          state_change
           break if interval.zero?
           log_with_severity :debug, "Sleeping for #{interval} seconds"
           procline paused? ? "Paused" : "Waiting for #{queues.join(',')}"
@@ -240,10 +256,12 @@ module Resque
       end
 
       unregister_worker
+      run_hook :worker_exit
     rescue Exception => exception
       return if exception.class == SystemExit && !@child && run_at_exit_hooks
       log_with_severity :error, "Failed to start worker : #{exception.inspect}"
       unregister_worker(exception)
+      run_hook :worker_exit
     end
 
     def work_one_job(job = nil, &block)
@@ -482,20 +500,22 @@ module Resque
     # Returns a list of workers that have sent a heartbeat in the past, but which
     # already expired (does NOT include workers that have never sent a heartbeat at all).
     def self.all_workers_with_expired_heartbeats
-      workers = Worker.all
+      # Use `Worker.all_heartbeats` instead of `Worker.all`
+      # to prune workers which haven't been registered but have set a heartbeat.
+      # https://github.com/resque/resque/pull/1751
       heartbeats = Worker.all_heartbeats
       now = data_store.server_time
 
-      workers.select do |worker|
-        id = worker.to_s
-        heartbeat = heartbeats[id]
-
+      heartbeats.select do |id, heartbeat|
         if heartbeat
           seconds_since_heartbeat = (now - Time.parse(heartbeat)).to_i
           seconds_since_heartbeat > Resque.prune_interval
         else
           false
         end
+      end.each_key.map do |id|
+        # skip_exists must be true to include not registered workers
+        find(id, :skip_exists => true)
       end
     end
 
@@ -559,7 +579,7 @@ module Resque
 
     # are we paused?
     def paused?
-      @paused
+      @paused || redis.get('pause-all-workers').to_s.strip.downcase == 'true'
     end
 
     # Stop processing jobs after the current one has completed (if we're
@@ -588,23 +608,27 @@ module Resque
     # By checking the current Redis state against the actual
     # environment, we can determine if Redis is old and clean it up a bit.
     def prune_dead_workers
-      all_workers = Worker.all
+      return unless data_store.acquire_pruning_dead_worker_lock(self, Resque.heartbeat_interval)
 
-      unless all_workers.empty?
-        known_workers = worker_pids
-        all_workers_with_expired_heartbeats = Worker.all_workers_with_expired_heartbeats
-      end
+      all_workers = Worker.all
 
-      all_workers.each do |worker|
+      known_workers = worker_pids
+      all_workers_with_expired_heartbeats = Worker.all_workers_with_expired_heartbeats
+      all_workers_with_expired_heartbeats.each do |worker|
         # If the worker hasn't sent a heartbeat, remove it from the registry.
         #
         # If the worker hasn't ever sent a heartbeat, we won't remove it since
         # the first heartbeat is sent before the worker is registred it means
         # that this is a worker that doesn't support heartbeats, e.g., another
         # client library or an older version of Resque. We won't touch these.
+        log_with_severity :info, "Pruning dead worker: #{worker}"
+
+        job_class = worker.job(false)['payload']['class'] rescue nil
+        worker.unregister_worker(PruneDeadWorkerDirtyExit.new(worker.to_s, job_class))
+      end
+
+      all_workers.each do |worker|
         if all_workers_with_expired_heartbeats.include?(worker)
-          log_with_severity :info, "Pruning dead worker: #{worker}"
-          worker.unregister_worker(PruneDeadWorkerDirtyExit.new(worker.to_s))
           next
         end
 
@@ -635,7 +659,8 @@ module Resque
 
     # Runs a named hook, passing along any arguments.
     def run_hook(name, *args)
-      return unless hooks = Resque.send(name)
+      hooks = Resque.send(name)
+      return if hooks.empty?
       return if name == :before_first_fork && @before_first_fork_hook_ran
       msg = "Running #{name} hooks"
       msg << " with #{args.inspect}" if args.any?
@@ -672,9 +697,9 @@ module Resque
 
       kill_background_threads
 
-      data_store.unregister_worker(self) do
-        Stat.clear("processed:#{self}")
-        Stat.clear("failed:#{self}")
+      data_store.unregister_worker(self) do |**opts|
+        Stat.clear("processed:#{self}", **opts)
+        Stat.clear("failed:#{self}",    **opts)
       end
     rescue Exception => exception_while_unregistering
       message = exception_while_unregistering.message
@@ -695,13 +720,22 @@ module Resque
         :run_at  => Time.now.utc.iso8601,
         :payload => job.payload
       data_store.set_worker_payload(self,data)
+      state_change
     end
 
     # Called when we are done working - clears our `working_on` state
     # and tells Redis we processed a job.
     def done_working
-      data_store.worker_done_working(self) do
-        processed!
+      data_store.worker_done_working(self) do |**opts|
+        processed!(**opts)
+      end
+    end
+
+    def state_change
+      current_state = state
+      if current_state != @last_state
+        run_hook :queue_empty if current_state == :idle
+        @last_state = current_state
       end
     end
 
@@ -711,9 +745,9 @@ module Resque
     end
 
     # Tell Redis we've processed a job.
-    def processed!
-      Stat << "processed"
-      Stat << "processed:#{self}"
+    def processed!(**opts)
+      Stat.incr("processed",         1, **opts)
+      Stat.incr("processed:#{self}", 1, **opts)
     end
 
     # How many failed jobs has this worker seen? Returns an int.
@@ -808,7 +842,7 @@ module Resque
     # machine. Useful when pruning dead workers on startup.
     def windows_worker_pids
       tasklist_output = `tasklist /FI "IMAGENAME eq ruby.exe" /FO list`.encode("UTF-8", Encoding.locale_charmap)
-      tasklist_output.split($/).select { |line| line =~ /^PID:/}.collect{ |line| line.gsub /PID:\s+/, '' }
+      tasklist_output.split($/).select { |line| line =~ /^PID:/ }.collect { |line| line.gsub(/PID:\s+/, '') }
     end
 
     # Find Resque worker pids on Linux and OS X.
@@ -827,7 +861,7 @@ module Resque
       `ps -A -o pid,comm | grep "[r]uby" | grep -v "resque-web"`.split("\n").map do |line|
         real_pid = line.split(' ')[0]
         pargs_command = `pargs -a #{real_pid} 2>/dev/null | grep [r]esque | grep -v "resque-web"`
-        if pargs_command.split(':')[1] == " resque-#{Resque::Version}"
+        if pargs_command.split(':')[1] == " resque-#{Resque::VERSION}"
           real_pid
         end
       end.compact
@@ -837,7 +871,7 @@ module Resque
     # Procline is always in the format of:
     #   RESQUE_PROCLINE_PREFIXresque-VERSION: STRING
     def procline(string)
-      $0 = "#{ENV['RESQUE_PROCLINE_PREFIX']}resque-#{Resque::Version}: #{string}"
+      $0 = "#{ENV['RESQUE_PROCLINE_PREFIX']}resque-#{Resque::VERSION}: #{string}"
       log_with_severity :debug, $0
     end
 
@@ -850,13 +884,7 @@ module Resque
     end
 
 
-    def verbose
-      @verbose
-    end
-
-    def very_verbose
-      @very_verbose
-    end
+    attr_reader :verbose, :very_verbose
 
     def verbose=(value);
       if value && !very_verbose
@@ -909,7 +937,7 @@ module Resque
         nil
       end
 
-      job.fail(DirtyExit.new("Child process received unhandled signal #{$?.stopsig}", $?)) if $?.signaled?
+      job.fail(DirtyExit.new("Child process received unhandled signal #{$?}", $?)) if $?.signaled?
       @child = nil
     end