qless 0.9.3 → 0.10.0

data/lib/qless/server/views/queue.erb

@@ -40,12 +40,13 @@
 
 <div class="subnav subnav-fixed">
   <ul class="nav nav-pills">
-    <li class="<%= tab == 'stats'     ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/stats" %>">Stats</a></li>
-    <li class="<%= tab == 'running'   ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/running" %>">Running</a></li>
-    <li class="<%= tab == 'waiting'   ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/waiting" %>">Waiting</a></li>
-    <li class="<%= tab == 'scheduled' ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/scheduled" %>">Scheduled</a></li>
-    <li class="<%= tab == 'stalled'   ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/stalled" %>">Stalled</a></li>
-    <li class="<%= tab == 'depends'   ? 'active' : '' %>"><a href="<%= u "/queues/#{queue['name']}/depends" %>">Depends</a></li>
+    <li class="<%= tab == 'stats'     ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/stats" %>">Stats</a></li>
+    <li class="<%= tab == 'running'   ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/running" %>">Running</a></li>
+    <li class="<%= tab == 'waiting'   ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/waiting" %>">Waiting</a></li>
+    <li class="<%= tab == 'scheduled' ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/scheduled" %>">Scheduled</a></li>
+    <li class="<%= tab == 'stalled'   ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/stalled" %>">Stalled</a></li>
+    <li class="<%= tab == 'depends'   ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/depends" %>">Depends</a></li>
+    <li class="<%= tab == 'recurring' ? 'active' : '' %>"><a href="<%= u "/queues/#{CGI::escape(queue['name'])}/recurring" %>">Recurring</a></li>
   </ul>
 </div>
 
@@ -53,7 +54,7 @@
 
 <div class="row">
   <div class="span8">
-    <h2><a href="<%= u "/queues/#{queue['name']}" %>"><%= queue['name'] %></a> |
+    <h2><a href="<%= u "/queues/#{CGI::escape(queue['name'])}" %>"><%= queue['name'] %></a> |
       <%= queue['running']   %> /
       <%= queue['waiting']   %> /
       <%= queue['scheduled'] %> /
@@ -73,7 +74,7 @@
   </div>
 </div>
 
-<% if ['running', 'waiting', 'scheduled', 'stalled', 'depends'].include?(tab) %>
+<% if ['running', 'waiting', 'scheduled', 'stalled', 'depends', 'recurring'].include?(tab) %>
   <hr/>
   <%= erb :_job_list, :locals => { :jobs => jobs, :queues => queues } %>
 <% else %>

data/lib/qless/server/views/queues.erb

@@ -10,7 +10,24 @@
   <% queues.each do |queue| %>
   <div class="row">
     <div class="span4">
-      <h3><a href="<%= u "/queues/#{queue['name']}" %>"><%= queue['name'] %></a></h3>
+      <h3>
+        <% if queue['paused'] %>
+          <button
+            id="<%= queue['name'] %>-pause"
+            title="Unpause"
+            class="btn btn-success"
+            onclick="unpause('<%= queue['name'] %>')"><i class="icon-play"></i>
+          </button>
+        <% else %>
+          <button
+            id="<%= queue['name'] %>-pause"
+            title="Pause"
+            class="btn btn-warning"
+            onclick="pause('<%= queue['name'] %>')"><i class="icon-pause"></i>
+          </button>
+        <% end %>
+        <a href="<%= u "/queues/#{CGI::escape(queue['name'])}" %>"><%= queue['name'] %></a>
+      </h3>
     </div>
     <div class="span8">
       <h3> |

data/lib/qless/subscriber.rb

@@ -1,48 +1,63 @@
+# Encoding: utf-8
+
 require 'thread'
-require 'qless/wait_until'
 
 module Qless
+  # A class used for subscribing to messages in a thread
   class Subscriber
     def self.start(*args, &block)
-      new(*args, &block).start_pub_sub_listener
+      new(*args, &block).tap(&:start)
     end
 
-    attr_reader :client, :channel
+    attr_reader :channel, :redis
 
-    def initialize(client, channel, &message_received_callback)
-      @client = client
+    def initialize(client, channel, options = {}, &message_received_callback)
       @channel = channel
       @message_received_callback = message_received_callback
+      @log = options.fetch(:log) { ::Logger.new($stderr) }
 
-      # pub/sub blocks the connection so we must use a different redis connection
-      @client_redis = client.redis
+      # pub/sub blocks the connection so we must use a different redis
+      # connection
+      @client_redis   = client.redis
       @listener_redis = client.new_redis_connection
 
       @my_channel = Qless.generate_jid
     end
 
-    def start_pub_sub_listener
-      @thread = ::Thread.start do
-        @listener_redis.subscribe(channel, @my_channel) do |on|
-          on.message do |_channel, message|
-            if _channel == @my_channel
-              @listener_redis.unsubscribe(@my_channel)
-            else
-              @message_received_callback.call(self, JSON.parse(message))
-            end
+    # Start a thread listening
+    def start
+      queue = ::Queue.new
+
+      @thread = Thread.start do
+        @listener_redis.subscribe(@channel, @my_channel) do |on|
+          on.subscribe do |channel|
+            queue.push(:subscribed) if channel == @channel
+          end
+
+          on.message do |channel, message|
+            handle_message(channel, message)
           end
         end
       end
 
-      wait_until_thread_listening
+      queue.pop
     end
 
-    def wait_until_thread_listening
-      Qless::WaitUntil.wait_until(10) do
-        @client_redis.publish(@my_channel, 'disconnect') == 1
+    def stop
+      @client_redis.publish(@my_channel, 'disconnect')
+      @thread.join
+    end
+
+  private
+
+    def handle_message(channel, message)
+      if channel == @my_channel
+        @listener_redis.unsubscribe(@channel, @my_channel) if message == "disconnect"
+      else
+        @message_received_callback.call(self, JSON.parse(message))
       end
+    rescue Exception => error
+      @log.error("Qless::Subscriber") { error }
     end
   end
 end
-
-

data/lib/qless/tasks.rb

@@ -1,10 +1,5 @@
-namespace :qless do
-  task :setup # no-op; users should define their own setup
-
-  desc "Start a Qless worker using env vars: QUEUES, JOB_RESERVER, REDIS_URL, INTERVAL, VERBOSE, VVERBOSE"
-  task :work => :setup do
-    require 'qless/worker'
-    Qless::Worker.start
-  end
-end
-
+# Encoding: utf-8
+warn "Qless tasks are deprecated (they haven't worked for " \
+     "quite some time) and you should start a worker by " \
+     "writing a bit of ruby code that instantiates and " \
+     "runs a worker instead."

data/lib/qless/test_helpers/worker_helpers.rb

@@ -0,0 +1,55 @@
+module Qless
+  module WorkerHelpers
+    # Yield with a worker running, and then clean the worker up afterwards
+    def run_worker_concurrently_with(worker, &block)
+      thread = Thread.start { stop_worker_after(worker, &block) }
+      thread.abort_on_exception = true
+      worker.run
+    ensure
+      thread.join(0.1)
+    end
+
+    def stop_worker_after(worker, &block)
+      yield
+    ensure
+      worker.stop!
+    end
+
+    # Run only the given number of jobs, then stop
+    def run_jobs(worker, count)
+      worker.extend Module.new {
+        define_method(:jobs) do
+          base_enum = super()
+          Enumerator.new do |enum|
+            count.times { enum << base_enum.next }
+          end
+        end
+      }
+
+      thread = Thread.start { yield } if block_given?
+      thread.abort_on_exception if thread
+      worker.run
+    ensure
+      thread.join(0.1) if thread
+    end
+
+    # Runs the worker until it has no more jobs to process,
+    # effectively drainig its queues.
+    def drain_worker_queues(worker)
+      worker.extend Module.new {
+        # For the child: stop as soon as it can't pop more jobs.
+        def no_job_available
+          shutdown
+        end
+
+        # For the parent: when the child stops,
+        # don't try to restart it; shutdown instead.
+        def spawn_replacement_child(*)
+          shutdown
+        end
+      }
+
+      worker.run
+    end
+  end
+end

data/lib/qless/version.rb

@@ -1,3 +1,5 @@
+# Encoding: utf-8
+
 module Qless
-  VERSION = "0.9.3"
+  VERSION = '0.10.0'
 end

data/lib/qless/worker.rb

@@ -1,414 +1,5 @@
-require 'qless'
-require 'time'
-require 'qless/job_reservers/ordered'
-require 'qless/job_reservers/round_robin'
-require 'qless/job_reservers/shuffled_round_robin'
-require 'qless/subscriber'
-require 'qless/wait_until'
-
-module Qless
-  # This is heavily inspired by Resque's excellent worker:
-  # https://github.com/defunkt/resque/blob/v1.20.0/lib/resque/worker.rb
-  class Worker
-    def initialize(job_reserver, options = {})
-      self.job_reserver = job_reserver
-      @shutdown = @paused = false
-
-      self.very_verbose = options[:very_verbose]
-      self.verbose = options[:verbose]
-      self.run_as_single_process = options[:run_as_single_process]
-      self.output = options.fetch(:output, $stdout)
-      self.term_timeout = options.fetch(:term_timeout, 4.0)
-      @backtrace_replacements = { Dir.pwd => '.' }
-      @backtrace_replacements[ENV['GEM_HOME']] = '<GEM_HOME>' if ENV.has_key?('GEM_HOME')
-
-      output.puts "\n\n\n" if verbose || very_verbose
-      log "Instantiated Worker"
-    end
-
-    # Whether the worker should log basic info to STDOUT
-    attr_accessor :verbose
-
-    # Whether the worker should log lots of info to STDOUT
-    attr_accessor  :very_verbose
-
-    # Whether the worker should run in a single prcoess
-    # i.e. not fork a child process to do the work
-    # This should only be true in a dev/test environment
-    attr_accessor :run_as_single_process
-
-    # An IO-like object that logging output is sent to.
-    # Defaults to $stdout.
-    attr_accessor :output
-
-    # The object responsible for reserving jobs from the Qless server,
-    # using some reasonable strategy (e.g. round robin or ordered)
-    attr_accessor :job_reserver
-
-    # How long the child process is given to exit before forcibly killing it.
-    attr_accessor :term_timeout
-
-    # Starts a worker based on ENV vars. Supported ENV vars:
-    #   - REDIS_URL=redis://host:port/db-num (the redis gem uses this automatically)
-    #   - QUEUES=high,medium,low or QUEUE=blah
-    #   - JOB_RESERVER=Ordered or JOB_RESERVER=RoundRobin
-    #   - INTERVAL=3.2
-    #   - VERBOSE=true (to enable logging)
-    #   - VVERBOSE=true (to enable very verbose logging)
-    #   - RUN_AS_SINGLE_PROCESS=true (false will fork children to do work, true will keep it single process)
-    # This is designed to be called from a rake task
-    def self.start
-      client = Qless::Client.new
-      queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',').map { |q| client.queues[q.strip] }
-      if queues.none?
-        raise "No queues provided. You must pass QUEUE or QUEUES when starting a worker."
-      end
-
-      reserver = JobReservers.const_get(ENV.fetch('JOB_RESERVER', 'Ordered')).new(queues)
-      interval = Float(ENV.fetch('INTERVAL', 5.0))
-
-      options = {}
-      options[:verbose] = !!ENV['VERBOSE']
-      options[:very_verbose] = !!ENV['VVERBOSE']
-      options[:run_as_single_process] = !!ENV['RUN_AS_SINGLE_PROCESS']
-
-      new(reserver, options).work(interval)
-    end
-
-    def work(interval = 5.0)
-      procline "Starting #{@job_reserver.description}"
-      register_parent_signal_handlers
-      uniq_clients.each { |client| start_parent_pub_sub_listener_for(client) }
-
-      loop do
-        break if shutdown?
-        if paused?
-          sleep interval
-          next
-        end
-
-        unless job = reserve_job
-          break if interval.zero?
-          procline "Waiting for #{@job_reserver.description}"
-          log! "Sleeping for #{interval} seconds"
-          sleep interval
-          next
-        end
-
-        perform_job_in_child_process(job)
-      end
-    ensure
-      # make sure the worker deregisters on shutdown
-      deregister
-    end
-
-    def perform(job)
-      around_perform(job)
-    rescue Exception => error
-      fail_job(job, error, caller)
-    else
-      try_complete(job)
-    end
-
-    def reserve_job
-      @job_reserver.reserve
-    rescue Exception => error
-      # We want workers to durably stay up, so we don't want errors
-      # during job reserving (e.g. network timeouts, etc) to kill
-      # the worker.
-      log "Got an error while reserving a job: #{error.class}: #{error.message}"
-    end
-
-    def perform_job_in_child_process(job)
-      with_job(job) do
-        @child = fork do
-          job.reconnect_to_redis
-          register_child_signal_handlers
-          start_child_pub_sub_listener_for(job.client)
-          procline "Processing #{job.description}"
-          perform(job)
-          exit! # don't run at_exit hooks
-        end
-
-        if @child
-          wait_for_child
-        else
-          procline "Single processing #{job.description}"
-          perform(job)
-        end
-      end
-    end
-
-    def shutdown
-      @shutdown = true
-    end
-
-    def shutdown!
-      shutdown
-      kill_child unless run_as_single_process
-    end
-
-    def shutdown?
-      @shutdown
-    end
-
-    def paused?
-      @paused
-    end
-
-    def pause_processing
-      log "USR2 received; pausing job processing"
-      @paused = true
-      procline "Paused -- #{@job_reserver.description}"
-    end
-
-    def unpause_processing
-      log "CONT received; resuming job processing"
-      @paused = false
-    end
-
-  private
-
-    def fork
-      super unless run_as_single_process
-    end
-
-    def deregister
-      uniq_clients.each do |client|
-        client.deregister_workers(Qless.worker_name)
-      end
-    end
-
-    def uniq_clients
-      @uniq_clients ||= @job_reserver.queues.map(&:client).uniq
-    end
-
-    def try_complete(job)
-      job.complete unless job.state_changed?
-    rescue Job::CantCompleteError => e
-      # There's not much we can do here. Complete fails in a few cases:
-      #   - The job is already failed (i.e. by another worker)
-      #   - The job is being worked on by another worker
-      #   - The job has been cancelled
-      #
-      # We don't want to (or are able to) fail the job with this error in
-      # any of these cases, so the best we can do is log the failure.
-      log "Failed to complete #{job.inspect}: #{e.message}"
-    end
-
-    # Allow middleware modules to be mixed in and override the
-    # definition of around_perform while providing a default
-    # implementation so our code can assume the method is present.
-    include Module.new {
-      def around_perform(job)
-        job.perform
-      end
-    }
-
-    def fail_job(job, error, worker_backtrace)
-      group = "#{job.klass_name}:#{error.class}"
-      message = "#{truncated_message(error)}\n\n#{format_failure_backtrace(error.backtrace, worker_backtrace)}"
-      log "Got #{group} failure from #{job.inspect}"
-      job.fail(group, message)
-    end
-
-    # TODO: pull this out into a config option.
-    MAX_ERROR_MESSAGE_SIZE = 10_000
-    def truncated_message(error)
-      return error.message if error.message.length <= MAX_ERROR_MESSAGE_SIZE
-      error.message.slice(0, MAX_ERROR_MESSAGE_SIZE) + "... (truncated due to length)"
-    end
-
-    def format_failure_backtrace(error_backtrace, worker_backtrace)
-      (error_backtrace - worker_backtrace).map do |line|
-        @backtrace_replacements.inject(line) do |line, (original, new)|
-          line.sub(original, new)
-        end
-      end.join("\n")
-    end
-
-    def procline(value)
-      $0 = "Qless-#{Qless::VERSION}: #{value} at #{Time.now.iso8601}"
-      log! $0
-    end
-
-    def wait_for_child
-      srand # Reseeding
-      procline "Forked #{@child} at #{Time.now.to_i}"
-      begin
-        Process.waitpid(@child)
-      rescue SystemCallError
-        nil
-      end
-    end
-
-    # Kills the forked child immediately with minimal remorse. The job it
-    # is processing will not be completed. Send the child a TERM signal,
-    # wait 5 seconds, and then a KILL signal if it has not quit
-    def kill_child
-      return unless @child
-
-      if Process.waitpid(@child, Process::WNOHANG)
-        log "Child #{@child} already quit."
-        return
-      end
-
-      signal_child("TERM", @child)
-
-      signal_child("KILL", @child) unless quit_gracefully?(@child)
-    rescue SystemCallError
-      log "Child #{@child} already quit and reaped."
-    end
-
-    # send a signal to a child, have it logged.
-    def signal_child(signal, child)
-      log "Sending #{signal} signal to child #{child}"
-      Process.kill(signal, child)
-    end
-
-    # has our child quit gracefully within the timeout limit?
-    def quit_gracefully?(child)
-      (term_timeout.to_f * 10).round.times do |i|
-        sleep(0.1)
-        return true if Process.waitpid(child, Process::WNOHANG)
-      end
-
-      false
-    end
-
-    # This was originally stolen directly from resque... (thanks, @defunkt!)
-    # Registers the various signal handlers a worker responds to.
-    #
-    # TERM: Shutdown immediately, stop processing jobs.
-    #  INT: Shutdown immediately, stop processing jobs.
-    # QUIT: Shutdown after the current job has finished processing.
-    # USR1: Kill the forked child immediately, continue processing jobs.
-    # USR2: Don't process any new jobs; dump the backtrace.
-    # CONT: Start processing jobs again after a USR2
-    def register_parent_signal_handlers
-      trap('TERM') { shutdown!  }
-      trap('INT')  { shutdown!  }
-
-      begin
-        trap('QUIT') { shutdown   }
-        trap('USR1') { kill_child }
-        trap('USR2') do
-          log "Current backtrace (parent): \n\n#{caller.join("\n")}\n\n"
-          pause_processing
-        end
-
-        trap('CONT') { unpause_processing }
-      rescue ArgumentError
-        warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
-      end
-    end
-
-    def register_child_signal_handlers
-      trap('TERM') { raise SignalException.new("SIGTERM") }
-      trap('INT', 'DEFAULT')
-
-      begin
-        trap('QUIT', 'DEFAULT')
-        trap('USR1', 'DEFAULT')
-        trap('USR2', 'DEFAULT')
-
-        trap('USR2') do
-          log "Current backtrace (child): \n\n#{caller.join("\n")}\n\n"
-        end
-      rescue ArgumentError
-      end
-    end
-
-    # Log a message to STDOUT if we are verbose or very_verbose.
-    def log(message)
-      if verbose
-        output.puts "*** #{message}"
-      elsif very_verbose
-        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
-        output.puts "** [#{time}] #$$: #{message}"
-      end
-    end
-
-    # Logs a very verbose message to STDOUT.
-    def log!(message)
-      log message if very_verbose
-    end
-
-    def start_parent_pub_sub_listener_for(client)
-      Subscriber.start(client, "ql:w:#{Qless.worker_name}") do |subscriber, message|
-        if message["event"] == "lock_lost" && message["jid"] == current_job_jid
-          fail_job_due_to_timeout
-          kill_child
-        end
-      end
-    end
-
-    def start_child_pub_sub_listener_for(client)
-      Subscriber.start(client, "ql:w:#{Qless.worker_name}:#{Process.pid}") do |subscriber, message|
-        if message["event"] == "notify_backtrace"
-          notify_parent_of_job_backtrace(client, message.fetch('notify_list'))
-        end
-      end
-    end
-
-    def with_job(job)
-      @job = job
-      yield
-    ensure
-      @job = nil
-    end
-
-    # To prevent race conditions (with our listener thread),
-    # we cannot use a pattern like `use(@job) if @job` because
-    # the value of `@job` could change between the checking of
-    # it and the use of it. Here we use a pattern that avoids
-    # the issue -- get the job into a local, and yield that if
-    # it is set.
-    def access_current_job
-      if job = @job
-        yield job
-      end
-    end
-
-    def current_job_jid
-      access_current_job &:jid
-    end
-
-    JobLockLost = Class.new(StandardError)
-
-    def fail_job_due_to_timeout
-      access_current_job do |job|
-        error = JobLockLost.new
-        error.set_backtrace(get_backtrace_from_child(job.client.redis))
-        fail_job(job, error, caller)
-      end
-    end
-
-    def notify_parent_of_job_backtrace(client, list)
-      job_backtrace = Thread.main.backtrace
-      client.redis.lpush list, JSON.dump(job_backtrace)
-      client.redis.pexpire list, BACKTRACE_EXPIRATION_TIMEOUT_MS
-    end
-
-    WAIT_FOR_CHILD_BACKTRACE_TIMEOUT = 4
-    BACKTRACE_EXPIRATION_TIMEOUT_MS = 60_000 # timeout after a minute
-
-    def get_backtrace_from_child(child_redis)
-      notification_list = "ql:child_backtraces:#{Qless.generate_jid}"
-      request_backtrace = { "event"       => "notify_backtrace",
-                            "notify_list" => notification_list }
-
-      if child_redis.publish("ql:w:#{Qless.worker_name}:#{@child}", JSON.dump(request_backtrace)).zero?
-        return ["Could not obtain child backtrace since it was not listening."]
-      end
-
-      begin
-        _, backtrace_json = child_redis.blpop(notification_list, WAIT_FOR_CHILD_BACKTRACE_TIMEOUT)
-        JSON.parse(backtrace_json)
-      rescue => e
-        ["Could not obtain child backtrace: #{e.class}: #{e.message}"] + e.backtrace
-      end
-    end
-  end
-end
+# Encoding: utf-8
 
+require 'qless/worker/base'
+require 'qless/worker/serial'
+require 'qless/worker/forking'