qless 0.9.2 → 0.9.3

data/lib/qless/server/static/css/docs.css

@@ -12,7 +12,6 @@ body {
   position: relative;
   padding-top: 90px;
   background-color: #fff;
-  background-image: url(../img/grid-18px-masked.png);
   background-repeat: repeat-x;
   background-position: 0 40px;
 }
@@ -817,3 +816,24 @@ form.well {
   }
 
 }
+
+/* For proper failed job display*/
+.l-sidebyside {
+  overflow:auto;
+}
+
+.l-sidebyside > * {
+  display:inline-block;
+  *display: inline;
+  *zoom: 1;
+}
+
+.failed-job > .min-col-size {
+  min-width: 395px;
+}
+
+.failed-job .row {
+  margin-left:0;
+  margin-right:10px;
+  height: 30px;
+}

data/lib/qless/server/views/_job.erb

@@ -112,12 +112,12 @@
 	      <div class="span6">
 	        <h3><small>History</small></h3>
 	        <div style="overflow-y:scroll; height: 200px">
-	          <% job.history.reverse.each do |h| %>
+	          <% job.queue_history.reverse.each do |h| %>
 	          <pre><strong><%= h['q'] %></strong>
-	    Put: <%= strftime(Time.at(h['put'])) %><% if not h['popped'].nil? %>
-	    Pop: <%= strftime(Time.at(h['popped'])) %> by <%= h['worker'] %><% end %><% if not h['completed'].nil? %>
-	    Completed: <%= strftime(Time.at(h['completed'])) %><% end %><% if not h['failed'].nil? %>
-	    Failed: <%= strftime(Time.at(h['failed'])) %><% end %></pre>
+	    Put: <%= strftime(h['put']) %><% if not h['popped'].nil? %>
+	    Pop: <%= strftime(h['popped']) %> by <%= h['worker'] %><% end %><% if not h['completed'].nil? %>
+	    Completed: <%= strftime(h['completed']) %><% end %><% if not h['failed'].nil? %>
+	    Failed: <%= strftime(h['failed']) %><% end %></pre>
 	          <% end %>
 	        </div>
 	      </div>

data/lib/qless/server/views/overview.erb

@@ -18,7 +18,7 @@
         <%= queue['waiting']   %> /
         <%= queue['scheduled'] %> /
         <%= queue['stalled']   %> /
-        <%= queue['depends']   %> / 
+        <%= queue['depends']   %> /
         <%= queue['recurring'] %> <small>(running / waiting / scheduled / stalled / depends / recurring)</small>
       </h3>
     </div>
@@ -34,16 +34,22 @@
   <div class="page-header">
     <h1>Failed Jobs <small>D'oh!</small></h1>
   </div>
-  <% failed.sort_by { |t, count| -count }.each do |t, count| %>
-  <div class="row">
-    <div class="span4">
-      <h3 style="text-overflow: ellipsis; white-space: nowrap; overflow: hidden"><a href="<%= u "/failed/#{t}" %>"><%= t %></a></h3>
+  <div class="l-sidebyside failed-job">
+    <div class="min-col-size">
+      <% failed.sort_by { |t, count| -count }.each do |t, count| %>
+      <div class="row">
+        <h3><a href="<%= u "/failed/#{t}" %>"><%= t %></a></h3>
+      </div>
+      <% end %>
     </div>
-    <div class="span8">
-      <h3>| <%= count %> <small>Jobs</small></h3>
+    <div>
+      <% failed.sort_by { |t, count| -count }.each do |t, count| %>
+      <div class="row">
+        <h3>| <%= count %> <small>Jobs</small></h3>
+      </div>
+      <% end %>
     </div>
   </div>
-  <% end %>
 <% end %>
 
 <% if tracked['jobs'].empty? %>
@@ -67,7 +73,6 @@
   <% end %>
 <% end %>
 
-
 <% if workers.empty? %>
   <div class="page-header">
     <h1>No Workers <small>Nobody's doin' nothin'!</small></h1>

data/lib/qless/subscriber.rb

@@ -0,0 +1,48 @@
+require 'thread'
+require 'qless/wait_until'
+
+module Qless
+  class Subscriber
+    def self.start(*args, &block)
+      new(*args, &block).start_pub_sub_listener
+    end
+
+    attr_reader :client, :channel
+
+    def initialize(client, channel, &message_received_callback)
+      @client = client
+      @channel = channel
+      @message_received_callback = message_received_callback
+
+      # pub/sub blocks the connection so we must use a different redis connection
+      @client_redis = client.redis
+      @listener_redis = client.new_redis_connection
+
+      @my_channel = Qless.generate_jid
+    end
+
+    def start_pub_sub_listener
+      @thread = ::Thread.start do
+        @listener_redis.subscribe(channel, @my_channel) do |on|
+          on.message do |_channel, message|
+            if _channel == @my_channel
+              @listener_redis.unsubscribe(@my_channel)
+            else
+              @message_received_callback.call(self, JSON.parse(message))
+            end
+          end
+        end
+      end
+
+      wait_until_thread_listening
+    end
+
+    def wait_until_thread_listening
+      Qless::WaitUntil.wait_until(10) do
+        @client_redis.publish(@my_channel, 'disconnect') == 1
+      end
+    end
+  end
+end
+
+

data/lib/qless/version.rb

@@ -1,3 +1,3 @@
 module Qless
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

data/lib/qless/wait_until.rb

@@ -0,0 +1,19 @@
+module Qless
+  module WaitUntil
+    TimeoutError = Class.new(StandardError)
+
+    def wait_until(timeout)
+      timeout_at = Time.now + timeout
+
+      loop do
+        return if yield
+        sleep 0.002
+        if Time.now > timeout_at
+          raise TimeoutError, "Timed out after #{timeout} seconds"
+        end
+      end
+    end
+
+    module_function :wait_until
+  end
+end

data/lib/qless/worker.rb

@@ -2,19 +2,25 @@ require 'qless'
 require 'time'
 require 'qless/job_reservers/ordered'
 require 'qless/job_reservers/round_robin'
+require 'qless/job_reservers/shuffled_round_robin'
+require 'qless/subscriber'
+require 'qless/wait_until'
 
 module Qless
   # This is heavily inspired by Resque's excellent worker:
   # https://github.com/defunkt/resque/blob/v1.20.0/lib/resque/worker.rb
   class Worker
-    def initialize(client, job_reserver, options = {})
-      @client, @job_reserver = client, job_reserver
+    def initialize(job_reserver, options = {})
+      self.job_reserver = job_reserver
       @shutdown = @paused = false
 
       self.very_verbose = options[:very_verbose]
       self.verbose = options[:verbose]
       self.run_as_single_process = options[:run_as_single_process]
       self.output = options.fetch(:output, $stdout)
+      self.term_timeout = options.fetch(:term_timeout, 4.0)
+      @backtrace_replacements = { Dir.pwd => '.' }
+      @backtrace_replacements[ENV['GEM_HOME']] = '<GEM_HOME>' if ENV.has_key?('GEM_HOME')
 
       output.puts "\n\n\n" if verbose || very_verbose
       log "Instantiated Worker"
@@ -35,6 +41,13 @@ module Qless
     # Defaults to $stdout.
     attr_accessor :output
 
+    # The object responsible for reserving jobs from the Qless server,
+    # using some reasonable strategy (e.g. round robin or ordered)
+    attr_accessor :job_reserver
+
+    # How long the child process is given to exit before forcibly killing it.
+    attr_accessor :term_timeout
+
     # Starts a worker based on ENV vars. Supported ENV vars:
     #   - REDIS_URL=redis://host:port/db-num (the redis gem uses this automatically)
     #   - QUEUES=high,medium,low or QUEUE=blah
@@ -59,18 +72,22 @@ module Qless
       options[:very_verbose] = !!ENV['VVERBOSE']
       options[:run_as_single_process] = !!ENV['RUN_AS_SINGLE_PROCESS']
 
-      new(client, reserver, options).work(interval)
+      new(reserver, options).work(interval)
     end
 
     def work(interval = 5.0)
       procline "Starting #{@job_reserver.description}"
-      register_signal_handlers
+      register_parent_signal_handlers
+      uniq_clients.each { |client| start_parent_pub_sub_listener_for(client) }
 
       loop do
         break if shutdown?
-        next  if paused?
+        if paused?
+          sleep interval
+          next
+        end
 
-        unless job = @job_reserver.reserve
+        unless job = reserve_job
           break if interval.zero?
           procline "Waiting for #{@job_reserver.description}"
           log! "Sleeping for #{interval} seconds"
@@ -78,31 +95,48 @@ module Qless
           next
         end
 
-        log "got: #{job.inspect}"
-
-        if run_as_single_process
-          # We're staying in the same process
-          procline "Single processing #{job.description}"
-          perform(job)
-        elsif @child = fork
-          # We're in the parent process
-          procline "Forked #{@child} for #{job.description}"
-          Process.wait(@child)
-        else
-          # We're in the child process
-          procline "Processing #{job.description}"
-          perform(job)
-          exit!
-        end
+        perform_job_in_child_process(job)
       end
+    ensure
+      # make sure the worker deregisters on shutdown
+      deregister
     end
 
     def perform(job)
       around_perform(job)
     rescue Exception => error
-      fail_job(job, error)
+      fail_job(job, error, caller)
     else
-      job.complete unless job.state_changed?
+      try_complete(job)
+    end
+
+    def reserve_job
+      @job_reserver.reserve
+    rescue Exception => error
+      # We want workers to durably stay up, so we don't want errors
+      # during job reserving (e.g. network timeouts, etc) to kill
+      # the worker.
+      log "Got an error while reserving a job: #{error.class}: #{error.message}"
+    end
+
+    def perform_job_in_child_process(job)
+      with_job(job) do
+        @child = fork do
+          job.reconnect_to_redis
+          register_child_signal_handlers
+          start_child_pub_sub_listener_for(job.client)
+          procline "Processing #{job.description}"
+          perform(job)
+          exit! # don't run at_exit hooks
+        end
+
+        if @child
+          wait_for_child
+        else
+          procline "Single processing #{job.description}"
+          perform(job)
+        end
+      end
     end
 
     def shutdown
@@ -135,6 +169,33 @@ module Qless
 
   private
 
+    def fork
+      super unless run_as_single_process
+    end
+
+    def deregister
+      uniq_clients.each do |client|
+        client.deregister_workers(Qless.worker_name)
+      end
+    end
+
+    def uniq_clients
+      @uniq_clients ||= @job_reserver.queues.map(&:client).uniq
+    end
+
+    def try_complete(job)
+      job.complete unless job.state_changed?
+    rescue Job::CantCompleteError => e
+      # There's not much we can do here. Complete fails in a few cases:
+      #   - The job is already failed (i.e. by another worker)
+      #   - The job is being worked on by another worker
+      #   - The job has been cancelled
+      #
+      # We don't want to (or are able to) fail the job with this error in
+      # any of these cases, so the best we can do is log the failure.
+      log "Failed to complete #{job.inspect}: #{e.message}"
+    end
+
     # Allow middleware modules to be mixed in and override the
     # definition of around_perform while providing a default
     # implementation so our code can assume the method is present.
@@ -144,47 +205,120 @@ module Qless
       end
     }
 
-    def fail_job(job, error)
-      group = "#{job.klass}:#{error.class}"
-      message = "#{error.message}\n\n#{error.backtrace.join("\n")}"
+    def fail_job(job, error, worker_backtrace)
+      group = "#{job.klass_name}:#{error.class}"
+      message = "#{truncated_message(error)}\n\n#{format_failure_backtrace(error.backtrace, worker_backtrace)}"
       log "Got #{group} failure from #{job.inspect}"
       job.fail(group, message)
     end
 
+    # TODO: pull this out into a config option.
+    MAX_ERROR_MESSAGE_SIZE = 10_000
+    def truncated_message(error)
+      return error.message if error.message.length <= MAX_ERROR_MESSAGE_SIZE
+      error.message.slice(0, MAX_ERROR_MESSAGE_SIZE) + "... (truncated due to length)"
+    end
+
+    def format_failure_backtrace(error_backtrace, worker_backtrace)
+      (error_backtrace - worker_backtrace).map do |line|
+        @backtrace_replacements.inject(line) do |line, (original, new)|
+          line.sub(original, new)
+        end
+      end.join("\n")
+    end
+
     def procline(value)
       $0 = "Qless-#{Qless::VERSION}: #{value} at #{Time.now.iso8601}"
       log! $0
     end
 
+    def wait_for_child
+      srand # Reseeding
+      procline "Forked #{@child} at #{Time.now.to_i}"
+      begin
+        Process.waitpid(@child)
+      rescue SystemCallError
+        nil
+      end
+    end
+
+    # Kills the forked child immediately with minimal remorse. The job it
+    # is processing will not be completed. Send the child a TERM signal,
+    # wait 5 seconds, and then a KILL signal if it has not quit
     def kill_child
       return unless @child
-      return unless system("ps -o pid,state -p #{@child}")
-      Process.kill("KILL", @child) rescue nil
+
+      if Process.waitpid(@child, Process::WNOHANG)
+        log "Child #{@child} already quit."
+        return
+      end
+
+      signal_child("TERM", @child)
+
+      signal_child("KILL", @child) unless quit_gracefully?(@child)
+    rescue SystemCallError
+      log "Child #{@child} already quit and reaped."
+    end
+
+    # send a signal to a child, have it logged.
+    def signal_child(signal, child)
+      log "Sending #{signal} signal to child #{child}"
+      Process.kill(signal, child)
+    end
+
+    # has our child quit gracefully within the timeout limit?
+    def quit_gracefully?(child)
+      (term_timeout.to_f * 10).round.times do |i|
+        sleep(0.1)
+        return true if Process.waitpid(child, Process::WNOHANG)
+      end
+
+      false
     end
 
-    # This is stolen directly from resque... (thanks, @defunkt!)
+    # This was originally stolen directly from resque... (thanks, @defunkt!)
     # Registers the various signal handlers a worker responds to.
     #
     # TERM: Shutdown immediately, stop processing jobs.
     #  INT: Shutdown immediately, stop processing jobs.
     # QUIT: Shutdown after the current job has finished processing.
     # USR1: Kill the forked child immediately, continue processing jobs.
-    # USR2: Don't process any new jobs
+    # USR2: Don't process any new jobs; dump the backtrace.
     # CONT: Start processing jobs again after a USR2
-    def register_signal_handlers
+    def register_parent_signal_handlers
       trap('TERM') { shutdown!  }
       trap('INT')  { shutdown!  }
 
       begin
         trap('QUIT') { shutdown   }
         trap('USR1') { kill_child }
-        trap('USR2') { pause_processing }
+        trap('USR2') do
+          log "Current backtrace (parent): \n\n#{caller.join("\n")}\n\n"
+          pause_processing
+        end
+
         trap('CONT') { unpause_processing }
       rescue ArgumentError
         warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
       end
     end
 
+    def register_child_signal_handlers
+      trap('TERM') { raise SignalException.new("SIGTERM") }
+      trap('INT', 'DEFAULT')
+
+      begin
+        trap('QUIT', 'DEFAULT')
+        trap('USR1', 'DEFAULT')
+        trap('USR2', 'DEFAULT')
+
+        trap('USR2') do
+          log "Current backtrace (child): \n\n#{caller.join("\n")}\n\n"
+        end
+      rescue ArgumentError
+      end
+    end
+
     # Log a message to STDOUT if we are verbose or very_verbose.
     def log(message)
       if verbose
@@ -199,6 +333,82 @@ module Qless
     def log!(message)
       log message if very_verbose
     end
+
+    def start_parent_pub_sub_listener_for(client)
+      Subscriber.start(client, "ql:w:#{Qless.worker_name}") do |subscriber, message|
+        if message["event"] == "lock_lost" && message["jid"] == current_job_jid
+          fail_job_due_to_timeout
+          kill_child
+        end
+      end
+    end
+
+    def start_child_pub_sub_listener_for(client)
+      Subscriber.start(client, "ql:w:#{Qless.worker_name}:#{Process.pid}") do |subscriber, message|
+        if message["event"] == "notify_backtrace"
+          notify_parent_of_job_backtrace(client, message.fetch('notify_list'))
+        end
+      end
+    end
+
+    def with_job(job)
+      @job = job
+      yield
+    ensure
+      @job = nil
+    end
+
+    # To prevent race conditions (with our listener thread),
+    # we cannot use a pattern like `use(@job) if @job` because
+    # the value of `@job` could change between the checking of
+    # it and the use of it. Here we use a pattern that avoids
+    # the issue -- get the job into a local, and yield that if
+    # it is set.
+    def access_current_job
+      if job = @job
+        yield job
+      end
+    end
+
+    def current_job_jid
+      access_current_job &:jid
+    end
+
+    JobLockLost = Class.new(StandardError)
+
+    def fail_job_due_to_timeout
+      access_current_job do |job|
+        error = JobLockLost.new
+        error.set_backtrace(get_backtrace_from_child(job.client.redis))
+        fail_job(job, error, caller)
+      end
+    end
+
+    def notify_parent_of_job_backtrace(client, list)
+      job_backtrace = Thread.main.backtrace
+      client.redis.lpush list, JSON.dump(job_backtrace)
+      client.redis.pexpire list, BACKTRACE_EXPIRATION_TIMEOUT_MS
+    end
+
+    WAIT_FOR_CHILD_BACKTRACE_TIMEOUT = 4
+    BACKTRACE_EXPIRATION_TIMEOUT_MS = 60_000 # timeout after a minute
+
+    def get_backtrace_from_child(child_redis)
+      notification_list = "ql:child_backtraces:#{Qless.generate_jid}"
+      request_backtrace = { "event"       => "notify_backtrace",
+                            "notify_list" => notification_list }
+
+      if child_redis.publish("ql:w:#{Qless.worker_name}:#{@child}", JSON.dump(request_backtrace)).zero?
+        return ["Could not obtain child backtrace since it was not listening."]
+      end
+
+      begin
+        _, backtrace_json = child_redis.blpop(notification_list, WAIT_FOR_CHILD_BACKTRACE_TIMEOUT)
+        JSON.parse(backtrace_json)
+      rescue => e
+        ["Could not obtain child backtrace: #{e.class}: #{e.message}"] + e.backtrace
+      end
+    end
   end
 end