qless 0.9.3 → 0.10.0

data/lib/qless/config.rb

@@ -1,4 +1,6 @@
-require "json"
+# Encoding: utf-8
+
+require 'json'
 
 module Qless
   # A configuration class associated with a qless client
@@ -6,24 +8,24 @@ module Qless
     def initialize(client)
       @client = client
     end
-    
+
     def [](key)
-      @client._config.call([], ['get', key])
+      @client.call('config.get', key)
     end
-    
+
     def []=(key, value)
-      @client._config.call([], ['set', key, value])
+      @client.call('config.set', key, value)
     end
-    
+
     # Get the specified `qless` configuration option, or if
     # none is provided, get the complete current configuration
     def all
-      return JSON.parse(@client._config.call([], ['get']))
+      JSON.parse(@client.call('config.get'))
     end
-    
+
     # Restore this option to the default (remove this option)
     def clear(option)
-      @client._config.call([], ['unset', option])
+      @client.call('config.unset', option)
     end
   end
 end

data/lib/qless/failure_formatter.rb

@@ -0,0 +1,43 @@
+# Encoding: utf-8
+
+module Qless
+  # A helper for formatting failure messages
+  class FailureFormatter
+    Failure = Struct.new(:group, :message) do
+      # allow de-structring assignment
+      def to_ary
+        [group, message]
+      end
+    end
+
+    def initialize
+      @replacements = { Dir.pwd => '.' }
+      @replacements[ENV['GEM_HOME']] = '<GEM_HOME>' if ENV.key?('GEM_HOME')
+    end
+
+    def format(job, error, lines_to_remove = caller(2))
+      group = "#{job.klass_name}:#{error.class}"
+      message = "#{truncated_message(error)}\n\n" +
+        "#{format_failure_backtrace(error.backtrace, lines_to_remove)}"
+      Failure.new(group, message)
+    end
+
+  private
+
+    # TODO: pull this out into a config option.
+    MAX_ERROR_MESSAGE_SIZE = 10_000
+    def truncated_message(error)
+      return error.message if error.message.length <= MAX_ERROR_MESSAGE_SIZE
+      error.message.slice(0, MAX_ERROR_MESSAGE_SIZE) +
+        "\n... (truncated due to length)"
+    end
+
+    def format_failure_backtrace(error_backtrace, lines_to_remove)
+      (error_backtrace - lines_to_remove).map do |line|
+        @replacements.reduce(line) do |formatted, (original, new)|
+          formatted.sub(original, new)
+        end
+      end.join("\n")
+    end
+  end
+end

data/lib/qless/job.rb

@@ -1,9 +1,13 @@
-require "qless"
-require "qless/queue"
-require "redis"
-require "json"
+# Encoding: utf-8
+
+require 'qless'
+require 'qless/queue'
+require 'qless/lua_script'
+require 'redis'
+require 'json'
 
 module Qless
+  # The base for both Job and RecurringJob
   class BaseJob
     attr_reader :client
 
@@ -13,18 +17,35 @@ module Qless
     end
 
     def klass
-      @klass ||= @klass_name.split('::').inject(Object) { |context, name| context.const_get(name) }
+      @klass ||= @klass_name.split('::').reduce(Object) do |context, name|
+        context.const_get(name)
+      end
     end
 
     def queue
       @queue ||= Queue.new(@queue_name, @client)
     end
+
+    def ==(other)
+      self.class == other.class &&
+      jid == other.jid &&
+      client == other.client
+    end
+    alias eql? ==
+
+    def hash
+      self.class.hash ^ jid.hash ^ client.hash
+    end
   end
 
+  # A Qless job
   class Job < BaseJob
-    attr_reader :jid, :expires_at, :state, :queue_name, :worker_name, :failure, :klass_name, :tracked, :dependencies, :dependents
+    attr_reader :jid, :expires_at, :state, :queue_name, :worker_name, :failure, :spawned_from_jid
+    attr_reader :klass_name, :tracked, :dependencies, :dependents
     attr_reader :original_retries, :retries_left, :raw_queue_history
+    attr_reader :state_changed
     attr_accessor :data, :priority, :tags
+    alias_method(:state_changed?, :state_changed)
 
     MiddlewareMisconfiguredError = Class.new(StandardError)
 
@@ -35,14 +56,29 @@ module Qless
     end
 
     def perform
+      # If we can't find the class, we should fail the job, not try to process
+      begin
+        klass
+      rescue NameError
+        return fail("#{queue_name}-NameError", "Cannot find #{klass_name}")
+      end
+
+      # log a real process executing job -- before we start processing
+      log("started by pid:#{Process.pid}")
+
       middlewares = Job.middlewares_on(klass)
 
       if middlewares.last == SupportsMiddleware
         klass.around_perform(self)
       elsif middlewares.any?
-        raise MiddlewareMisconfiguredError, "The middleware chain for #{klass} " +
-          "(#{middlewares.inspect}) is misconfigured. Qless::Job::SupportsMiddleware " +
-          "must be extended onto your job class first if you want to use any middleware."
+        raise MiddlewareMisconfiguredError, 'The middleware chain for ' +
+              "#{klass} (#{middlewares.inspect}) is misconfigured." +
+              'Qless::Job::SupportsMiddleware must be extended onto your job' +
+              'class first if you want to use any middleware.'
+      elsif !klass.respond_to?(:perform)
+        # If the klass doesn't have a :perform method, we should raise an error
+        fail("#{queue_name}-method-missing",
+             "#{klass_name} has no perform method")
       else
         klass.perform(self)
       end
@@ -50,41 +86,46 @@ module Qless
 
     def self.build(client, klass, attributes = {})
       defaults = {
-        "jid"              => Qless.generate_jid,
-        "data"             => {},
-        "klass"            => klass.to_s,
-        "priority"         => 0,
-        "tags"             => [],
-        "worker"           => "mock_worker",
-        "expires"          => Time.now + (60 * 60), # an hour from now
-        "state"            => "running",
-        "tracked"          => false,
-        "queue"            => "mock_queue",
-        "retries"          => 5,
-        "remaining"        => 5,
-        "failure"          => {},
-        "history"          => [],
-        "dependencies"     => [],
-        "dependents"       => []
+        'jid'              => Qless.generate_jid,
+        'spawned_from_jid' => nil,
+        'data'             => {},
+        'klass'            => klass.to_s,
+        'priority'         => 0,
+        'tags'             => [],
+        'worker'           => 'mock_worker',
+        'expires'          => Time.now + (60 * 60), # an hour from now
+        'state'            => 'running',
+        'tracked'          => false,
+        'queue'            => 'mock_queue',
+        'retries'          => 5,
+        'remaining'        => 5,
+        'failure'          => {},
+        'history'          => [],
+        'dependencies'     => [],
+        'dependents'       => []
       }
       attributes = defaults.merge(Qless.stringify_hash_keys(attributes))
-      attributes["data"] = JSON.parse(JSON.dump attributes["data"])
+      attributes['data'] = JSON.dump(attributes['data'])
       new(client, attributes)
     end
 
     def self.middlewares_on(job_klass)
-      job_klass.singleton_class.ancestors.select do |ancestor|
-        ancestor.method_defined?(:around_perform)
+      singleton_klass = job_klass.singleton_class
+      singleton_klass.ancestors.select do |ancestor|
+        ancestor != singleton_klass && ancestor.method_defined?(:around_perform)
       end
     end
 
     def initialize(client, atts)
       super(client, atts.fetch('jid'))
       %w{jid data priority tags state tracked
-        failure dependencies dependents}.each do |att|
-        self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
+         failure dependencies dependents spawned_from_jid}.each do |att|
+        instance_variable_set(:"@#{att}", atts.fetch(att))
       end
 
+      # Parse the data string
+      @data = JSON.parse(@data)
+
       @expires_at        = atts.fetch('expires')
       @klass_name        = atts.fetch('klass')
       @queue_name        = atts.fetch('queue')
@@ -103,9 +144,7 @@ module Qless
     end
 
     def priority=(priority)
-      if @client._priority.call([], [@jid, priority])
-        @priority = priority
-      end
+      @priority = priority if @client.call('priority', @jid, priority)
     end
 
     def [](key)
@@ -137,19 +176,20 @@ module Qless
     end
 
     def history
-      warn "WARNING: Qless::Job#history is deprecated; use Qless::Job#raw_queue_history instead" +
-           "; called from:\n#{caller.first}\n"
+      warn 'WARNING: Qless::Job#history is deprecated; use' +
+           "Qless::Job#raw_queue_history instead; from:\n#{caller.first}"
       raw_queue_history
     end
 
     def queue_history
       @queue_history ||= @raw_queue_history.map do |history_event|
         history_event.each_with_object({}) do |(key, value), hash|
-          # The only Numeric (Integer or Float) values we get in the history are timestamps
-          hash[key] = if value.is_a?(Numeric)
-            Time.at(value).utc
+          # The only Numeric (Integer or Float) values we get in the history
+          # are timestamps
+          if value.is_a?(Numeric)
+            hash[key] = Time.at(value).utc
           else
-            value
+            hash[key] = value
           end
         end
       end
@@ -159,9 +199,14 @@ module Qless
       @initially_put_at ||= history_timestamp('put', :min)
     end
 
+    def spawned_from
+      @spawned_from ||= @client.jobs[@spawned_from_jid]
+    end
+
     def to_hash
       {
         jid: jid,
+        spawned_from_jid: spawned_from_jid,
         expires_at: expires_at,
         state: state,
         queue_name: queue_name,
@@ -181,108 +226,129 @@ module Qless
     end
 
     # Move this from it's current queue into another
-    def move(queue)
-      note_state_change :move do
-        @client._put.call([queue], [
-          @jid, @klass_name, JSON.generate(@data), Time.now.to_f, 0
-        ])
+    def requeue(queue, opts = {})
+      queue_name = case queue
+                     when String, Symbol then queue
+                     else queue.name
+                   end
+
+      note_state_change :requeue do
+        @client.call('requeue', @client.worker_name, queue_name, @jid, @klass_name,
+                     JSON.dump(opts.fetch(:data, @data)),
+                     opts.fetch(:delay, 0),
+                     'priority', opts.fetch(:priority, @priority),
+                     'tags', JSON.dump(opts.fetch(:tags, @tags)),
+                     'retries', opts.fetch(:retries, @original_retries),
+                     'depends', JSON.dump(opts.fetch(:depends, @dependencies))
+        )
       end
     end
+    alias move requeue # for backwards compatibility
+
+    CantFailError = Class.new(Qless::LuaScriptError)
 
     # Fail a job
     def fail(group, message)
       note_state_change :fail do
-        @client._fail.call([], [
+        @client.call(
+          'fail',
           @jid,
           @worker_name,
           group, message,
-          Time.now.to_f,
-          JSON.generate(@data)]) || false
+          JSON.dump(@data)) || false
       end
+    rescue Qless::LuaScriptError => err
+      raise CantFailError.new(err.message)
     end
 
     # Heartbeat a job
-    def heartbeat()
-      @client._heartbeat.call([], [
+    def heartbeat
+      @expires_at = @client.call(
+        'heartbeat',
         @jid,
         @worker_name,
-        Time.now.to_f,
-        JSON.generate(@data)]) || false
+        JSON.dump(@data))
     end
 
-    CantCompleteError = Class.new(Qless::Error)
+    CantCompleteError = Class.new(Qless::LuaScriptError)
 
     # Complete a job
     # Options include
     # => next (String) the next queue
     # => delay (int) how long to delay it in the next queue
-    def complete(nxt=nil, options={})
+    def complete(nxt = nil, options = {})
       note_state_change :complete do
-        response = if nxt.nil?
-          @client._complete.call([], [
-            @jid, @worker_name, @queue_name, Time.now.to_f, JSON.generate(@data)])
+        if nxt.nil?
+          @client.call(
+            'complete', @jid, @worker_name, @queue_name, JSON.dump(@data))
         else
-          @client._complete.call([], [
-            @jid, @worker_name, @queue_name, Time.now.to_f, JSON.generate(@data), 'next', nxt, 'delay',
-            options.fetch(:delay, 0), 'depends', JSON.generate(options.fetch(:depends, []))])
-        end
-
-        if response
-          response
-        else
-          description = if reloaded_instance = @client.jobs[@jid]
-                          reloaded_instance.description
-                        else
-                          self.description + " -- can't be reloaded"
-                        end
-
-          raise CantCompleteError, "Failed to complete #{description}"
+          @client.call('complete', @jid, @worker_name, @queue_name,
+                       JSON.dump(@data), 'next', nxt, 'delay',
+                       options.fetch(:delay, 0), 'depends',
+                       JSON.dump(options.fetch(:depends, [])))
         end
       end
-    end
-
-    def state_changed?
-      @state_changed
+    rescue Qless::LuaScriptError => err
+      raise CantCompleteError.new(err.message)
     end
 
     def cancel
       note_state_change :cancel do
-        @client._cancel.call([], [@jid])
+        @client.call('cancel', @jid)
       end
     end
 
-    def track()
-      @client._track.call([], ['track', @jid, Time.now.to_f])
+    def track
+      @client.call('track', 'track', @jid)
     end
 
     def untrack
-      @client._track.call([], ['untrack', @jid, Time.now.to_f])
+      @client.call('track', 'untrack', @jid)
     end
 
     def tag(*tags)
-      @client._tag.call([], ['add', @jid, Time.now.to_f] + tags)
+      @client.call('tag', 'add', @jid, *tags)
     end
 
     def untag(*tags)
-      @client._tag.call([], ['remove', @jid, Time.now.to_f] + tags)
+      @client.call('tag', 'remove', @jid, *tags)
     end
 
-    def retry(delay=0)
+    def retry(delay = 0, group = nil, message = nil)
       note_state_change :retry do
-        results = @client._retry.call([], [@jid, @queue_name, @worker_name, Time.now.to_f, delay])
-        results.nil? ? false : results
+        if group.nil?
+          results = @client.call(
+            'retry', @jid, @queue_name, @worker_name, delay)
+          results.nil? ? false : results
+        else
+          results = @client.call(
+            'retry', @jid, @queue_name, @worker_name, delay, group, message)
+          results.nil? ? false : results
+        end
       end
     end
 
     def depend(*jids)
-      !!@client._depends.call([], [@jid, 'on'] + jids)
+      !!@client.call('depends', @jid, 'on', *jids)
     end
 
     def undepend(*jids)
-      !!@client._depends.call([], [@jid, 'off'] + jids)
+      !!@client.call('depends', @jid, 'off', *jids)
+    end
+
+    def timeout
+      @client.call('timeout', @jid)
     end
 
-    [:fail, :complete, :cancel, :move, :retry].each do |event|
+    def log(message, data = nil)
+      if data
+        @client.call('log', @jid, message, JSON.dump(data))
+      else
+        @client.call('log', @jid, message)
+      end
+    end
+
+    [:fail, :complete, :cancel, :requeue, :retry].each do |event|
       define_method :"before_#{event}" do |&block|
         @before_callbacks[event] << block
       end
@@ -291,8 +357,8 @@ module Qless
         @after_callbacks[event].unshift block
       end
     end
-
-  private
+    alias before_move before_requeue
+    alias after_move  after_requeue
 
     def note_state_change(event)
       @before_callbacks[event].each { |blk| blk.call(self) }
@@ -302,65 +368,98 @@ module Qless
       result
     end
 
+  private
+
     def history_timestamp(name, selector)
-      queue_history.map { |q| q[name] }.compact.send(selector)
+      items = queue_history.select do |q|
+        q['what'] == name
+      end
+      items.map do |q|
+        q['when']
+      end.public_send(selector)
     end
   end
 
+  # Wraps a recurring job
   class RecurringJob < BaseJob
-    attr_reader :jid, :data, :priority, :tags, :retries, :interval, :count, :queue_name, :klass_name
+    attr_reader :jid, :data, :priority, :tags, :retries, :interval, :count
+    attr_reader :queue_name, :klass_name, :backlog
 
     def initialize(client, atts)
       super(client, atts.fetch('jid'))
-      %w{jid data priority tags retries interval count}.each do |att|
-        self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
+      %w{jid data priority tags retries interval count backlog}.each do |att|
+        instance_variable_set("@#{att}".to_sym, atts.fetch(att))
       end
 
+      # Parse the data string
+      @data        = JSON.parse(@data)
       @klass_name  = atts.fetch('klass')
       @queue_name  = atts.fetch('queue')
       @tags        = [] if @tags == {}
     end
 
     def priority=(value)
-      @client._recur.call([], ['update', @jid, 'priority', value])
+      @client.call('recur.update', @jid, 'priority', value)
       @priority = value
     end
 
     def retries=(value)
-      @client._recur.call([], ['update', @jid, 'retries', value])
+      @client.call('recur.update', @jid, 'retries', value)
       @retries = value
     end
 
     def interval=(value)
-      @client._recur.call([], ['update', @jid, 'interval', value])
+      @client.call('recur.update', @jid, 'interval', value)
       @interval = value
     end
 
     def data=(value)
-      @client._recur.call([], ['update', @jid, 'data', JSON.generate(value)])
+      @client.call('recur.update', @jid, 'data', JSON.dump(value))
       @data = value
     end
 
     def klass=(value)
-      @client._recur.call([], ['update', @jid, 'klass', value.to_s])
+      @client.call('recur.update', @jid, 'klass', value.to_s)
       @klass_name = value.to_s
     end
 
+    def backlog=(value)
+      @client.call('recur.update', @jid, 'backlog', value.to_s)
+      @backlog = value
+    end
+
     def move(queue)
-      @client._recur.call([], ['update', @jid, 'queue', queue])
+      @client.call('recur.update', @jid, 'queue', queue)
       @queue_name = queue
     end
+    alias requeue move # for API parity with normal jobs
 
     def cancel
-      @client._recur.call([], ['off', @jid])
+      @client.call('unrecur', @jid)
     end
 
     def tag(*tags)
-      @client._recur.call([], ['tag', @jid] + tags)
+      @client.call('recur.tag', @jid, *tags)
     end
 
     def untag(*tags)
-      @client._recur.call([], ['untag', @jid] + tags)
+      @client.call('recur.untag', @jid, *tags)
+    end
+
+    def last_spawned_jid
+      return nil if never_spawned?
+      "#{jid}-#{count}"
+    end
+
+    def last_spawned_job
+      return nil if never_spawned?
+      @client.jobs[last_spawned_jid]
+    end
+
+  private
+
+    def never_spawned?
+      count.zero?
     end
   end
 end