qless 0.9.2 → 0.9.3

data/lib/qless/job_reservers/round_robin.rb

@@ -19,11 +19,13 @@ module Qless
       end
 
       def description
-        @description ||= @queues.map(&:name).join(', ') + " (round robin)"
+        @description ||= @queues.map(&:name).join(', ') + " (#{self.class::TYPE_DESCRIPTION})"
       end
 
     private
 
+      TYPE_DESCRIPTION = "round robin"
+
       def next_queue
         @last_popped_queue_index = (@last_popped_queue_index + 1) % @num_queues
         @queues[@last_popped_queue_index]

data/lib/qless/job_reservers/shuffled_round_robin.rb

@@ -0,0 +1,14 @@
+require 'qless/job_reservers/round_robin'
+
+module Qless
+  module JobReservers
+    class ShuffledRoundRobin < RoundRobin
+      def initialize(queues)
+        super(queues.shuffle)
+      end
+
+      TYPE_DESCRIPTION = "shuffled round robin"
+    end
+  end
+end
+

data/lib/qless/lua_script.rb

@@ -0,0 +1,42 @@
+require 'digest/sha1'
+
+module Qless
+  class LuaScript
+    LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
+
+    def initialize(name, redis)
+      @name  = name
+      @redis = redis
+      @sha   = Digest::SHA1.hexdigest(script_contents)
+    end
+
+    attr_reader :name, :redis, :sha
+
+    def reload()
+      @sha = @redis.script(:load, script_contents)
+    end
+
+    def call(keys, argv)
+      _call(keys, argv)
+    rescue
+      reload
+      _call(keys, argv)
+    end
+
+  private
+
+    if USING_LEGACY_REDIS_VERSION
+      def _call(keys, argv)
+        @redis.evalsha(@sha, keys.length, *(keys + argv))
+      end
+    else
+      def _call(keys, argv)
+        @redis.evalsha(@sha, keys: keys, argv: argv)
+      end
+    end
+
+    def script_contents
+      @script_contents ||= File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua"))
+    end
+  end
+end

data/lib/qless/middleware/redis_reconnect.rb

@@ -0,0 +1,24 @@
+module Qless
+  module Middleware
+    module RedisReconnect
+      def self.new(*redis_connections, &block)
+        Module.new do
+          define_singleton_method :to_s do
+            "Qless::Middleware::RedisReconnect"
+          end
+
+          block ||= lambda { |job| redis_connections }
+
+          define_method :around_perform do |job|
+            Array(block.call(job)).each do |redis|
+              redis.client.reconnect
+            end
+
+            super(job)
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/qless/middleware/retry_exceptions.rb

@@ -0,0 +1,43 @@
+module Qless
+  module Middleware
+    module RetryExceptions
+      def around_perform(job)
+        super
+      rescue *retryable_exception_classes => e
+        raise if job.retries_left <= 0
+
+        attempt_num = (job.original_retries - job.retries_left) + 1
+        job.retry(backoff_strategy.call(attempt_num))
+      end
+
+      def retryable_exception_classes
+        @retryable_exception_classes ||= []
+      end
+
+      def retry_on(*exception_classes)
+        retryable_exception_classes.push(*exception_classes)
+      end
+
+      NO_BACKOFF_STRATEGY = lambda { |num| 0 }
+
+      def use_backoff_strategy(strategy = nil, &block)
+        @backoff_strategy = strategy || block
+      end
+
+      def backoff_strategy
+        @backoff_strategy ||= NO_BACKOFF_STRATEGY
+      end
+
+      def exponential(base, options = {})
+        rand_fuzz = options.fetch(:rand_fuzz, 1)
+        lambda do |num|
+          base ** num + rand(rand_fuzz)
+        end
+      end
+    end
+  end
+
+  # For backwards compatibility
+  RetryExceptions = Middleware::RetryExceptions
+end
+

data/lib/qless/middleware/sentry.rb

@@ -0,0 +1,70 @@
+require 'raven'
+
+module Qless
+  module Middleware
+    # This middleware logs errors to the sentry exception notification service:
+    # http://getsentry.com/
+    module Sentry
+      def around_perform(job)
+        super
+      rescue Exception => e
+        SentryLogger.new(e, job).log
+        raise
+      end
+
+      # Logs a single exception to Sentry, adding pertinent job info.
+      class SentryLogger
+        def initialize(exception, job)
+          @exception, @job = exception, job
+        end
+
+        def log
+          event = ::Raven::Event.capture_exception(@exception) do |evt|
+            evt.extra = { job: job_metadata }
+          end
+
+          safely_send event
+        end
+
+      private
+
+        def safely_send(event)
+          return unless event
+          ::Raven.send(event)
+        rescue
+          # We don't want to silence our errors when the Sentry server
+          # responds with an error. We'll still see the errors on the
+          # Qless Web UI.
+        end
+
+
+        def job_metadata
+          {
+            jid:      @job.jid,
+            klass:    @job.klass_name,
+            history:  job_history,
+            data:     @job.data,
+            queue:    @job.queue_name,
+            worker:   @job.worker_name,
+            tags:     @job.tags,
+            priority: @job.priority
+          }
+        end
+
+        # We want to log formatted timestamps rather than integer timestamps
+        def job_history
+          @job.queue_history.map do |history_event|
+            history_event.each_with_object({}) do |(key, value), hash|
+              hash[key] = if value.is_a?(Time)
+                value.iso8601
+              else
+                value
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/qless/qless-core/cancel.lua

@@ -9,63 +9,93 @@
 
 if #KEYS > 0 then error('Cancel(): No Keys should be provided') end
 
-local jid    = assert(ARGV[1], 'Cancel(): Arg "jid" missing.')
-
--- Find any stage it's associated with and remove its from that stage
-local state, queue, failure, worker = unpack(redis.call('hmget', 'ql:j:' .. jid, 'state', 'queue', 'failure', 'worker'))
-
-if state == 'complete' then
-	return false
-else
-	-- If this job has dependents, then we should probably fail
-	if redis.call('scard', 'ql:j:' .. jid .. '-dependents') > 0 then
-		error('Cancel(): ' .. jid .. ' has un-canceled jobs that depend on it')
-	end
-
-	-- Remove this job from whatever worker has it, if any
-	if worker then
-		redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
-	end
-
-	-- Remove it from that queue
-	if queue then
-		redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
-		redis.call('zrem', 'ql:q:' .. queue .. '-locks', jid)
-		redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
-		redis.call('zrem', 'ql:q:' .. queue .. '-depends', jid)
-	end
-
-	-- We should probably go through all our dependencies and remove ourselves
-	-- from the list of dependents
-	for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')) do
-		redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
-	end
-
-	-- Delete any notion of dependencies it has
-	redis.call('del', 'ql:j:' .. jid .. '-dependencies')
-
-	-- If we're in the failed state, remove all of our data
-	if state == 'failed' then
-		failure = cjson.decode(failure)
-		-- We need to make this remove it from the failed queues
-		redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
-		if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
-			redis.call('srem', 'ql:failures', failure.group)
-		end
-	end
-
-	-- Remove it as a job that's tagged with this particular tag
-	local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
-	for i, tag in ipairs(tags) do
-		redis.call('zrem', 'ql:t:' .. tag, jid)
-		redis.call('zincrby', 'ql:tags', -1, tag)
-	end
-
-	-- If the job was being tracked, we should notify
-	if redis.call('zscore', 'ql:tracked', jid) ~= false then
-		redis.call('publish', 'canceled', jid)
-	end
-
-	-- Just go ahead and delete our data
-	redis.call('del', 'ql:j:' .. jid)
+local function cancel(jid, jid_set)
+  if not jid_set[jid] then
+    error('Cancel(): ' .. jid .. ' is a dependency of one of the jobs but is not in the provided jid set')
+  end
+
+  -- Find any stage it's associated with and remove its from that stage
+  local state, queue, failure, worker = unpack(redis.call('hmget', 'ql:j:' .. jid, 'state', 'queue', 'failure', 'worker'))
+
+  if state == 'complete' then
+    return false
+  else
+    -- If this job has dependents, then we should probably fail
+    local dependents = redis.call('smembers', 'ql:j:' .. jid .. '-dependents')
+    for _, dependent_jid in ipairs(dependents) do
+      cancel(dependent_jid, jid_set)
+    end
+
+    -- Send a message out on the appropriate channels
+    local encoded = cjson.encode({
+      jid    = jid,
+      worker = worker,
+      event  = 'canceled',
+      queue  = queue
+    })
+    redis.call('publish', 'ql:log', encoded)
+
+    -- Remove this job from whatever worker has it, if any
+    if worker then
+      redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
+      -- If necessary, send a message to the appropriate worker, too
+      redis.call('publish', 'ql:w:' .. worker, encoded)
+    end
+
+    -- Remove it from that queue
+    if queue then
+      redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
+      redis.call('zrem', 'ql:q:' .. queue .. '-locks', jid)
+      redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
+      redis.call('zrem', 'ql:q:' .. queue .. '-depends', jid)
+    end
+
+    -- We should probably go through all our dependencies and remove ourselves
+    -- from the list of dependents
+    for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')) do
+      redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
+    end
+
+    -- Delete any notion of dependencies it has
+    redis.call('del', 'ql:j:' .. jid .. '-dependencies')
+
+    -- If we're in the failed state, remove all of our data
+    if state == 'failed' then
+      failure = cjson.decode(failure)
+      -- We need to make this remove it from the failed queues
+      redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
+      if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
+        redis.call('srem', 'ql:failures', failure.group)
+      end
+    end
+
+    -- Remove it as a job that's tagged with this particular tag
+    local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
+    for i, tag in ipairs(tags) do
+      redis.call('zrem', 'ql:t:' .. tag, jid)
+      redis.call('zincrby', 'ql:tags', -1, tag)
+    end
+
+    -- If the job was being tracked, we should notify
+    if redis.call('zscore', 'ql:tracked', jid) ~= false then
+      redis.call('publish', 'canceled', jid)
+    end
+
+    -- Just go ahead and delete our data
+    redis.call('del', 'ql:j:' .. jid)
+  end
+end
+
+-- Taken from: http://www.lua.org/pil/11.5.html
+local function to_set(list)
+  local set = {}
+  for _, l in ipairs(list) do set[l] = true end
+  return set
+end
+
+local jids    = assert(ARGV, 'Cancel(): Arg "jid" missing.')
+local jid_set = to_set(jids)
+
+for _, jid in ipairs(jids) do
+  cancel(jid, jid_set)
 end

data/lib/qless/qless-core/complete.lua

@@ -118,12 +118,20 @@ if redis.call('zscore', 'ql:tracked', jid) ~= false then
 end
 
 if nextq then
+	-- Send a message out to log
+	redis.call('publish', 'ql:log', cjson.encode({
+		jid   = jid,
+		event = 'advanced',
+		queue = queue,
+		to    = nextq
+	}))
+
 	-- Enqueue the job
 	table.insert(history, {
 		q     = nextq,
 		put   = math.floor(now)
 	})
-	
+
 	-- We're going to make sure that this queue is in the
 	-- set of known queues
 	if redis.call('zscore', 'ql:queues', nextq) == false then
@@ -158,6 +166,13 @@ if nextq then
 		end
 	end
 else
+	-- Send a message out to log
+	redis.call('publish', 'ql:log', cjson.encode({
+		jid   = jid,
+		event = 'completed',
+		queue = queue
+	}))
+
 	redis.call('hmset', 'ql:j:' .. jid, 'state', 'complete', 'worker', '', 'failure', '{}',
 		'queue', '', 'expires', 0, 'history', cjson.encode(history), 'remaining', tonumber(retries))
 	

data/lib/qless/qless-core/config.lua

@@ -33,9 +33,21 @@ if command == 'get' then
 elseif command == 'set' then
 	local option = assert(ARGV[2], 'Config(): Arg "option" missing')
 	local value  = assert(ARGV[3], 'Config(): Arg "value" missing')
+	-- Send out a log message
+	redis.call('publish', 'ql:log', cjson.encode({
+		event  = 'config_set',
+		option = option
+	}))
+
 	redis.call('hset', 'ql:config', option, value)
 elseif command == 'unset' then
 	local option = assert(ARGV[2], 'Config(): Arg "option" missing')
+	-- Send out a log message
+	redis.call('publish', 'ql:log', cjson.encode({
+		event  = 'config_unset',
+		option = option
+	}))
+
 	redis.call('hdel', 'ql:config', option)
 else
 	error('Config(): Unrecognized command ' .. command)

data/lib/qless/qless-core/deregister_workers.lua

@@ -0,0 +1,12 @@
+-- DeregisterWorkers(0, worker)
+-- This script takes the name of a worker(s) on removes it/them 
+-- from the ql:workers set.
+--
+-- Args: The list of workers to deregister.
+
+if #KEYS > 0 then error('DeregisterWorkers(): No Keys should be provided') end
+if #ARGV < 1 then error('DeregisterWorkers(): Must provide at least one worker to deregister') end
+
+local key = 'ql:workers'
+
+redis.call('zrem', key, unpack(ARGV))

data/lib/qless/qless-core/fail.lua

@@ -1,20 +1,21 @@
 -- Fail(0, jid, worker, group, message, now, [data])
 -- -------------------------------------------------
--- Mark the particular job as failed, with the provided group, and a more specific
--- message. By `group`, we mean some phrase that might be one of several categorical
--- modes of failure. The `message` is something more job-specific, like perhaps
--- a traceback.
+-- Mark the particular job as failed, with the provided group, and a more
+-- specific message. By `group`, we mean some phrase that might be one of
+-- several categorical modes of failure. The `message` is something more
+-- job-specific, like perhaps a traceback.
 -- 
--- This method should __not__ be used to note that a job has been dropped or has 
--- failed in a transient way. This method __should__ be used to note that a job has
--- something really wrong with it that must be remedied.
+-- This method should __not__ be used to note that a job has been dropped or
+-- has failed in a transient way. This method __should__ be used to note that
+-- a job has something really wrong with it that must be remedied.
 -- 
--- The motivation behind the `group` is so that similar errors can be grouped together.
--- Optionally, updated data can be provided for the job. A job in any state can be
--- marked as failed. If it has been given to a worker as a job, then its subsequent
--- requests to heartbeat or complete that job will fail. Failed jobs are kept until
--- they are canceled or completed. __Returns__ the id of the failed job if successful,
--- or `False` on failure.
+-- The motivation behind the `group` is so that similar errors can be grouped
+-- together. Optionally, updated data can be provided for the job. A job in
+-- any state can be marked as failed. If it has been given to a worker as a 
+-- job, then its subsequent requests to heartbeat or complete that job will
+-- fail. Failed jobs are kept until they are canceled or completed.
+--
+-- __Returns__ the id of the failed job if successful, or `False` on failure.
 --
 -- Args:
 --    1) jid
@@ -49,6 +50,15 @@ if state ~= 'running' then
 	return false
 end
 
+-- Send out a log message
+redis.call('publish', 'ql:log', cjson.encode({
+	jid     = jid,
+	event   = 'failed',
+	worker  = worker,
+	group   = group,
+	message = message
+}))
+
 if redis.call('zscore', 'ql:tracked', jid) ~= false then
 	redis.call('publish', 'failed', jid)
 end
@@ -104,4 +114,4 @@ redis.call('lpush', 'ql:f:' .. group, jid)
 -- Here is where we'd intcrement stats about the particular stage
 -- and possibly the workers
 
-return jid
+return jid