qless 0.9.1 → 0.9.2

data/bin/install_phantomjs

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+version=phantomjs-1.7.0-linux-i686
+wget http://phantomjs.googlecode.com/files/$version.tar.bz2
+tar xjf $version.tar.bz2
+mv $version phantomjs
+

data/lib/qless.rb

@@ -153,6 +153,10 @@ module Qless
       @queues  = ClientQueues.new(self)
       @workers = ClientWorkers.new(self)
     end
+
+    def inspect
+      "<Qless::Client #{@options} >"
+    end
     
     def events
       # Events needs its own redis instance of the same configuration, because

data/lib/qless/job.rb

@@ -10,21 +10,21 @@ module Qless
       @client = client
       @jid    = jid
     end
-    
+
     def klass
       @klass ||= @klass_name.split('::').inject(Kernel) { |context, name| context.const_get(name) }
     end
-    
+
     def queue
       @queue ||= Queue.new(@queue_name, @client)
     end
   end
-  
+
   class Job < BaseJob
     attr_reader :jid, :expires_at, :state, :queue_name, :history, :worker_name, :failure, :klass_name, :tracked, :dependencies, :dependents
     attr_reader :original_retries, :retries_left
     attr_accessor :data, :priority, :tags
-    
+
     def perform
       klass.perform(self)
     end
@@ -52,14 +52,14 @@ module Qless
       attributes["data"] = JSON.load(JSON.dump attributes["data"])
       new(client, attributes)
     end
-    
+
     def initialize(client, atts)
       super(client, atts.fetch('jid'))
       %w{jid data priority tags state tracked
         failure history dependencies dependents}.each do |att|
         self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
       end
-      
+
       @expires_at       = atts.fetch('expires')
       @klass_name       = atts.fetch('klass')
       @queue_name       = atts.fetch('queue')
@@ -73,21 +73,21 @@ module Qless
       @dependencies = [] if @dependencies == {}
       @state_changed = false
     end
-    
+
     def priority=(priority)
       if @client._priority.call([], [@jid, priority])
         @priority = priority
       end
     end
-    
+
     def [](key)
       @data[key]
     end
-    
+
     def []=(key, val)
       @data[key] = val
     end
-    
+
     def to_s
       inspect
     end
@@ -95,15 +95,15 @@ module Qless
     def description
       "#{@jid} (#{@klass_name} / #{@queue_name})"
     end
-    
+
     def inspect
       "<Qless::Job #{description}>"
     end
-    
+
     def ttl
       @expires_at - Time.now.to_f
     end
-    
+
     # Move this from it's current queue into another
     def move(queue)
       note_state_change do
@@ -112,7 +112,7 @@ module Qless
         ])
       end
     end
-    
+
     # Fail a job
     def fail(group, message)
       note_state_change do
@@ -124,7 +124,7 @@ module Qless
           JSON.generate(@data)]) || false
       end
     end
-    
+
     # Heartbeat a job
     def heartbeat()
       @client._heartbeat.call([], [
@@ -133,7 +133,7 @@ module Qless
         Time.now.to_f,
         JSON.generate(@data)]) || false
     end
-    
+
     # Complete a job
     # Options include
     # => next (String) the next queue
@@ -155,38 +155,40 @@ module Qless
     def state_changed?
       @state_changed
     end
-    
+
     def cancel
       note_state_change do
         @client._cancel.call([], [@jid])
       end
     end
-    
+
     def track()
       @client._track.call([], ['track', @jid, Time.now.to_f])
     end
-    
+
     def untrack
       @client._track.call([], ['untrack', @jid, Time.now.to_f])
     end
-    
+
     def tag(*tags)
       @client._tag.call([], ['add', @jid, Time.now.to_f] + tags)
     end
-    
+
     def untag(*tags)
       @client._tag.call([], ['remove', @jid, Time.now.to_f] + tags)
     end
-    
+
     def retry(delay=0)
-      results = @client._retry.call([], [@jid, @queue_name, @worker_name, Time.now.to_f, delay])
-      results.nil? ? false : results
+      note_state_change do
+        results = @client._retry.call([], [@jid, @queue_name, @worker_name, Time.now.to_f, delay])
+        results.nil? ? false : results
+      end
     end
-    
+
     def depend(*jids)
       !!@client._depends.call([], [@jid, 'on'] + jids)
     end
-    
+
     def undepend(*jids)
       !!@client._depends.call([], [@jid, 'off'] + jids)
     end
@@ -199,59 +201,59 @@ module Qless
       result
     end
   end
-  
+
   class RecurringJob < BaseJob
     attr_reader :jid, :data, :priority, :tags, :retries, :interval, :count, :queue_name, :klass_name
-    
+
     def initialize(client, atts)
       super(client, atts.fetch('jid'))
       %w{jid data priority tags retries interval count}.each do |att|
         self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
       end
-      
+
       @klass_name  = atts.fetch('klass')
       @queue_name  = atts.fetch('queue')
       @tags        = [] if @tags == {}
     end
-    
+
     def priority=(value)
       @client._recur.call([], ['update', @jid, 'priority', value])
       @priority = value
     end
-    
+
     def retries=(value)
       @client._recur.call([], ['update', @jid, 'retries', value])
       @retries = value
     end
-    
+
     def interval=(value)
       @client._recur.call([], ['update', @jid, 'interval', value])
       @interval = value
     end
-    
+
     def data=(value)
       @client._recur.call([], ['update', @jid, 'data', JSON.generate(value)])
       @data = value
     end
-    
+
     def klass=(value)
       @client._recur.call([], ['update', @jid, 'klass', value.to_s])
       @klass_name = value.to_s
     end
-    
+
     def move(queue)
       @client._recur.call([], ['update', @jid, 'queue', queue])
       @queue_name = queue
     end
-    
+
     def cancel
       @client._recur.call([], ['off', @jid])
     end
-    
+
     def tag(*tags)
       @client._recur.call([], ['tag', @jid] + tags)
     end
-    
+
     def untag(*tags)
       @client._recur.call([], ['untag', @jid] + tags)
     end

data/lib/qless/qless-core/cancel.lua

@@ -15,18 +15,18 @@ local jid    = assert(ARGV[1], 'Cancel(): Arg "jid" missing.')
 local state, queue, failure, worker = unpack(redis.call('hmget', 'ql:j:' .. jid, 'state', 'queue', 'failure', 'worker'))
 
 if state == 'complete' then
-	return False
+	return false
 else
 	-- If this job has dependents, then we should probably fail
 	if redis.call('scard', 'ql:j:' .. jid .. '-dependents') > 0 then
 		error('Cancel(): ' .. jid .. ' has un-canceled jobs that depend on it')
 	end
-	
+
 	-- Remove this job from whatever worker has it, if any
 	if worker then
 		redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
 	end
-	
+
 	-- Remove it from that queue
 	if queue then
 		redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
@@ -34,16 +34,16 @@ else
 		redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
 		redis.call('zrem', 'ql:q:' .. queue .. '-depends', jid)
 	end
-	
+
 	-- We should probably go through all our dependencies and remove ourselves
 	-- from the list of dependents
 	for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')) do
 		redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
 	end
-	
+
 	-- Delete any notion of dependencies it has
 	redis.call('del', 'ql:j:' .. jid .. '-dependencies')
-	
+
 	-- If we're in the failed state, remove all of our data
 	if state == 'failed' then
 		failure = cjson.decode(failure)
@@ -53,19 +53,19 @@ else
 			redis.call('srem', 'ql:failures', failure.group)
 		end
 	end
-	
+
 	-- Remove it as a job that's tagged with this particular tag
 	local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
 	for i, tag in ipairs(tags) do
 		redis.call('zrem', 'ql:t:' .. tag, jid)
 		redis.call('zincrby', 'ql:tags', -1, tag)
 	end
-	
+
 	-- If the job was being tracked, we should notify
 	if redis.call('zscore', 'ql:tracked', jid) ~= false then
 		redis.call('publish', 'canceled', jid)
 	end
-	
+
 	-- Just go ahead and delete our data
 	redis.call('del', 'ql:j:' .. jid)
 end

data/lib/qless/qless-core/failed.lua

@@ -43,7 +43,7 @@ if group then
 		total = redis.call('llen', 'ql:f:' .. group),
 		jobs  = {}
 	}
-	local jids = redis.call('lrange', 'ql:f:' .. group, start, limit)
+	local jids = redis.call('lrange', 'ql:f:' .. group, start, limit - 1)
 	for index, jid in ipairs(jids) do
 		local job = redis.call(
 		    'hmget', 'ql:j:' .. jid, 'jid', 'klass', 'state', 'queue', 'worker', 'priority',

data/lib/qless/qless-core/peek.lua

@@ -34,7 +34,10 @@ end
 -- If we still need jobs in order to meet demand, then we should
 -- look for all the recurring jobs that need jobs run
 if #keys < count then
-	local r = redis.call('zrangebyscore', key .. '-recur', 0, now)
+	-- This is how many jobs we've moved so far
+	local moved = 0
+	-- These are the recurring jobs that need work
+	local r = redis.call('zrangebyscore', key .. '-recur', 0, now, 'LIMIT', 0, count)
 	for index, jid in ipairs(r) do
 		-- For each of the jids that need jobs scheduled, first
 		-- get the last time each of them was run, and then increment
@@ -43,8 +46,12 @@ if #keys < count then
 		local klass, data, priority, tags, retries, interval = unpack(redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority', 'tags', 'retries', 'interval'))
 		local _tags = cjson.decode(tags)
 		
-		while math.floor(tonumber(redis.call('zscore', key .. '-recur', jid))) < now do
+		-- We're saving this value so that in the history, we can accurately 
+		-- reflect when the job would normally have been scheduled
+		local score = math.floor(tonumber(redis.call('zscore', key .. '-recur', jid)))
+		while (score <= now) and (moved < (count - #keys)) do
 			local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
+			moved = moved + 1
 			
 			-- Add this job to the list of jobs tagged with whatever tags were supplied
 			for i, tag in ipairs(_tags) do
@@ -54,28 +61,31 @@ if #keys < count then
 			
 			-- First, let's save its data
 			redis.call('hmset', 'ql:j:' .. jid .. '-' .. count,
-			    'jid'      , jid .. '-' .. count,
+				'jid'      , jid .. '-' .. count,
 				'klass'    , klass,
-			    'data'     , data,
-			    'priority' , priority,
-			    'tags'     , tags,
-			    'state'    , 'waiting',
-			    'worker'   , '',
+				'data'     , data,
+				'priority' , priority,
+				'tags'     , tags,
+				'state'    , 'waiting',
+				'worker'   , '',
 				'expires'  , 0,
-			    'queue'    , queue,
+				'queue'    , queue,
 				'retries'  , retries,
 				'remaining', retries,
-			    'history'  , cjson.encode({{
+				'history'  , cjson.encode({{
+					-- The job was essentially put in this queue at this time,
+					-- and not the current time
 					q     = queue,
-					put   = math.floor(now)
+					put   = math.floor(score)
 				}}))
 			
 			-- Now, if a delay was provided, and if it's in the future,
 			-- then we'll have to schedule it. Otherwise, we're just
 			-- going to add it to the work queue.
-			redis.call('zadd', key .. '-work', priority - (now / 10000000000), jid .. '-' .. count)
+			redis.call('zadd', key .. '-work', priority - (score / 10000000000), jid .. '-' .. count)
 			
 			redis.call('zincrby', key .. '-recur', interval, jid)
+			score = score + interval
 		end
 	end
 end

data/lib/qless/qless-core/pop.lua

@@ -42,6 +42,10 @@ redis.call('zadd', 'ql:workers', now, worker)
 -- Iterate through all the expired locks and add them to the list
 -- of keys that we'll return
 for index, jid in ipairs(redis.call('zrangebyscore', key .. '-locks', 0, now, 'LIMIT', 0, count)) do
+	-- Remove this job from the jobs that the worker that was running it has
+	local w = redis.call('hget', 'ql:j:' .. jid, 'worker')
+	redis.call('zrem', 'ql:w:' .. w .. ':jobs', jid)
+	
 	-- For each of these, decrement their retries. If any of them
 	-- have exhausted their retries, then we should mark them as
 	-- failed.
@@ -82,10 +86,6 @@ for index, jid in ipairs(redis.call('zrangebyscore', key .. '-locks', 0, now, 'L
 			redis.call('publish', 'stalled', jid)
 		end
 	end
-	
-	-- Remove this job from the jobs that the worker that was running it has
-	local w = redis.call('hget', 'ql:j:' .. jid, 'worker')
-	redis.call('zrem', 'ql:w:' .. w .. ':jobs', jid)
 end
 -- Now we've checked __all__ the locks for this queue the could
 -- have expired, and are no more than the number requested.
@@ -97,7 +97,10 @@ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'retries', #keys)
 -- If we still need jobs in order to meet demand, then we should
 -- look for all the recurring jobs that need jobs run
 if #keys < count then
-	local r = redis.call('zrangebyscore', key .. '-recur', 0, now)
+	-- This is how many jobs we've moved so far
+	local moved = 0
+	-- These are the recurring jobs that need work
+	local r = redis.call('zrangebyscore', key .. '-recur', 0, now, 'LIMIT', 0, (count - #keys))
 	for index, jid in ipairs(r) do
 		-- For each of the jids that need jobs scheduled, first
 		-- get the last time each of them was run, and then increment
@@ -106,7 +109,16 @@ if #keys < count then
 		local klass, data, priority, tags, retries, interval = unpack(redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority', 'tags', 'retries', 'interval'))
 		local _tags = cjson.decode(tags)
 		
-		while math.floor(tonumber(redis.call('zscore', key .. '-recur', jid))) <= now do
+		-- We're saving this value so that in the history, we can accurately 
+		-- reflect when the job would normally have been scheduled
+		local score = math.floor(tonumber(redis.call('zscore', key .. '-recur', jid)))
+
+		while (score <= now) and (moved < (count - #keys)) do
+			-- Increment the count of how many jobs we've moved from recurring
+			-- to 'work'
+			moved = moved + 1
+
+			-- the count'th job that we've moved from this recurring job
 			local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
 			
 			-- Add this job to the list of jobs tagged with whatever tags were supplied
@@ -117,28 +129,31 @@ if #keys < count then
 			
 			-- First, let's save its data
 			redis.call('hmset', 'ql:j:' .. jid .. '-' .. count,
-			    'jid'      , jid .. '-' .. count,
+				'jid'      , jid .. '-' .. count,
 				'klass'    , klass,
-			    'data'     , data,
-			    'priority' , priority,
-			    'tags'     , tags,
-			    'state'    , 'waiting',
-			    'worker'   , '',
+				'data'     , data,
+				'priority' , priority,
+				'tags'     , tags,
+				'state'    , 'waiting',
+				'worker'   , '',
 				'expires'  , 0,
-			    'queue'    , queue,
+				'queue'    , queue,
 				'retries'  , retries,
 				'remaining', retries,
-			    'history'  , cjson.encode({{
+				'history'  , cjson.encode({{
+					-- The job was essentially put in this queue at this time,
+					-- and not the current time
 					q     = queue,
-					put   = math.floor(now)
+					put   = math.floor(score)
 				}}))
 			
 			-- Now, if a delay was provided, and if it's in the future,
 			-- then we'll have to schedule it. Otherwise, we're just
 			-- going to add it to the work queue.
-			redis.call('zadd', key .. '-work', priority - (now / 10000000000), jid .. '-' .. count)
+			redis.call('zadd', key .. '-work', priority - (score / 10000000000), jid .. '-' .. count)
 			
 			redis.call('zincrby', key .. '-recur', interval, jid)
+			score = score + interval
 		end
 	end
 end