qless 0.9.1

data/lib/qless/job.rb

@@ -0,0 +1,259 @@
+require "qless"
+require "qless/queue"
+require "qless/lua"
+require "redis"
+require "json"
+
+module Qless
+  class BaseJob
+    def initialize(client, jid)
+      @client = client
+      @jid    = jid
+    end
+    
+    def klass
+      @klass ||= @klass_name.split('::').inject(Kernel) { |context, name| context.const_get(name) }
+    end
+    
+    def queue
+      @queue ||= Queue.new(@queue_name, @client)
+    end
+  end
+  
+  class Job < BaseJob
+    attr_reader :jid, :expires_at, :state, :queue_name, :history, :worker_name, :failure, :klass_name, :tracked, :dependencies, :dependents
+    attr_reader :original_retries, :retries_left
+    attr_accessor :data, :priority, :tags
+    
+    def perform
+      klass.perform(self)
+    end
+
+    def self.build(client, klass, attributes = {})
+      defaults = {
+        "jid"              => Qless.generate_jid,
+        "data"             => {},
+        "klass"            => klass.to_s,
+        "priority"         => 0,
+        "tags"             => [],
+        "worker"           => "mock_worker",
+        "expires"          => Time.now + (60 * 60), # an hour from now
+        "state"            => "running",
+        "tracked"          => false,
+        "queue"            => "mock_queue",
+        "retries"          => 5,
+        "remaining"        => 5,
+        "failure"          => {},
+        "history"          => [],
+        "dependencies"     => [],
+        "dependents"       => []
+      }
+      attributes = defaults.merge(Qless.stringify_hash_keys(attributes))
+      attributes["data"] = JSON.load(JSON.dump attributes["data"])
+      new(client, attributes)
+    end
+    
+    def initialize(client, atts)
+      super(client, atts.fetch('jid'))
+      %w{jid data priority tags state tracked
+        failure history dependencies dependents}.each do |att|
+        self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
+      end
+      
+      @expires_at       = atts.fetch('expires')
+      @klass_name       = atts.fetch('klass')
+      @queue_name       = atts.fetch('queue')
+      @worker_name      = atts.fetch('worker')
+      @original_retries = atts.fetch('retries')
+      @retries_left     = atts.fetch('remaining')
+
+      # This is a silly side-effect of Lua doing JSON parsing
+      @tags         = [] if @tags == {}
+      @dependents   = [] if @dependents == {}
+      @dependencies = [] if @dependencies == {}
+      @state_changed = false
+    end
+    
+    def priority=(priority)
+      if @client._priority.call([], [@jid, priority])
+        @priority = priority
+      end
+    end
+    
+    def [](key)
+      @data[key]
+    end
+    
+    def []=(key, val)
+      @data[key] = val
+    end
+    
+    def to_s
+      inspect
+    end
+
+    def description
+      "#{@jid} (#{@klass_name} / #{@queue_name})"
+    end
+    
+    def inspect
+      "<Qless::Job #{description}>"
+    end
+    
+    def ttl
+      @expires_at - Time.now.to_f
+    end
+    
+    # Move this from it's current queue into another
+    def move(queue)
+      note_state_change do
+        @client._put.call([queue], [
+          @jid, @klass_name, JSON.generate(@data), Time.now.to_f, 0
+        ])
+      end
+    end
+    
+    # Fail a job
+    def fail(group, message)
+      note_state_change do
+        @client._fail.call([], [
+          @jid,
+          @worker_name,
+          group, message,
+          Time.now.to_f,
+          JSON.generate(@data)]) || false
+      end
+    end
+    
+    # Heartbeat a job
+    def heartbeat()
+      @client._heartbeat.call([], [
+        @jid,
+        @worker_name,
+        Time.now.to_f,
+        JSON.generate(@data)]) || false
+    end
+    
+    # Complete a job
+    # Options include
+    # => next (String) the next queue
+    # => delay (int) how long to delay it in the next queue
+    def complete(nxt=nil, options={})
+      response = note_state_change do
+        if nxt.nil?
+          @client._complete.call([], [
+            @jid, @worker_name, @queue_name, Time.now.to_f, JSON.generate(@data)])
+        else
+          @client._complete.call([], [
+            @jid, @worker_name, @queue_name, Time.now.to_f, JSON.generate(@data), 'next', nxt, 'delay',
+            options.fetch(:delay, 0), 'depends', JSON.generate(options.fetch(:depends, []))])
+        end
+      end
+      response.nil? ? false : response
+    end
+
+    def state_changed?
+      @state_changed
+    end
+    
+    def cancel
+      note_state_change do
+        @client._cancel.call([], [@jid])
+      end
+    end
+    
+    def track()
+      @client._track.call([], ['track', @jid, Time.now.to_f])
+    end
+    
+    def untrack
+      @client._track.call([], ['untrack', @jid, Time.now.to_f])
+    end
+    
+    def tag(*tags)
+      @client._tag.call([], ['add', @jid, Time.now.to_f] + tags)
+    end
+    
+    def untag(*tags)
+      @client._tag.call([], ['remove', @jid, Time.now.to_f] + tags)
+    end
+    
+    def retry(delay=0)
+      results = @client._retry.call([], [@jid, @queue_name, @worker_name, Time.now.to_f, delay])
+      results.nil? ? false : results
+    end
+    
+    def depend(*jids)
+      !!@client._depends.call([], [@jid, 'on'] + jids)
+    end
+    
+    def undepend(*jids)
+      !!@client._depends.call([], [@jid, 'off'] + jids)
+    end
+
+  private
+
+    def note_state_change
+      result = yield
+      @state_changed = true
+      result
+    end
+  end
+  
+  class RecurringJob < BaseJob
+    attr_reader :jid, :data, :priority, :tags, :retries, :interval, :count, :queue_name, :klass_name
+    
+    def initialize(client, atts)
+      super(client, atts.fetch('jid'))
+      %w{jid data priority tags retries interval count}.each do |att|
+        self.instance_variable_set("@#{att}".to_sym, atts.fetch(att))
+      end
+      
+      @klass_name  = atts.fetch('klass')
+      @queue_name  = atts.fetch('queue')
+      @tags        = [] if @tags == {}
+    end
+    
+    def priority=(value)
+      @client._recur.call([], ['update', @jid, 'priority', value])
+      @priority = value
+    end
+    
+    def retries=(value)
+      @client._recur.call([], ['update', @jid, 'retries', value])
+      @retries = value
+    end
+    
+    def interval=(value)
+      @client._recur.call([], ['update', @jid, 'interval', value])
+      @interval = value
+    end
+    
+    def data=(value)
+      @client._recur.call([], ['update', @jid, 'data', JSON.generate(value)])
+      @data = value
+    end
+    
+    def klass=(value)
+      @client._recur.call([], ['update', @jid, 'klass', value.to_s])
+      @klass_name = value.to_s
+    end
+    
+    def move(queue)
+      @client._recur.call([], ['update', @jid, 'queue', queue])
+      @queue_name = queue
+    end
+    
+    def cancel
+      @client._recur.call([], ['off', @jid])
+    end
+    
+    def tag(*tags)
+      @client._recur.call([], ['tag', @jid] + tags)
+    end
+    
+    def untag(*tags)
+      @client._recur.call([], ['untag', @jid] + tags)
+    end
+  end
+end

data/lib/qless/job_reservers/ordered.rb

@@ -0,0 +1,23 @@
+module Qless
+  module JobReservers
+    class Ordered
+      attr_reader :queues
+
+      def initialize(queues)
+        @queues = queues
+      end
+
+      def reserve
+        @queues.each do |q|
+          job = q.pop
+          return job if job
+        end
+        nil
+      end
+
+      def description
+        @description ||= @queues.map(&:name).join(', ') + " (ordered)"
+      end
+    end
+  end
+end

data/lib/qless/job_reservers/round_robin.rb

@@ -0,0 +1,34 @@
+module Qless
+  module JobReservers
+    class RoundRobin
+      attr_reader :queues
+
+      def initialize(queues)
+        @queues = queues
+        @num_queues = queues.size
+        @last_popped_queue_index = @num_queues - 1
+      end
+
+      def reserve
+        @num_queues.times do |i|
+          if job = next_queue.pop
+            return job
+          end
+        end
+        nil
+      end
+
+      def description
+        @description ||= @queues.map(&:name).join(', ') + " (round robin)"
+      end
+
+    private
+
+      def next_queue
+        @last_popped_queue_index = (@last_popped_queue_index + 1) % @num_queues
+        @queues[@last_popped_queue_index]
+      end
+    end
+  end
+end
+

data/lib/qless/lua.rb

@@ -0,0 +1,25 @@
+module Qless
+  class Lua
+    LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
+    
+    def initialize(name, redis)
+      @sha   = nil
+      @name  = name
+      @redis = redis
+      reload()
+    end
+    
+    def reload()
+      @sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
+    end
+    
+    def call(keys, args)
+      begin
+        return @redis.evalsha(@sha, keys.length, *(keys + args))
+      rescue
+        reload
+        return @redis.evalsha(@sha, keys.length, *(keys + args))
+      end
+    end
+  end
+end

data/lib/qless/qless-core/cancel.lua

@@ -0,0 +1,71 @@
+-- Cancel(0, jid)
+-- --------------
+-- Cancel a job from taking place. It will be deleted from the system, and any
+-- attempts to renew a heartbeat will fail, and any attempts to complete it
+-- will fail. If you try to get the data on the object, you will get nothing.
+--
+-- Args:
+--    1) jid
+
+if #KEYS > 0 then error('Cancel(): No Keys should be provided') end
+
+local jid    = assert(ARGV[1], 'Cancel(): Arg "jid" missing.')
+
+-- Find any stage it's associated with and remove its from that stage
+local state, queue, failure, worker = unpack(redis.call('hmget', 'ql:j:' .. jid, 'state', 'queue', 'failure', 'worker'))
+
+if state == 'complete' then
+	return False
+else
+	-- If this job has dependents, then we should probably fail
+	if redis.call('scard', 'ql:j:' .. jid .. '-dependents') > 0 then
+		error('Cancel(): ' .. jid .. ' has un-canceled jobs that depend on it')
+	end
+	
+	-- Remove this job from whatever worker has it, if any
+	if worker then
+		redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
+	end
+	
+	-- Remove it from that queue
+	if queue then
+		redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
+		redis.call('zrem', 'ql:q:' .. queue .. '-locks', jid)
+		redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
+		redis.call('zrem', 'ql:q:' .. queue .. '-depends', jid)
+	end
+	
+	-- We should probably go through all our dependencies and remove ourselves
+	-- from the list of dependents
+	for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')) do
+		redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
+	end
+	
+	-- Delete any notion of dependencies it has
+	redis.call('del', 'ql:j:' .. jid .. '-dependencies')
+	
+	-- If we're in the failed state, remove all of our data
+	if state == 'failed' then
+		failure = cjson.decode(failure)
+		-- We need to make this remove it from the failed queues
+		redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
+		if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
+			redis.call('srem', 'ql:failures', failure.group)
+		end
+	end
+	
+	-- Remove it as a job that's tagged with this particular tag
+	local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
+	for i, tag in ipairs(tags) do
+		redis.call('zrem', 'ql:t:' .. tag, jid)
+		redis.call('zincrby', 'ql:tags', -1, tag)
+	end
+	
+	-- If the job was being tracked, we should notify
+	if redis.call('zscore', 'ql:tracked', jid) ~= false then
+		redis.call('publish', 'canceled', jid)
+	end
+	
+	-- Just go ahead and delete our data
+	redis.call('del', 'ql:j:' .. jid)
+end

data/lib/qless/qless-core/complete.lua

@@ -0,0 +1,218 @@
+-- Complete(0, jid, worker, queue, now, data, [next, q, [(delay, d) | (depends, '["jid1","jid2",...]')])
+-- -----------------------------------------------------------------------------------------------------
+-- Complete a job and optionally put it in another queue, either scheduled or to
+-- be considered waiting immediately. It can also optionally accept other jids
+-- on which this job will be considered dependent before it's considered valid.
+--
+-- Args:
+--    1) jid
+--    2) worker
+--    3) queue
+--    4) now
+--    5) data
+--    *) [next, q, [delay, d]], [depends, '...']
+
+if #KEYS > 0 then error('Complete(): No Keys should be provided') end
+
+local jid    = assert(ARGV[1]               , 'Complete(): Arg "jid" missing.')
+local worker = assert(ARGV[2]               , 'Complete(): Arg "worker" missing.')
+local queue  = assert(ARGV[3]               , 'Complete(): Arg "queue" missing.')
+local now    = assert(tonumber(ARGV[4])     , 'Complete(): Arg "now" not a number or missing: ' .. tostring(ARGV[4]))
+local data   = assert(cjson.decode(ARGV[5]) , 'Complete(): Arg "data" missing or not JSON: '    .. tostring(ARGV[5]))
+
+-- Read in all the optional parameters
+local options = {}
+for i = 6, #ARGV, 2 do options[ARGV[i]] = ARGV[i + 1] end
+
+-- Sanity check on optional args
+local nextq   = options['next']
+local delay   = assert(tonumber(options['delay'] or 0))
+local depends = assert(cjson.decode(options['depends'] or '[]'), 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends']))
+
+-- Delay and depends are not allowed together
+if delay > 0 and #depends > 0 then
+	error('Complete(): "delay" and "depends" are not allowed to be used together')
+end
+
+-- Depends doesn't make sense without nextq
+if options['delay'] and nextq == nil then
+	error('Complete(): "delay" cannot be used without a "next".')
+end
+
+-- Depends doesn't make sense without nextq
+if options['depends'] and nextq == nil then
+	error('Complete(): "depends" cannot be used without a "next".')
+end
+
+-- The bin is midnight of the provided day
+-- 24 * 60 * 60 = 86400
+local bin = now - (now % 86400)
+
+-- First things first, we should see if the worker still owns this job
+local lastworker, history, state, priority, retries = unpack(redis.call('hmget', 'ql:j:' .. jid, 'worker', 'history', 'state', 'priority', 'retries', 'dependents'))
+
+if (lastworker ~= worker) or (state ~= 'running') then
+	return false
+end
+
+-- Now we can assume that the worker does own the job. We need to
+--    1) Remove the job from the 'locks' from the old queue
+--    2) Enqueue it in the next stage if necessary
+--    3) Update the data
+--    4) Mark the job as completed, remove the worker, remove expires, and update history
+
+-- Unpack the history, and update it
+history = cjson.decode(history)
+history[#history]['done'] = math.floor(now)
+
+if data then
+	redis.call('hset', 'ql:j:' .. jid, 'data', cjson.encode(data))
+end
+
+-- Remove the job from the previous queue
+redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
+redis.call('zrem', 'ql:q:' .. queue .. '-locks', jid)
+redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
+
+----------------------------------------------------------
+-- This is the massive stats update that we have to do
+----------------------------------------------------------
+-- This is how long we've been waiting to get popped
+local waiting = math.floor(now) - history[#history]['popped']
+-- Now we'll go through the apparently long and arduous process of update
+local count, mean, vk = unpack(redis.call('hmget', 'ql:s:run:' .. bin .. ':' .. queue, 'total', 'mean', 'vk'))
+count = count or 0
+if count == 0 then
+	mean  = waiting
+	vk    = 0
+	count = 1
+else
+	count = count + 1
+	local oldmean = mean
+	mean  = mean + (waiting - mean) / count
+	vk    = vk + (waiting - mean) * (waiting - oldmean)
+end
+-- Now, update the histogram
+-- - `s1`, `s2`, ..., -- second-resolution histogram counts
+-- - `m1`, `m2`, ..., -- minute-resolution
+-- - `h1`, `h2`, ..., -- hour-resolution
+-- - `d1`, `d2`, ..., -- day-resolution
+waiting = math.floor(waiting)
+if waiting < 60 then -- seconds
+	redis.call('hincrby', 'ql:s:run:' .. bin .. ':' .. queue, 's' .. waiting, 1)
+elseif waiting < 3600 then -- minutes
+	redis.call('hincrby', 'ql:s:run:' .. bin .. ':' .. queue, 'm' .. math.floor(waiting / 60), 1)
+elseif waiting < 86400 then -- hours
+	redis.call('hincrby', 'ql:s:run:' .. bin .. ':' .. queue, 'h' .. math.floor(waiting / 3600), 1)
+else -- days
+	redis.call('hincrby', 'ql:s:run:' .. bin .. ':' .. queue, 'd' .. math.floor(waiting / 86400), 1)
+end		
+redis.call('hmset', 'ql:s:run:' .. bin .. ':' .. queue, 'total', count, 'mean', mean, 'vk', vk)
+----------------------------------------------------------
+
+-- Remove this job from the jobs that the worker that was running it has
+redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
+
+if redis.call('zscore', 'ql:tracked', jid) ~= false then
+	redis.call('publish', 'completed', jid)
+end
+
+if nextq then
+	-- Enqueue the job
+	table.insert(history, {
+		q     = nextq,
+		put   = math.floor(now)
+	})
+	
+	-- We're going to make sure that this queue is in the
+	-- set of known queues
+	if redis.call('zscore', 'ql:queues', nextq) == false then
+		redis.call('zadd', 'ql:queues', now, nextq)
+	end
+	
+	redis.call('hmset', 'ql:j:' .. jid, 'state', 'waiting', 'worker', '', 'failure', '{}',
+		'queue', nextq, 'expires', 0, 'history', cjson.encode(history), 'remaining', tonumber(retries))
+	
+	if delay > 0 then
+	    redis.call('zadd', 'ql:q:' .. nextq .. '-scheduled', now + delay, jid)
+		return 'scheduled'
+	else
+		-- These are the jids we legitimately have to wait on
+		local count = 0
+		for i, j in ipairs(depends) do
+			-- Make sure it's something other than 'nil' or complete.
+			local state = redis.call('hget', 'ql:j:' .. j, 'state')
+			if (state and state ~= 'complete') then
+				count = count + 1
+				redis.call('sadd', 'ql:j:' .. j .. '-dependents'  , jid)
+				redis.call('sadd', 'ql:j:' .. jid .. '-dependencies', j)
+			end
+		end
+		if count > 0 then
+			redis.call('zadd', 'ql:q:' .. nextq .. '-depends', now, jid)
+			redis.call('hset', 'ql:j:' .. jid, 'state', 'depends')
+			return 'depends'
+		else
+			redis.call('zadd', 'ql:q:' .. nextq .. '-work', priority - (now / 10000000000), jid)
+			return 'waiting'
+		end
+	end
+else
+	redis.call('hmset', 'ql:j:' .. jid, 'state', 'complete', 'worker', '', 'failure', '{}',
+		'queue', '', 'expires', 0, 'history', cjson.encode(history), 'remaining', tonumber(retries))
+	
+	-- Do the completion dance
+	local count, time = unpack(redis.call('hmget', 'ql:config', 'jobs-history-count', 'jobs-history'))
+	
+	-- These are the default values
+	count = tonumber(count or 50000)
+	time  = tonumber(time  or 7 * 24 * 60 * 60)
+	
+	-- Schedule this job for destructination eventually
+	redis.call('zadd', 'ql:completed', now, jid)
+	
+	-- Now look at the expired job data. First, based on the current time
+	local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
+	-- Any jobs that need to be expired... delete
+	for index, jid in ipairs(jids) do
+		local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
+		for i, tag in ipairs(tags) do
+			redis.call('zrem', 'ql:t:' .. tag, jid)
+			redis.call('zincrby', 'ql:tags', -1, tag)
+		end
+		redis.call('del', 'ql:j:' .. jid)
+	end
+	-- And now remove those from the queued-for-cleanup queue
+	redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
+	
+	-- Now take the all by the most recent 'count' ids
+	jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
+	for index, jid in ipairs(jids) do
+		local tags = cjson.decode(redis.call('hget', 'ql:j:' .. jid, 'tags') or '{}')
+		for i, tag in ipairs(tags) do
+			redis.call('zrem', 'ql:t:' .. tag, jid)
+			redis.call('zincrby', 'ql:tags', -1, tag)
+		end
+		redis.call('del', 'ql:j:' .. jid)
+	end
+	redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
+	
+	-- Alright, if this has any dependents, then we should go ahead
+	-- and unstick those guys.
+	for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependents')) do
+		redis.call('srem', 'ql:j:' .. j .. '-dependencies', jid)
+		if redis.call('scard', 'ql:j:' .. j .. '-dependencies') == 0 then
+			local q, p = unpack(redis.call('hmget', 'ql:j:' .. j, 'queue', 'priority'))
+			if q then
+				redis.call('zrem', 'ql:q:' .. q .. '-depends', j)
+				redis.call('zadd', 'ql:q:' .. q .. '-work', p, j)
+				redis.call('hset', 'ql:j:' .. j, 'state', 'waiting')
+			end
+		end
+	end
+	
+	-- Delete our dependents key
+	redis.call('del', 'ql:j:' .. jid .. '-dependents')
+	
+	return 'complete'
+end