qless 0.9.3 → 0.10.0

data/lib/qless/queue.rb

@@ -1,43 +1,52 @@
-require "qless/job"
-require "redis"
-require "json"
+# Encoding: utf-8
+
+require 'qless/job'
+require 'redis'
+require 'json'
 
 module Qless
+  # A class for interacting with jobs in different states in a queue. Not meant
+  # to be instantiated directly, it's accessed with Queue#jobs
   class QueueJobs
     def initialize(name, client)
       @name   = name
       @client = client
     end
 
-    def running(start=0, count=25)
-      @client._jobs.call([], ['running', Time.now.to_f, @name, start, count])
+    def running(start = 0, count = 25)
+      @client.call('jobs', 'running', @name, start, count)
     end
 
-    def stalled(start=0, count=25)
-      @client._jobs.call([], ['stalled', Time.now.to_f, @name, start, count])
+    def stalled(start = 0, count = 25)
+      @client.call('jobs', 'stalled', @name, start, count)
     end
 
-    def scheduled(start=0, count=25)
-      @client._jobs.call([], ['scheduled', Time.now.to_f, @name, start, count])
+    def scheduled(start = 0, count = 25)
+      @client.call('jobs', 'scheduled', @name, start, count)
     end
 
-    def depends(start=0, count=25)
-      @client._jobs.call([], ['depends', Time.now.to_f, @name, start, count])
+    def depends(start = 0, count = 25)
+      @client.call('jobs', 'depends', @name, start, count)
     end
 
-    def recurring(start=0, count=25)
-      @client._jobs.call([], ['recurring', Time.now.to_f, @name, start, count])
+    def recurring(start = 0, count = 25)
+      @client.call('jobs', 'recurring', @name, start, count)
     end
   end
 
+  # A class for interacting with a specific queue. Not meant to be instantiated
+  # directly, it's accessed with Client#queues[...]
   class Queue
     attr_reader   :name, :client
-    attr_accessor :worker_name
 
     def initialize(name, client)
       @client = client
       @name   = name
-      self.worker_name = Qless.worker_name
+    end
+
+    # Our worker name is the same as our client's
+    def worker_name
+      @client.worker_name
     end
 
     def jobs
@@ -45,7 +54,7 @@ module Qless
     end
 
     def counts
-      JSON.parse(@client._queues.call([], [Time.now.to_i, @name]))
+      JSON.parse(@client.call('queues', @name))
     end
 
     def heartbeat
@@ -57,20 +66,36 @@ module Qless
     end
 
     def max_concurrency
-      value = get_config(:"max-concurrency")
+      value = get_config('max-concurrency')
       value && Integer(value)
     end
 
     def max_concurrency=(value)
-      set_config :"max-concurrency", value
+      set_config 'max-concurrency', value
+    end
+
+    def paused?
+      counts['paused']
     end
 
-    def pause
-      @client._pause.call([], [name])
+    def pause(opts = {})
+      @client.call('pause', name)
+      @client.call('timeout', jobs.running(0, -1)) unless opts[:stopjobs].nil?
     end
 
     def unpause
-      @client._unpause.call([], [name])
+      @client.call('unpause', name)
+    end
+
+    QueueNotEmptyError = Class.new(StandardError)
+
+    def forget
+      job_count = length
+      if job_count.zero?
+        @client.call('queue.forget', name)
+      else
+        raise QueueNotEmptyError, "The queue is not empty. It has #{job_count} jobs."
+      end
     end
 
     # Put the described job in this queue
@@ -78,20 +103,18 @@ module Qless
     # => priority (int)
     # => tags (array of strings)
     # => delay (int)
-    def put(klass, data, opts={})
+    def put(klass, data, opts = {})
       opts = job_options(klass, data, opts)
-
-      @client._put.call([@name], [
-        (opts[:jid] or Qless.generate_jid),
-        klass.name,
-        JSON.generate(data),
-        Time.now.to_f,
-        opts.fetch(:delay, 0),
-        'priority', opts.fetch(:priority, 0),
-        'tags', JSON.generate(opts.fetch(:tags, [])),
-        'retries', opts.fetch(:retries, 5),
-        'depends', JSON.generate(opts.fetch(:depends, []))
-      ])
+      @client.call('put', worker_name, @name,
+                   (opts[:jid] || Qless.generate_jid),
+                   klass.is_a?(String) ? klass : klass.name,
+                   JSON.generate(data),
+                   opts.fetch(:delay, 0),
+                   'priority', opts.fetch(:priority, 0),
+                   'tags', JSON.generate(opts.fetch(:tags, [])),
+                   'retries', opts.fetch(:retries, 5),
+                   'depends', JSON.generate(opts.fetch(:depends, []))
+      )
     end
 
     # Make a recurring job in this queue
@@ -100,52 +123,64 @@ module Qless
     # => tags (array of strings)
     # => retries (int)
     # => offset (int)
-    def recur(klass, data, interval, opts={})
+    def recur(klass, data, interval, opts = {})
       opts = job_options(klass, data, opts)
-
-      @client._recur.call([], [
-        'on',
+      @client.call(
+        'recur',
         @name,
-        (opts[:jid] or Qless.generate_jid),
-        klass.to_s,
+        (opts[:jid] || Qless.generate_jid),
+        klass.is_a?(String) ? klass : klass.name,
         JSON.generate(data),
-        Time.now.to_f,
         'interval', interval, opts.fetch(:offset, 0),
         'priority', opts.fetch(:priority, 0),
         'tags', JSON.generate(opts.fetch(:tags, [])),
-        'retries', opts.fetch(:retries, 5)
-      ])
+        'retries', opts.fetch(:retries, 5),
+        'backlog', opts.fetch(:backlog, 0)
+      )
     end
 
     # Pop a work item off the queue
-    def pop(count=nil)
-      results = @client._pop.call([@name], [worker_name, (count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
-      count.nil? ? results[0] : results
+    def pop(count = nil)
+      jids = JSON.parse(@client.call('pop', @name, worker_name, (count || 1)))
+      jobs = jids.map { |j| Job.new(@client, j) }
+      count.nil? ? jobs[0] : jobs
     end
 
     # Peek at a work item
-    def peek(count=nil)
-      results = @client._peek.call([@name], [(count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
-      count.nil? ? results[0] : results
+    def peek(count = nil)
+      jids = JSON.parse(@client.call('peek', @name, (count || 1)))
+      jobs = jids.map { |j| Job.new(@client, j) }
+      count.nil? ? jobs[0] : jobs
     end
 
-    def stats(date=nil)
-      JSON.parse(@client._stats.call([], [@name, (date || Time.now.to_f)]))
+    def stats(date = nil)
+      JSON.parse(@client.call('stats', @name, (date || Time.now.to_f)))
     end
 
     # How many items in the queue?
     def length
       (@client.redis.multi do
-        @client.redis.zcard("ql:q:#{@name}-locks")
-        @client.redis.zcard("ql:q:#{@name}-work")
-        @client.redis.zcard("ql:q:#{@name}-scheduled")
+        %w[ locks work scheduled depends ].each do |suffix|
+          @client.redis.zcard("ql:q:#{@name}-#{suffix}")
+        end
       end).inject(0, :+)
     end
 
     def to_s
       "#<Qless::Queue #{@name}>"
     end
-    alias inspect to_s
+    alias_method :inspect, :to_s
+
+    def ==(other)
+      self.class == other.class &&
+      client == other.client &&
+      name.to_s == other.name.to_s
+    end
+    alias eql? ==
+
+    def hash
+      self.class.hash ^ client.hash ^ name.to_s.hash
+    end
 
   private
 

data/lib/qless/server.rb

@@ -1,9 +1,10 @@
+# Encoding: utf-8
+
 require 'sinatra/base'
 require 'qless'
 
-# Much of this is shamelessly poached from the resque web client
-
 module Qless
+  # The Qless web interface
   class Server < Sinatra::Base
     # Path-y-ness
     dir = File.dirname(File.expand_path(__FILE__))
@@ -27,7 +28,7 @@ module Qless
       include Rack::Utils
 
       def url_path(*path_parts)
-        [ path_prefix, path_parts ].join("/").squeeze('/')
+        [path_prefix, path_parts].join('/').squeeze('/')
       end
       alias_method :u, :url_path
 
@@ -49,11 +50,11 @@ module Qless
       end
 
       def next_page_url
-        page_url 1
+        page_url(1)
       end
 
       def prev_page_url
-        page_url -1
+        page_url(-1)
       end
 
       def current_page
@@ -75,34 +76,35 @@ module Qless
       end
 
       def tabs
-        return [
-          {:name => 'Queues'  , :path => '/queues'  },
-          {:name => 'Workers' , :path => '/workers' },
-          {:name => 'Track'   , :path => '/track'   },
-          {:name => 'Failed'  , :path => '/failed'  },
-          {:name => 'Config'  , :path => '/config'  },
-          {:name => 'About'   , :path => '/about'   }
+        [
+          { name: 'Queues'   , path: '/queues'   },
+          { name: 'Workers'  , path: '/workers'  },
+          { name: 'Track'    , path: '/track'    },
+          { name: 'Failed'   , path: '/failed'   },
+          { name: 'Completed', path: '/completed'},
+          { name: 'Config'   , path: '/config'   },
+          { name: 'About'    , path: '/about'    }
         ]
       end
 
       def application_name
-        return client.config['application']
+        client.config['application']
       end
 
       def queues
-        return client.queues.counts
+        client.queues.counts
       end
 
       def tracked
-        return client.jobs.tracked
+        client.jobs.tracked
       end
 
       def workers
-        return client.workers.counts
+        client.workers.counts
       end
 
       def failed
-        return client.jobs.failed
+        client.jobs.failed
       end
 
       # Return the supplied object back as JSON
@@ -113,45 +115,46 @@ module Qless
 
       # Make the id acceptable as an id / att in HTML
       def sanitize_attr(attr)
-        return attr.gsub(/[^a-zA-Z\:\_]/, '-')
+        attr.gsub(/[^a-zA-Z\:\_]/, '-')
       end
 
       # What are the top tags? Since it might go on, say, every
       # page, then we should probably be caching it
       def top_tags
         @top_tags ||= {
-          :top     => client.tags,
-          :fetched => Time.now
+          top: client.tags,
+          fetched: Time.now
         }
-        if (Time.now - @top_tags[:fetched]) > 60 then
+        if (Time.now - @top_tags[:fetched]) > 60
           @top_tags = {
-            :top     => client.tags,
-            :fetched => Time.now
+            top: client.tags,
+            fetched: Time.now
           }
         end
         @top_tags[:top]
       end
 
       def strftime(t)
-        # From http://stackoverflow.com/questions/195740/how-do-you-do-relative-time-in-rails
+        # From http://stackoverflow.com/questions/195740
         diff_seconds = Time.now - t
+        formatted = t.strftime('%b %e, %Y %H:%M:%S')
         case diff_seconds
-          when 0 .. 59
-            "#{diff_seconds.to_i} seconds ago"
-          when 60 ... 3600
-            "#{(diff_seconds/60).to_i} minutes ago"
-          when 3600 ... 3600*24
-            "#{(diff_seconds/3600).to_i} hours ago"
-          when (3600*24) ... (3600*24*30)
-            "#{(diff_seconds/(3600*24)).to_i} days ago"
-          else
-            t.strftime('%b %e, %Y %H:%M:%S %Z (%z)')
+        when 0 .. 59
+          "#{formatted} (#{diff_seconds.to_i} seconds ago)"
+        when 60 ... 3600
+          "#{formatted} (#{(diff_seconds / 60).to_i} minutes ago)"
+        when 3600 ... 3600 * 24
+          "#{formatted} (#{(diff_seconds / 3600).to_i} hours ago)"
+        when (3600 * 24) ... (3600 * 24 * 30)
+          "#{formatted} (#{(diff_seconds / (3600 * 24)).to_i} days ago)"
+        else
+          formatted
         end
       end
     end
 
     get '/?' do
-      erb :overview, :layout => true, :locals => { :title => "Overview" }
+      erb :overview, layout: true, locals: { title: 'Overview' }
     end
 
     # Returns a JSON blob with the job counts for various queues
@@ -160,8 +163,8 @@ module Qless
     end
 
     get '/queues/?' do
-      erb :queues, :layout => true, :locals => {
-        :title   => 'Queues'
+      erb :queues, layout: true, locals: {
+        title: 'Queues'
       }
     end
 
@@ -175,20 +178,19 @@ module Qless
       queue = client.queues[params[:name]]
       tab   = params.fetch('tab', 'stats')
 
-      jobs = if tab == 'waiting'
-        queue.peek(20)
+      jobs = []
+      if tab == 'waiting'
+        jobs = queue.peek(20)
       elsif filtered_tabs.include?(tab)
-        paginated(queue.jobs, tab).map { |jid| client.jobs[jid] }
-      else
-        []
+        jobs = paginated(queue.jobs, tab).map { |jid| client.jobs[jid] }
       end
 
-      erb :queue, :layout => true, :locals => {
-        :title   => "Queue #{params[:name]}",
-        :tab     => tab,
-        :jobs    => jobs,
-        :queue   => client.queues[params[:name]].counts,
-        :stats   => queue.stats
+      erb :queue, layout: true, locals: {
+        title: "Queue #{params[:name]}",
+        tab: tab,
+        jobs: jobs,
+        queue: client.queues[params[:name]].counts,
+        stats: queue.stats
       }
     end
 
@@ -200,106 +202,117 @@ module Qless
       # qless-core doesn't provide functionality this way, so we'll
       # do it ourselves. I'm not sure if this is how the core library
       # should behave or not.
-      erb :failed, :layout => true, :locals => {
-        :title  => 'Failed',
-        :failed => client.jobs.failed.keys.map { |t| client.jobs.failed(t).tap { |f| f['type'] = t } }
+      erb :failed, layout: true, locals: {
+        title: 'Failed',
+        failed: client.jobs.failed.keys.map do |t|
+          client.jobs.failed(t).tap { |f| f['type'] = t }
+        end
       }
     end
 
     get '/failed/:type/?' do
-      erb :failed_type, :layout => true, :locals => {
-        :title  => 'Failed | ' + params[:type],
-        :type   => params[:type],
-        :failed => paginated(client.jobs, :failed, params[:type])
+      erb :failed_type, layout: true, locals: {
+        title: 'Failed | ' + params[:type],
+        type: params[:type],
+        failed: paginated(client.jobs, :failed, params[:type])
+      }
+    end
+
+    get '/completed/?' do
+      completed = paginated(client.jobs, :complete)
+      erb :completed, layout: true, locals: {
+        title: 'Completed',
+        jobs: completed.map { |jid| client.jobs[jid] }
       }
     end
 
     get '/track/?' do
-      erb :track, :layout => true, :locals => {
-        :title   => 'Track'
+      erb :track, layout: true, locals: {
+        title: 'Track'
       }
     end
 
     get '/jobs/:jid' do
-      erb :job, :layout => true, :locals => {
-        :title => "Job | #{params[:jid]}",
-        :jid   => params[:jid],
-        :job   => client.jobs[params[:jid]]
+      erb :job, layout: true, locals: {
+        title: "Job | #{params[:jid]}",
+        jid: params[:jid],
+        job: client.jobs[params[:jid]]
       }
     end
 
     get '/workers/?' do
-      erb :workers, :layout => true, :locals => {
-        :title   => 'Workers'
+      erb :workers, layout: true, locals: {
+        title: 'Workers'
       }
     end
 
     get '/workers/:worker' do
-      erb :worker, :layout => true, :locals => {
-        :title  => 'Worker | ' + params[:worker],
-        :worker => client.workers[params[:worker]].tap { |w|
+      erb :worker, layout: true, locals: {
+        title: 'Worker | ' + params[:worker],
+        worker: client.workers[params[:worker]].tap do |w|
           w['jobs']    = w['jobs'].map { |j| client.jobs[j] }
           w['stalled'] = w['stalled'].map { |j| client.jobs[j] }
           w['name']    = params[:worker]
-        }
+        end
       }
     end
 
     get '/tag/?' do
       jobs = paginated(client.jobs, :tagged, params[:tag])
-      erb :tag, :layout => true, :locals => {
-        :title => "Tag | #{params[:tag]}",
-        :tag   => params[:tag],
-        :jobs  => jobs['jobs'].map { |jid| client.jobs[jid] },
-        :total => jobs['total']
+      erb :tag, layout: true, locals: {
+        title: "Tag | #{params[:tag]}",
+        tag: params[:tag],
+        jobs: jobs['jobs'].map { |jid| client.jobs[jid] },
+        total: jobs['total']
       }
     end
 
     get '/config/?' do
-      erb :config, :layout => true, :locals => {
-        :title   => 'Config',
-        :options => client.config.all
+      erb :config, layout: true, locals: {
+        title: 'Config',
+        options: client.config.all
       }
     end
 
     get '/about/?' do
-      erb :about, :layout => true, :locals => {
-        :title   => 'About'
+      erb :about, layout: true, locals: {
+        title: 'About'
       }
     end
 
     # These are the bits where we accept AJAX requests
-    post "/track/?" do
+    post '/track/?' do
       # Expects a JSON-encoded hash with a job id, and optionally some tags
       data = JSON.parse(request.body.read)
-      job = client.jobs[data["id"]]
-      if not job.nil?
-        data.fetch("tags", false) ? job.track(*data["tags"]) : job.track()
+      job = client.jobs[data['id']]
+      if !job.nil?
+        data.fetch('tags', false) ? job.track(*data['tags']) : job.track
         if request.xhr?
-          json({ :tracked => [job.jid] })
+          json({ tracked: [job.jid] })
         else
           redirect to('/track')
         end
       else
         if request.xhr?
-          json({ :tracked => [] })
+          json({ tracked: [] })
         else
           redirect to(request.referrer)
         end
       end
     end
 
-    post "/untrack/?" do
+    post '/untrack/?' do
       # Expects a JSON-encoded array of job ids to stop tracking
-      jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }.select { |j| not j.nil? }
+      jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
+      jobs.compact!
       # Go ahead and cancel all the jobs!
       jobs.each do |job|
-        job.untrack()
+        job.untrack
       end
-      return json({ :untracked => jobs.map { |job| job.jid } })
+      return json({ untracked: jobs.map { |job| job.jid } })
     end
 
-    post "/priority/?" do
+    post '/priority/?' do
       # Expects a JSON-encoded dictionary of jid => priority
       response = Hash.new
       r = JSON.parse(request.body.read)
@@ -314,7 +327,40 @@ module Qless
       return json(response)
     end
 
-    post "/tag/?" do
+    post '/pause/?' do
+      # Expects JSON blob: {'queue': <queue>}
+      r = JSON.parse(request.body.read)
+      if r['queue']
+        @client.queues[r['queue']].pause
+        return json({ queue: 'paused' })
+      else
+        raise 'No queue provided'
+      end
+    end
+
+    post '/unpause/?' do
+      # Expects JSON blob: {'queue': <queue>}
+      r = JSON.parse(request.body.read)
+      if r['queue']
+        @client.queues[r['queue']].unpause
+        return json({ queue: 'unpaused' })
+      else
+        raise 'No queue provided'
+      end
+    end
+
+    post '/timeout/?' do
+      # Expects JSON blob: {'jid': <jid>}
+      r = JSON.parse(request.body.read)
+      if r['jid']
+        @client.jobs[r['jid']].timeout
+        return json({ jid: r['jid'] })
+      else
+        raise 'No jid provided'
+      end
+    end
+
+    post '/tag/?' do
       # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
       response = Hash.new
       JSON.parse(request.body.read).each_pair do |jid, tags|
@@ -328,7 +374,7 @@ module Qless
       return json(response)
     end
 
-    post "/untag/?" do
+    post '/untag/?' do
       # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
       response = Hash.new
       JSON.parse(request.body.read).each_pair do |jid, tags|
@@ -342,99 +388,100 @@ module Qless
       return json(response)
     end
 
-    post "/move/?" do
+    post '/move/?' do
       # Expects a JSON-encoded hash of id: jid, and queue: queue_name
       data = JSON.parse(request.body.read)
-      if data["id"].nil? or data["queue"].nil?
-        halt 400, "Need id and queue arguments"
+      if data['id'].nil? || data['queue'].nil?
+        halt 400, 'Need id and queue arguments'
       else
-        job = client.jobs[data["id"]]
+        job = client.jobs[data['id']]
         if job.nil?
-          halt 404, "Could not find job"
+          halt 404, 'Could not find job'
         else
-          job.move(data["queue"])
-          return json({ :id => data["id"], :queue => data["queue"]})
+          job.requeue(data['queue'])
+          return json({ id: data['id'], queue: data['queue'] })
         end
       end
     end
 
-    post "/undepend/?" do
+    post '/undepend/?' do
       # Expects a JSON-encoded hash of id: jid, and queue: queue_name
       data = JSON.parse(request.body.read)
-      if data["id"].nil?
-        halt 400, "Need id"
+      if data['id'].nil?
+        halt 400, 'Need id'
       else
-        job = client.jobs[data["id"]]
+        job = client.jobs[data['id']]
         if job.nil?
-          halt 404, "Could not find job"
+          halt 404, 'Could not find job'
         else
           job.undepend(data['dependency'])
-          return json({:id => data["id"]})
+          return json({ id: data['id'] })
         end
       end
     end
 
-    post "/retry/?" do
+    post '/retry/?' do
       # Expects a JSON-encoded hash of id: jid, and queue: queue_name
       data = JSON.parse(request.body.read)
-      if data["id"].nil?
-        halt 400, "Need id"
+      if data['id'].nil?
+        halt 400, 'Need id'
       else
-        job = client.jobs[data["id"]]
+        job = client.jobs[data['id']]
         if job.nil?
-          halt 404, "Could not find job"
+          halt 404, 'Could not find job'
         else
-          queue = job.raw_queue_history[-1]["q"]
-          job.move(queue)
-          return json({ :id => data["id"], :queue => queue})
+          job.requeue(job.queue_name)
+          return json({ id: data['id'], queue: job.queue_name })
         end
       end
     end
 
     # Retry all the failures of a particular type
-    post "/retryall/?" do
+    post '/retryall/?' do
       # Expects a JSON-encoded hash of type: failure-type
       data = JSON.parse(request.body.read)
-      if data["type"].nil?
-        halt 400, "Neet type"
+      if data['type'].nil?
+        halt 400, 'Neet type'
       else
-        return json(client.jobs.failed(data["type"], 0, 500)['jobs'].map do |job|
-          queue = job.raw_queue_history[-1]["q"]
-          job.move(queue)
-          { :id => job.jid, :queue => queue}
-        end)
+        jobs = client.jobs.failed(data['type'], 0, 500)['jobs']
+        results = jobs.map do |job|
+          job.requeue(job.queue_name)
+          { id: job.jid, queue: job.queue_name }
+        end
+        return json(results)
       end
     end
 
-    post "/cancel/?" do
+    post '/cancel/?' do
       # Expects a JSON-encoded array of job ids to cancel
-      jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }.select { |j| not j.nil? }
+      jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
+      jobs.compact!
       # Go ahead and cancel all the jobs!
       jobs.each do |job|
-        job.cancel()
+        job.cancel
       end
 
       if request.xhr?
-        return json({ :canceled => jobs.map { |job| job.jid } })
+        return json({ canceled: jobs.map { |job| job.jid } })
       else
         redirect to(request.referrer)
       end
     end
 
-    post "/cancelall/?" do
+    post '/cancelall/?' do
       # Expects a JSON-encoded hash of type: failure-type
       data = JSON.parse(request.body.read)
-      if data["type"].nil?
-        halt 400, "Neet type"
+      if data['type'].nil?
+        halt 400, 'Neet type'
       else
-        return json(client.jobs.failed(data["type"])['jobs'].map do |job|
-          job.cancel()
-          { :id => job.jid }
+        return json(client.jobs.failed(data['type'])['jobs'].map do |job|
+          job.cancel
+          { id: job.jid }
         end)
       end
     end
 
     # start the server if ruby file executed directly
-    run! if app_file == $0
+    run! if app_file == $PROGRAM_NAME
   end
 end