resque_ui 3.0.0 → 3.1.0

data/Gemfile

@@ -4,3 +4,4 @@ gem 'rails', '>= 3.0.0'
 gem 'redis', '>= 2.1.3'
 gem 'resque', '>= 1.5'
 gem 'resque-status', '>= 0.2.2'
+gem 'resque-cleaner', '>= 0.2.6'

data/Gemfile.lock

@@ -74,6 +74,8 @@ GEM
       redis-namespace (~> 1.0.2)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
+    resque-cleaner (0.2.6)
+      resque (~> 1.0)
     resque-status (0.2.3)
       redisk (>= 0.2.1)
       resque (>= 1.3.1)
@@ -99,4 +101,5 @@ DEPENDENCIES
   rails (>= 3.0.0)
   redis (>= 2.1.3)
   resque (>= 1.5)
+  resque-cleaner (>= 0.2.6)
   resque-status (>= 0.2.2)

data/History.txt

@@ -1,3 +1,7 @@
+== 3.1.0 2011-09-22
+
+*  Added Resque Cleaner to manage the failed queue.  Removed the Failed tab.
+
 == 3.0.0 2011-09-16
 
 *  converted plugging to a gem/engine for Rails 3

data/README.markdown

@@ -32,9 +32,11 @@ This engine now requires redis 2.1.3 or greater for the expiration of keys to wo
 
 This engine requires the resque 1.5 or higher gem.
 
-This engine now requires the resque-status 0.2.2 or higher gem as well.
+This engine now requires the resque-status 0.2.2 or higher gem.
 
-These gems will all be installed for automatically when you install resque_ui.
+This engine now requires the resque-cleaner 0.2.6 or higher gem.
+
+These gems will all be installed for you automatically when you install resque_ui.
 
 
 Times Fixed
@@ -67,9 +69,10 @@ This is really handy for those long running jobs to give you assurance the the j
 Restart Failed Jobs
 -------------------
 
-Added the ability to restart failed jobs, and clear individual failed jobs from the list
+Added the Resque Cleaner gem to manage the failed queue.  Have complete control over the failed queue now with the
+Cleaner tab by querying and restarting failed jobs.
 
-![Restart Failed Jobs](http://img.skitch.com/20100308-mbh5s8pcw5n4ei2hrseiqtshys.png)
+See what all you can do on the github page:  https://github.com/ono/resque-cleaner
 
 Remove Items from the Queue
 ---------------------------
@@ -165,7 +168,7 @@ deployed servers.
 
 The controller calls cap tasks to manage the workers.  To include the recipes in your application, add this line to your deploy.rb file:
 
-    require File.dirname(__FILE__) + '/../vendor/engines/resque_ui/lib/resque_ui/cap_recipes'
+    require 'resque_ui/cap_recipes'
 
 You will also need to make sure you have your rake path set in the deploy.rb file.
 
@@ -272,6 +275,7 @@ time into resolving it.
 Copyright (c) 2009 Chris Wanstrath
 Copyright (c) 2010 Ben VandenBos
 Copyright (c) 2010 Aaron Quint
+Copyright (c) 2011 Tatsuya Ono
 Copyright (c) 2010 Kevin Tyll, released under the MIT license
 
 Thanks to Karl Baum for doing the original heavy lifting for converting this to a rails engine for rails 3.

data/Rakefile

@@ -9,7 +9,7 @@ begin
     gemspec.name = "resque_ui"
     gemspec.author = "Kevin Tyll"
     gemspec.email = "kevintyll@gmail.com"
-    gemspec.homepage = %q{http://kevintyll.git.com/resque_ui}
+    gemspec.homepage = %q{https://github.com/kevintyll/resque_ui}
     gemspec.summary = "A Rails engine port of the Sinatra app that is included in Chris Wanstrath's resque gem."
     gemspec.description = "A Rails UI for Resque for managing workers, failures and schedules."
   end

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
 :major: 3
-:minor: 0
+:minor: 1
 :patch: 0
 :build: 

data/app/assets/stylesheets/resque_cleaner/cleaner.css

@@ -0,0 +1,62 @@
+#main .cleaner .title h1 { float: left }
+
+#main .cleaner .sub_header {color: #888; font-size: 80%; font-weight: bold}
+
+#main .cleaner .control_panel {
+  margin: 5px 0 10px; padding: 5px; background: #eee; display: inline-block;
+  -webkit-border-radius:5px; border:1px solid #ccc;
+}
+
+#main .cleaner #exec { color: black; }
+#main .cleaner #exec select { margin-right: 20px }
+#main .cleaner #exec a { margin-right: 5px}
+#main .cleaner #exec a.disabled { color: #aaa}
+
+#main .cleaner table.class_list { width: auto }
+#main .cleaner td.number { text-align: right }
+#main .cleaner tr.total { border-top: 3px double #ccc}
+
+#main .cleaner form { float: none; margin: 0}
+
+#main .cleaner .warning { margin: 30px 0; padding: 10px; background: #fdd; display: inline-block; }
+
+#main .clearfix:after {
+  content: ".";
+  display: block;
+  height: 0;
+  clear: both;
+  visibility: hidden;
+}
+#main .cleaner ul.failed {
+  margin-bottom: 10px
+}
+
+#main .cleaner .list_info{
+  color: #999; text-align: center;
+}
+#main .cleaner .list_info a{
+  color: #666
+}
+#main .cleaner .list_info .list_summary{
+  vertical-align: baseline;
+  display: inline-block;
+  margin: 0; padding:0;
+}
+#main .cleaner ul.pagination {
+  display: inline-block;
+  vertical-align: baseline;
+  margin:0; padding:0;
+}
+#main .cleaner ul.pagination li {
+  list-style:none; float: left; padding: 0 3px;
+}
+#main .cleaner ul.pagination li.current {
+  color: #333; font-weight: bold;
+}
+#main .cleaner ul.pagination li.summary{
+  padding-right: 10px
+}
+#main .cleaner div.dump{
+  float: right
+}
+

data/app/controllers/resque_controller.rb

@@ -1,5 +1,7 @@
 require 'resque'
 require 'resque/version'
+require 'digest/sha1'
+
 
 class ResqueController < ApplicationController
   unloadable(self) #needed to prevent errors with authenticated system in dev env.
@@ -8,10 +10,12 @@ class ResqueController < ApplicationController
 
   before_filter :check_connection
 
+  before_filter :get_cleaner, :only => [:cleaner, :cleaner_exec, :cleaner_list, :cleaner_stale, :cleaner_dump]
+
   verify :method => :post, :only => [:clear_failures, :clear_failure, :requeue_failure, :stop_worker, :restart_worker,
                                      :start_worker, :schedule_requeue, :remove_from_schedule, :add_scheduled_job,
                                      :start_scheduler, :stop_scheduler, :requeue_failures_in_class,
-                                     :kill, :clear_statuses],
+                                     :kill, :clear_statuses, :cleaner_exec, :cleaner_stale],
          :render => {:text => "<p>Please use the POST http method to post data to this API.</p>".html_safe}
 
 
@@ -43,36 +47,6 @@ class ResqueController < ApplicationController
     render(:text => (render_to_string(:action => 'statuses.html', :layout => false)))
   end
 
-  def failed
-    if Resque::Failure.url
-      redirect_to Resque::Failure.url
-    end
-  end
-
-  def clear_failures
-    Resque::Failure.clear
-    redirect_to(:action => 'failed')
-  end
-
-  def clear_failure
-    remove_failure_from_list(Resque.decode(params[:payload]))
-    redirect_to(:action => 'failed')
-  end
-
-  def requeue_failure
-    #first clear the job we're restarting from the failure list.
-    payload = Resque.decode(params["payload"])
-    remove_failure_from_list(payload)
-    args = payload["args"]
-    Resque.enqueue(eval(payload["class"]), *args)
-    redirect_to(:action => 'failed')
-  end
-
-  def requeue_failures_in_class
-    Resque::Failure.requeue_class(params['class'])
-    redirect_to(:action => 'failed')
-  end
-
   def remove_job
     Resque.dequeue(params['class'].constantize, *Resque.decode(params['args']))
     redirect_to request.referrer
@@ -118,6 +92,8 @@ class ResqueController < ApplicationController
     end
   end
 
+  # resque-scheduler actions
+
   def schedule
     @farm_status = ResqueScheduler.farm_status
   end
@@ -176,6 +152,8 @@ class ResqueController < ApplicationController
     redirect_to(:action => 'schedule')
   end
 
+  # resque-status actions
+
   def statuses
     @start = params[:start].to_i
     @end = @start + (params[:per_page] || 20)
@@ -206,6 +184,82 @@ class ResqueController < ApplicationController
     redirect_to(:action => 'statuses')
   end
 
+  def cleaner
+    load_cleaner_filter
+
+    @jobs = @cleaner.select
+    @stats, @total = {}, {"total" => 0, "1h" => 0, "3h" => 0, "1d" => 0, "3d" => 0, "7d" => 0}
+    @jobs.each do |job|
+      klass = job["payload"]["class"]
+      failed_at = Time.parse job["failed_at"]
+
+      @stats[klass] ||= {"total" => 0, "1h" => 0, "3h" => 0, "1d" => 0, "3d" => 0, "7d" => 0}
+      items = [@stats[klass], @total]
+
+      items.each { |a| a["total"] += 1 }
+      items.each { |a| a["1h"] += 1 } if failed_at >= hours_ago(1)
+      items.each { |a| a["3h"] += 1 } if failed_at >= hours_ago(3)
+      items.each { |a| a["1d"] += 1 } if failed_at >= hours_ago(24)
+      items.each { |a| a["3d"] += 1 } if failed_at >= hours_ago(24*3)
+      items.each { |a| a["7d"] += 1 } if failed_at >= hours_ago(24*7)
+    end
+  end
+
+  def cleaner_list
+    load_cleaner_filter
+
+    block = filter_block
+
+    @failed = @cleaner.select(&block).reverse
+
+    url = "cleaner_list?c=#{@klass}&ex=#{@exception}&f=#{@from}&t=#{@to}"
+    @dump_url = "cleaner_dump?c=#{@klass}&ex=#{@exception}&f=#{@from}&t=#{@to}"
+    @paginate = Paginate.new(@failed, url, params[:p].to_i)
+
+    @klasses = @cleaner.stats_by_class.keys
+    @exceptions = @cleaner.stats_by_exception.keys
+    @count = @cleaner.select(&block).size
+  end
+
+  def cleaner_exec
+    load_cleaner_filter
+
+    if params[:select_all_pages]!="1"
+      @sha1 = {}
+      params[:sha1].split(",").each { |s| @sha1[s] = true }
+    end
+
+    block = filter_block
+
+    @count =
+        case params[:form_action]
+          when "clear" then
+            @cleaner.clear(&block)
+          when "retry_and_clear" then
+            @cleaner.requeue(true, &block)
+          when "retry" then
+            @cleaner.requeue(false, {}, &block)
+        end
+
+    @link_url = "cleaner_list?c=#{@klass}&ex=#{@exception}&f=#{@from}&t=#{@to}"
+  end
+
+  def cleaner_dump
+    load_cleaner_filter
+
+    block = filter_block
+    failures = @cleaner.select(&block)
+    # pretty generate throws an error with the json gem on jruby
+    output = JSON.pretty_generate(failures) rescue failures.to_json
+    render :json => output
+  end
+
+  def cleaner_stale
+    @cleaner.clear_stale
+    redirect_to :action => "cleaner"
+  end
+
+
   private
 
   def check_connection
@@ -215,22 +269,92 @@ class ResqueController < ApplicationController
     false
   end
 
-  def remove_failure_from_list(payload)
-    count = Resque::Failure.count - 1
-    loop do
-      f = Resque::Failure.all(count, 1)
-      if f && f['payload'] == payload
-        Resque.redis.lrem(:failed, 0, f.to_json)
-      end
-      count = count - 1
-      break if count < 0
-    end
-  end
-
   def find_worker(worker)
     first_part, *rest = worker.split(':')
     first_part.gsub!(/_/, '.')
     Resque::Worker.find("#{first_part}:#{rest.join(':')}")
   end
 
+  # resque-cleaner methods
+
+  def get_cleaner
+    @cleaner ||= Resque::Plugins::ResqueCleaner.new
+    @cleaner.print_message = false
+    @cleaner
+  end
+
+  def load_cleaner_filter
+    @from = params[:f]=="" ? nil : params[:f]
+    @to = params[:t]=="" ? nil : params[:t]
+    @klass = params[:c]=="" ? nil : params[:c]
+    @exception = params[:ex]=="" ? nil : params[:ex]
+  end
+
+  def filter_block
+    block = lambda { |j|
+      (!@from || j.after?(hours_ago(@from))) &&
+          (!@to || j.before?(hours_ago(@to))) &&
+          (!@klass || j.klass?(@klass)) &&
+          (!@exception || j.exception?(@exception)) &&
+          (!@sha1 || @sha1[Digest::SHA1.hexdigest(j.to_json)])
+    }
+  end
+
+  def hours_ago(h)
+    Time.now - h.to_i*60*60
+  end
+
+end
+
+# Paginate class used by resque-cleaner
+
+#Pagination helper for list page.
+class Paginate
+  attr_accessor :page_size, :page, :jobs, :url
+
+  def initialize(jobs, url, page=1, page_size=20)
+    @jobs = jobs
+    @url = url
+    @page = (!page || page < 1) ? 1 : page
+    @page_size = 20
+  end
+
+  def first_index
+    @page_size * (@page-1)
+  end
+
+  def last_index
+    last = first_index + @page_size - 1
+    last > @jobs.size-1 ? @jobs.size-1 : last
+  end
+
+  def paginated_jobs
+    @jobs[first_index, @page_size]
+  end
+
+  def first_page?
+    @page <= 1
+  end
+
+  def last_page?
+    @page >= max_page
+  end
+
+  def page_url(page)
+    u = @url
+    u += @url.include?("?") ? "&" : "?"
+    if page.is_a?(Symbol)
+      page = @page - 1 if page==:prev
+      page = @page + 1 if page==:next
+    end
+    u += "p=#{page}"
+  end
+
+  def total_size
+    @jobs.size
+  end
+
+  def max_page
+    ((total_size-1) / @page_size) + 1
+  end
 end

data/app/helpers/resque_helper.rb

@@ -1,11 +1,4 @@
 module ResqueHelper
-  #include Rack::Utils
-  #alias_method :h, :escape_html
-
-
-  def classes_in_failure
-    Resque.list_range(:failed,0,1000).collect{|job| job['payload']['class']}.uniq
-  end
 
   def flash_helper
     [:notice, :warning, :message, :error].collect do |key|
@@ -18,16 +11,17 @@ module ResqueHelper
   end
 
   def current_section
-    request.path_info.sub('/','').split('/')[1].downcase
+    request.path_info.sub('/', '').split('/')[1].downcase
   end
 
   def current_page
-    url request.path_info.sub('/','').downcase
+    url request.path_info.sub('/', '').downcase
   end
 
   def url(*path_parts)
-    [ path_prefix, path_parts ].join("/").squeeze('/')
+    [path_prefix, path_parts].join("/").squeeze('/')
   end
+
   alias_method :u, :url
 
   def path_prefix
@@ -45,33 +39,33 @@ module ResqueHelper
 
   def find_worker(worker)
     first_part, *rest = worker.split(':')
-    first_part.gsub!(/_/,'.')
+    first_part.gsub!(/_/, '.')
     Resque::Worker.find("#{first_part}:#{rest.join(':')}")
   end
 
   def redis_get_size(key)
     case Resque.redis.type(key)
-    when 'none'
-      []
-    when 'list'
-      Resque.redis.llen(key)
-    when 'set'
-      Resque.redis.scard(key)
-    when 'string'
-      Resque.redis.get(key).length
+      when 'none'
+        []
+      when 'list'
+        Resque.redis.llen(key)
+      when 'set'
+        Resque.redis.scard(key)
+      when 'string'
+        Resque.redis.get(key).length
     end
   end
 
   def redis_get_value_as_array(key)
     case Resque.redis.type(key)
-    when 'none'
-      []
-    when 'list'
-      Resque.list_range(key, 0, 20)
-    when 'set'
-      Resque.redis.smembers(key)
-    when 'string'
-      [Resque.redis.get(key)]
+      when 'none'
+        []
+      when 'list'
+        Resque.list_range(key, 0, 20)
+      when 'set'
+        Resque.redis.smembers(key)
+      when 'string'
+        [Resque.redis.get(key)]
     end
   end
 
@@ -104,4 +98,43 @@ module ResqueHelper
   def resque
     Resque
   end
+
+  # resque-cleaner helpers
+
+  def time_filter(id, name, value)
+    html = "<select id=\"#{id}\" name=\"#{name}\">"
+    html += "<option value=\"\">-</option>"
+    [1, 3, 6, 12, 24].each do |h|
+      selected = h.to_s == value ? 'selected="selected"' : ''
+      html += "<option #{selected} value=\"#{h}\">#{h} #{h==1 ? "hour" : "hours"} ago</option>"
+    end
+    [3, 7, 14, 28].each do |d|
+      selected = (d*24).to_s == value ? 'selected="selected"' : ''
+      html += "<option #{selected} value=\"#{d*24}\">#{d} days ago</option>"
+    end
+    html += "</select>"
+    html.html_safe
+  end
+
+  def class_filter(id, name, klasses, value)
+    html = "<select id=\"#{id}\" name=\"#{name}\">"
+    html += "<option value=\"\">-</option>"
+    klasses.each do |k|
+      selected = k == value ? 'selected="selected"' : ''
+      html += "<option #{selected} value=\"#{k}\">#{k}</option>"
+    end
+    html += "</select>"
+    html.html_safe
+  end
+
+  def exception_filter(id, name, exceptions, value)
+    html = "<select id=\"#{id}\" name=\"#{name}\">"
+    html += "<option value=\"\">-</option>"
+    exceptions.each do |ex|
+      selected = ex == value ? 'selected="selected"' : ''
+      html += "<option #{selected} value=\"#{ex}\">#{ex}</option>"
+    end
+    html += "</select>"
+    html.html_safe
+  end
 end

data/app/views/layouts/resque.html.erb

@@ -6,6 +6,7 @@
     <%= yield :javascript %>
     <%= stylesheet_link_tag "resque/resque_reset", :media=>"screen" %>
     <%= stylesheet_link_tag "resque/resque", :media=>"screen" %>
+    <%= stylesheet_link_tag "resque_cleaner/cleaner", :media=>"screen" %>
     <%= yield :stylesheets %>
   </head>
   <body>

data/app/views/resque/_limiter.html.erb

@@ -0,0 +1,12 @@
+ <div class="warning">
+    <p>
+    There are more than <%= @cleaner.limiter.maximum%> jobs. ResqueCleaner handles only recent <%= @cleaner.limiter.maximum %> jobs. See the Limiter section on <a href="https://github.com/ono/resque-cleaner" target="_blank">README</a> for more detail.
+    </p>
+    <p>
+      <form method="post" action="cleaner_stale">
+        <input type="hidden" name="action" value="clear_stale" />
+        <input type="submit" onclick="return confirm('Are you sure?')" value="Clear all jobs older than <%= @cleaner.limiter.count %>th job."/>
+        (<%= @cleaner.failure.count - @cleaner.limiter.maximum %> jobs will be cleared)
+      </form>
+    </p>
+  </div>

data/app/views/resque/_paginate.html.erb

@@ -0,0 +1,53 @@
+<div class="list_info clearfix">
+
+  <ul class="pagination">
+    <li class="summary">
+      <%= @paginate.first_index + 1%> - <%= @paginate.last_index + 1%> / <%= @paginate.total_size%>
+    </li>
+    <% if @paginate.max_page>1 %>
+      <% if @paginate.first_page? %>
+        <li class="prev off">&lt;&lt;newer</li>
+      <% else %>
+        <li class="prev"><a href="<%= @paginate.page_url(:prev)%>">&lt;&lt;newer</a></li>
+      <% end %>
+
+      <% 1.upto(6<@paginate.max_page ? 6 : @paginate.max_page) do |pg| %>
+        <% if @paginate.page==pg %>
+          <li class="current"><%=pg%></li>
+        <% else %>
+          <li><a href="<%= @paginate.page_url(pg)%>"><%=pg%></a></li>
+        <% end %>
+      <% end %>
+
+      <% if @paginate.max_page > 6 %>
+        <% if @paginate.max_page!=@paginate.page %>
+          <% if @paginate.page>7 %>
+            <li class="off">...</li>
+          <% end %>
+          <% if @paginate.page>6 %>
+            <li class="current"><%=@paginate.page%></li>
+          <% end %>
+        <% end %>
+        <% if @paginate.max_page>7 && @paginate.page+1!=@paginate.max_page %>
+          <li class="off">...</li>
+        <% end %>
+
+        <% if @paginate.max_page==@paginate.page %>
+          <li class="current"><%=@paginate.max_page%></li>
+        <% else %>
+          <li><a href="<%= @paginate.page_url(@paginate.max_page)%>"><%=@paginate.max_page%></a></li>
+        <% end %>
+      <% end %>
+
+      <% if @paginate.last_page? %>
+        <li class="next off">older&gt;&gt;</li>
+      <% else %>
+        <li class="next"><a href="<%= @paginate.page_url(:next)%>">older&gt;&gt;</a></li>
+      <% end %>
+    <% end %>
+
+  </ul>
+    <div class="dump">
+        <a href="<%= @dump_url%>" target="_blank">Dump</a>
+    </div>
+</div>

data/app/views/resque/_queues.html.erb

@@ -15,7 +15,7 @@
     <% for job in (jobs = resque.peek(queue, start, 20)) %>
       <tr>
         <td class='class'><%= job['class'] %></td>
-        <td class='args'><%=h job['args'].inspect %></td>
+        <td class='args'><%= job['args'].inspect.html_safe %></td>
         <td>
           <%= button_to "Remove", {:controller=>'resque', :action=> 'remove_job',:class => job['class'], :args => Resque.encode(job['args'])}, :class => 'clear-failed', :method => :post %>
             </td>
@@ -44,7 +44,7 @@
       </tr>
     <% end %>
     <tr class='failed'>
-      <td class='queue failed'><%= link_to('failed', {:action => "failed"}, :class => 'queue') %></td>
+      <td class='queue failed'><%= link_to('failed', {:action => "cleaner"}, :class => 'queue') %></td>
       <td class='size'><%= Resque::Failure.count %></td>
     </tr>
   </table>

data/app/views/resque/_working.html.erb

@@ -21,7 +21,7 @@
       <td>
         <code><%= data['payload']['class'] %></code>
       </td>
-      <td><%=h data['payload']['args'].inspect %></td>
+      <td><%= data['payload']['args'].inspect.html_safe %></td>
     </tr>
   </table>