resque-cleanerer 0.5.0

data/lib/resque_cleaner/server/views/cleaner_exec.erb

@@ -0,0 +1,8 @@
+<%= erb File.read(ResqueCleaner::Server.erb_path("_cleaner_styles.erb")) %>
+
+<div class="cleaner">
+  <p class="message">Processed <%= @count %> jobs.</p>
+  <p class="back_to_list">
+  <a href="<%=@list_url%>">Back to List</a>
+  </p>
+</div>

data/lib/resque_cleaner/server/views/cleaner_list.erb

@@ -0,0 +1,179 @@
+<%= erb File.read(ResqueCleaner::Server.erb_path("_cleaner_styles.erb")) %>
+
+<!-- Many code was copied from failed.erb of the original resque -->
+<div class="cleaner">
+  <div class="title clearfix">
+    <h1>Failed Jobs</h1>
+  </div>
+
+  <div class="clearfix">
+    <div class="control_panel sub_header">
+      <form method="get">
+        <span class="class_filter">
+          Class: <%= class_filter("filter_class","c",@klasses,@klass)%>
+        </span>
+        <span class="exception_filter">
+          Exception: <%= exception_filter("filter_class","ex",@exceptions,@exception)%>
+        </span>
+        <span class="time_filter">
+          From: <%= time_filter("filter_from","f",@from)%>
+        </span>
+        <span class="time_filter">
+          To: <%= time_filter("filter_to","t",@to)%>
+        </span>
+        <span class="regex_filter">
+          Regex: <%= text_filter("filter_regex", "regex", Rack::Utils.escape_html(@regex))%>
+        </span>
+        <input type="submit" value="Filter" />
+      </form>
+    </div>
+  </div>
+
+  <% if @count > 0 %>
+    <div class="clearfix">
+      <div class="control_panel sub_header">
+        <form method="post" id="exec" action="cleaner_exec">
+          <input type="hidden" name="c" value="<%=@klass%>" />
+          <input type="hidden" name="f" value="<%=@from%>" />
+          <input type="hidden" name="t" value="<%=@to%>" />
+          <input type="hidden" name="p" value="<%=@paginate.page%>" />
+          <input type="hidden" name="ex" value="<%=@exception%>" />
+          <input type="hidden" name="regex" value="<%=Rack::Utils.escape_html(@regex)%>" />
+          <select id="form_action" name="action">
+            <option id="default_option" value="" selected="selected">-- Select Action --</option>
+            <option value="clear">Clear</option>
+            <option value="retry_and_clear">Retry and Clear</option>
+            <option value="retry">Retry</option>
+          </select>
+          <a href="#" id="select_all">select all</a>
+          <a href="#" id="reset_all">reset</a>
+
+          <% if @paginate.max_page > 1 %>
+            <input type="checkbox" name="select_all_pages" value="1" id="select_all_pages" />
+            <label for="select_all_pages">Select all <%=@count%> jobs</label>
+          <% end %>
+          <input type="hidden" name="sha1" id="sha1_list" />
+
+        </form>
+      </div>
+    </div>
+  <% end %>
+
+  <% start = 0 %>
+  <% failed = @paginate.paginated_jobs%>
+  <% index = 0 %>
+  <% date_format = "%Y/%m/%d %T %z" %>
+
+  <% if @paginate.max_page > 0 %>
+    <%= erb File.read(ResqueCleaner::Server.erb_path("_paginate.erb")) %>
+    <ul class='failed'>
+      <%for job in failed%>
+        <% index += 1 %>
+        <li>
+          <dl>
+            <% if job.nil? %>
+            <dt>Error</dt>
+            <dd>Job <%= index%> could not be parsed; perhaps it contains invalid JSON?</dd>
+            <% else %>
+            <dt>
+              <input type="checkbox" id="<%=Digest::SHA1.hexdigest job.to_json %>" />
+            </dt>
+            <dd>&nbsp;</dd>
+            <dt>Worker</dt>
+            <dd>
+              <a href="<%= u(:workers, job['worker']) %>"><%= job['worker'].split(':')[0...2].join(':') %></a> on <b class='queue-tag'><%= job['queue'] %></b > at <b><span class="time"><%= Time.parse(job['failed_at']).strftime(date_format) %></span></b>
+              <% if job['retried_at'] %>
+                <div class='retried'>
+                  Retried <b><span class="time"><%= job['retried_at'] %></span></b>
+                </div>
+              <% end %>
+            </dd>
+            <dt>Class</dt>
+            <dd><code><%= job.klass_name %></code></dd>
+            <dt>Arguments</dt>
+            <dd><pre><%=h job['payload'] ? show_job_args(job['payload']['args']) : 'nil' %></pre></dd>
+            <dt>Exception</dt>
+            <dd><code><%= job['exception'] %></code></dd>
+            <dt>Error</dt>
+            <dd class='error'>
+              <% if job['backtrace'] %>
+                <a href="#" class="backtrace"><%= h(job['error']) %></a>
+                <pre style='display:none'><%=h job['backtrace'].join("\n") %></pre>
+              <% else %>
+                <%=h job['error'] %>
+              <% end %>
+            </dd>
+            <% end %>
+          </dl>
+          <div class='r'>
+          </div>
+        </li>
+      <%end%>
+    </ul>
+    <%= erb File.read(ResqueCleaner::Server.erb_path("_paginate.erb")) %>
+  <% else %>
+    Clean!
+  <% end %>
+</div>
+
+<script>
+  $(document).ready(function(){
+    $('#select_all_pages').click(function() {
+      updateCheckboxStaus();
+    });
+
+    $('#select_all').click(function() {
+      if (!$(this).hasClass('disabled')) {
+        $('.failed input').attr('checked','checked');
+      }
+      return false;
+    });
+    $('#reset_all').click(function() {
+      if (!$(this).hasClass('disabled')) {
+        $('.failed input').removeAttr('checked');
+      }
+      return false;
+    });
+
+    $('#form_action').change( function() {
+      if ($('#form_action option:selected').val()=='') return;
+
+      if ($('#select_all_pages:checked, .failed input:checked').length==0) {
+        alert('Please select jobs.');
+        $('#default_option').attr('selected','selected');
+        return false;
+      }
+
+      if (!confirm('Do you really want to proceed?')) {
+        $('#default_option').attr('selected','selected');
+        return false;
+      }
+
+      if ($('#select_all_pages:checked').length==0) {
+        setSha1();
+      }
+      $('#exec').submit();
+    });
+  });
+
+  function setSha1() {
+    var sha1 = "";
+    $('.failed input:checked').each( function() {
+      if (sha1.length>0) sha1 += ",";
+      sha1 += $(this).attr("id");
+    });
+
+    $('#sha1_list').val(sha1);
+  }
+
+  function updateCheckboxStaus() {
+    if ($('#select_all_pages:checked').length==1) {
+      $('#exec a').addClass('disabled');
+      $('.failed input').attr('disabled','disabled');
+    } else {
+      $('#exec a').removeClass('disabled');
+      $('.failed input').removeAttr('disabled');
+    }
+  };
+
+</script>

data/lib/resque_cleaner/server.rb

@@ -0,0 +1,258 @@
+require 'yaml'
+
+# Extends Resque Web Based UI.
+# Structure has been borrowed from ResqueScheduler.
+module ResqueCleaner
+  module Server
+
+    def self.erb_path(filename)
+      File.join(File.dirname(__FILE__), 'server', 'views', filename)
+    end
+
+    # Pagination helper for list page.
+    class Paginate
+      attr_accessor :page_size, :page, :jobs, :url
+      def initialize(jobs, url, page=1, page_size=20)
+        @jobs = jobs
+        @url = url
+        @page = (!page || page < 1) ? 1 : page
+        @page_size = 20
+      end
+
+      def first_index
+        @page_size * (@page-1)
+      end
+
+      def last_index
+        last = first_index + @page_size - 1
+        last > @jobs.size-1 ? @jobs.size-1 : last
+      end
+
+      def paginated_jobs
+        @jobs[first_index,@page_size]
+      end
+
+      def first_page?
+        @page <= 1
+      end
+
+      def last_page?
+        @page >= max_page
+      end
+
+      def page_url(page)
+        u = @url
+        u += @url.include?("?") ? "&" : "?"
+        if page.is_a?(Symbol)
+          page = @page - 1 if page==:prev
+          page = @page + 1 if page==:next
+        end
+        u += "p=#{page}"
+      end
+
+      def total_size
+        @jobs.size
+      end
+
+      def max_page
+        ((total_size-1) / @page_size) + 1
+      end
+    end
+
+    def self.included(base)
+
+      base.class_eval do
+        helpers do
+          def time_filter(id, name, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            [1, 3, 6, 12, 24].each do |h|
+              selected = h.to_s == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{h}\">#{h} #{h==1 ? "hour" : "hours"} ago</option>"
+            end
+            [3, 7, 14, 28].each do |d|
+              selected = (d*24).to_s == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{d*24}\">#{d} days ago</option>"
+            end
+            html += "</select>"
+          end
+
+          def class_filter(id, name, klasses, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            klasses.each do |k|
+              selected = k == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{k}\">#{k}</option>"
+            end
+            html += "</select>"
+          end
+
+          def exception_filter(id, name, exceptions, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            exceptions.each do |ex|
+              selected = ex == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{ex}\">#{ex}</option>"
+            end
+            html += "</select>"
+          end
+
+          def show_job_args(args)
+            Array(args).map { |a| a.inspect }.join("\n")
+          end
+
+          def text_filter(id, name, value)
+            html = "<input id=\"#{id}\"  type=\"text\" name=\"#{name}\" value=\"#{value}\">"
+            html += "</input>"
+          end
+        end
+
+        mime_type :json, 'application/json'
+
+        get "/cleaner" do
+          load_library
+          load_cleaner_filter
+
+          @jobs = cleaner.select
+          @stats = { :klass => {}, :exception => {} }
+          @total = Hash.new(0)
+          @jobs.each do |job|
+            exception = job["exception"] || 'UNKNOWN'
+            failed_at = Time.parse job["failed_at"]
+            @stats[:klass][job.klass_name] ||= Hash.new(0)
+            @stats[:exception][exception] ||= Hash.new(0)
+
+            [
+              @stats[:klass][job.klass_name],
+              @stats[:exception][exception],
+              @total
+            ].each do |stat|
+              stat[:total] += 1
+              stat[:h1] += 1 if failed_at >= hours_ago(1)
+              stat[:h3] += 1 if failed_at >= hours_ago(3)
+              stat[:d1] += 1 if failed_at >= hours_ago(24)
+              stat[:d3] += 1 if failed_at >= hours_ago(24*3)
+              stat[:d7] += 1 if failed_at >= hours_ago(24*7)
+            end
+          end
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner.erb'))
+        end
+
+        get "/cleaner_list" do
+          load_library
+          load_cleaner_filter
+          build_urls
+
+          block = filter_block
+
+          @failed = cleaner.select(&block).reverse
+
+          @paginate = Paginate.new(@failed, @list_url, params[:p].to_i)
+
+          @klasses = cleaner.stats_by_class.keys
+          @exceptions = cleaner.stats_by_exception.keys
+          @count = cleaner.select(&block).size
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner_list.erb'))
+        end
+
+        post "/cleaner_exec" do
+          load_library
+          load_cleaner_filter
+          build_urls
+
+          if params[:select_all_pages]!="1"
+            @sha1 = {}
+            params[:sha1].split(",").each {|s| @sha1[s] = true }
+          end
+
+          block = filter_block
+
+          @count =
+            case params[:action]
+            when "clear" then cleaner.clear(&block)
+            when "retry_and_clear" then cleaner.requeue(true,&block)
+            when "retry" then cleaner.requeue(false,{},&block)
+            end
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner_exec.erb'))
+        end
+
+        get "/cleaner_dump" do
+          load_library
+          load_cleaner_filter
+
+          block = filter_block
+
+          content_type :json
+          JSON.pretty_generate(cleaner.select(&block))
+        end
+
+        post "/cleaner_stale" do
+          load_library
+          cleaner.clear_stale
+          redirect url_path(:cleaner)
+        end
+      end
+
+    end
+
+    def cleaner
+      @cleaner ||= Resque::Plugins::ResqueCleaner.new
+      @cleaner.print_message = false
+      @cleaner
+    end
+
+    def load_library
+      require 'digest/sha1'
+      begin
+        require 'yajl/json_gem' unless [].respond_to?(:to_json)
+      rescue Exception
+        require 'json'
+      end
+    end
+
+    def load_cleaner_filter
+      @from = params[:f]=="" ? nil : params[:f]
+      @to = params[:t]=="" ? nil : params[:t]
+      @klass = params[:c]=="" ? nil : params[:c]
+      @exception = params[:ex]=="" ? nil : params[:ex]
+      @regex = params[:regex]=="" ? nil : params[:regex]
+    end
+
+    def build_urls
+      params = {
+        c: @klass,
+        ex: @exception,
+        f: @from,
+        t: @to,
+        regex: @regex
+      }.map {|key,value| "#{key}=#{CGI.escape(value.to_s)}"}.join("&")
+
+      @list_url = "cleaner_list?#{params}"
+      @dump_url = "cleaner_dump?#{params}"
+    end
+
+    def filter_block
+      block = lambda{|j|
+        (!@from || j.after?(hours_ago(@from))) &&
+        (!@to || j.before?(hours_ago(@to))) &&
+        (!@klass || j.klass?(@klass)) &&
+        (!@exception || j.exception?(@exception)) &&
+        (!@sha1 || @sha1[Digest::SHA1.hexdigest(j.to_json)]) &&
+        (!@regex || j.to_s =~ /#{@regex}/)
+      }
+    end
+
+    def hours_ago(h)
+      Time.now - h.to_i*60*60
+    end
+    Resque::Server.tabs << 'Cleaner'
+  end
+end
+
+Resque::Server.class_eval do
+  include ResqueCleaner::Server
+end
+