misha-resque-cleaner 0.3.0

data/lib/resque-cleaner.rb

@@ -0,0 +1 @@
+require 'resque_cleaner'

data/lib/resque_cleaner.rb

@@ -0,0 +1,307 @@
+require 'time'
+require 'resque'
+require 'resque/server'
+
+module Resque
+  module Plugins
+    # ResqueCleaner class provides useful functionalities to retry or clean
+    # failed jobs. Let's clean up your failed list!
+    class ResqueCleaner
+      # ResqueCleaner fetches all elements from Redis and checks them
+      # by linear when filtering them. Since there is a performance concern,
+      # ResqueCleaner handles only the latest x(default 1000) jobs.
+      #
+      # You can change the value through limiter attribute.
+      # e.g. cleaner.limiter.maximum = 5000
+      attr_reader :limiter
+
+      # Set false if you don't show any message.
+      attr_accessor :print_message
+
+      # Initializes instance
+      def initialize
+        @failure = Resque::Failure.backend
+        @print_message = true
+        @limiter = Limiter.new self
+      end
+
+      # Returns redis instance.
+      def redis
+        Resque.redis
+      end
+
+      # Returns failure backend. Only supports redis backend.
+      def failure
+        @failure
+      end
+
+      # Stats by date.
+      def stats_by_date(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          date = job["failed_at"][0,10]
+          stats[date] ||= 0
+          stats[date] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Stats by class.
+      def stats_by_class(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          klass = job["payload"] && job["payload"]["class"] ? job["payload"]["class"] : "UNKNOWN"
+          stats[klass] ||= 0
+          stats[klass] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Stats by exception.
+      def stats_by_exception(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          exception = job["exception"]
+          stats[exception] ||= 0
+          stats[exception] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Print stats
+      def print_stats(stats)
+        log too_many_message if @limiter.on?
+        stats.keys.sort.each do |k|
+          log "%15s: %4d" % [k,stats[k]]
+        end
+        log "%15s: %4d" % ["total", @limiter.count]
+      end
+
+      # Returns every jobs for which block evaluates to true.
+      def select(&block)
+        jobs = @limiter.jobs
+        block_given? ? @limiter.jobs.select(&block) : jobs
+      end
+      alias :failure_jobs :select
+
+      def select_by_regex(regex)
+        select do |job|
+          job.to_s =~ regex
+        end
+      end
+
+      # Clears every jobs for which block evaluates to true.
+      def clear(&block)
+        cleared = 0
+        @limiter.lock do
+          @limiter.jobs.each_with_index do |job,i|
+            if !block_given? || block.call(job)
+              index = @limiter.start_index + i - cleared
+              # fetches again since you can't ensure that it is always true:
+              # a == endode(decode(a))
+              value = redis.lindex(:failed, index)
+              redis.lrem(:failed, 1, value)
+              cleared += 1
+            end
+          end
+        end
+        cleared
+      end
+
+      # Retries every jobs for which block evaluates to true.
+      def requeue(clear_after_requeue=false, options={}, &block)
+        requeued = 0
+        queue = options["queue"] || options[:queue]
+        @limiter.lock do
+          @limiter.jobs.each_with_index do |job,i|
+            if !block_given? || block.call(job)
+              index = @limiter.start_index + i - requeued
+
+              value = redis.lindex(:failed, index)
+              redis.multi do
+                Job.create(queue||job['queue'], job['payload']['class'], *job['payload']['args'])
+
+                if clear_after_requeue
+                  # remove job
+                  # TODO: should use ltrim. not sure why i used lrem here...
+                  redis.lrem(:failed, 1, value)
+                else
+                  # mark retried
+                  job['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+                  redis.lset(:failed, @limiter.start_index+i, Resque.encode(job))
+                end
+              end
+
+              requeued += 1
+            end
+          end
+        end
+        requeued
+      end
+
+      # Clears all jobs except the last X jobs
+      def clear_stale
+        return 0 unless @limiter.on?
+        c = @limiter.maximum
+        redis.ltrim(:failed, -c, -1)
+        c
+      end
+
+      # Exntends job(Hash instance) with some helper methods.
+      module FailedJobEx
+        # Returns true if the job has been already retried. Otherwise returns
+        # false.
+        def retried?
+          !self['retried_at'].nil?
+        end
+        alias :requeued? :retried?
+
+        # Returns true if the job processed(failed) before the given time.
+        # Otherwise returns false.
+        # You can pass Time object or String.
+        def before?(time)
+          time = Time.parse(time) if time.is_a?(String)
+          Time.parse(self['failed_at']) < time
+        end
+
+        # Returns true if the job processed(failed) after the given time.
+        # Otherwise returns false.
+        # You can pass Time object or String.
+        def after?(time)
+          time = Time.parse(time) if time.is_a?(String)
+          Time.parse(self['failed_at']) >= time
+        end
+
+        # Returns true if the class of the job matches. Otherwise returns false.
+        def klass?(klass_or_name)
+          if self["payload"] && self["payload"]["class"]
+            self["payload"]["class"] == klass_or_name.to_s
+          else
+            klass_or_name=="UNKNOWN"
+          end
+        end
+
+        # Returns true if the exception raised by the failed job matches. Otherwise returns false.
+        def exception?(exception)
+          self["exception"] == exception.to_s
+        end
+
+        # Returns true if the queue of the job matches. Otherwise returns false.
+        def queue?(queue)
+          self["queue"] == queue.to_s
+        end
+      end
+
+      # Through the Limiter class, you accesses only the last x(default 1000)
+      # jobs.
+      class Limiter
+        @@default_maximum ||= 1000
+
+        class << self
+          def default_maximum
+            @@default_maximum
+          end
+
+          def default_maximum=(v)
+            @@default_maximum = v
+          end
+        end
+
+        attr_accessor :maximum
+        def initialize(cleaner)
+          @cleaner = cleaner
+          @maximum = @@default_maximum
+          @locked = false
+        end
+
+        # Returns true if limiter is ON: number of failed jobs is more than
+        # maximum value.
+        def on?
+          @cleaner.failure.count > @maximum
+        end
+
+        # Returns limited count.
+        def count
+          if @locked
+            @jobs.size
+          else
+            on? ? @maximum : @cleaner.failure.count
+          end
+        end
+
+        # Returns jobs. If numbers of jobs is more than maximum, it returns only
+        # the maximum.
+        def jobs
+          if @locked
+            @jobs
+          else
+            all( - count, count)
+          end
+        end
+
+        # Wraps Resque's all and returns always array.
+        def all(index=0,count=1)
+          jobs = @cleaner.failure.all( index, count)
+          jobs = [] unless jobs
+          jobs = [jobs] unless jobs.is_a?(Array)
+          jobs.each{|j| j.extend FailedJobEx}
+          jobs
+        end
+
+        # Returns a start index of jobs in :failed list.
+        def start_index
+          if @locked
+            @start_index
+          else
+            on? ? @cleaner.failure.count-@maximum : 0
+          end
+        end
+
+        # Assuming new failures pushed while cleaner is dealing with failures,
+        # you need to lock the range.
+        def lock
+          old = @locked
+
+          unless @locked
+            total_count = @cleaner.failure.count
+            if total_count>@maximum
+              @start_index = total_count-@maximum
+              @jobs = all( @start_index, @maximum)
+            else
+              @start_index = 0
+              @jobs = all( 0, total_count)
+            end
+          end
+
+          @locked = true
+          yield
+        ensure
+          @locked = old
+        end
+      end
+
+      # Outputs message. Overrides this method when you want to change a output
+      # stream.
+      def log(msg)
+        puts msg if print?
+      end
+
+      def print?
+        @print_message
+      end
+
+      def too_many_message
+        "There are too many failed jobs(count=#{@failure.count}). This only looks at last #{@limiter.maximum} jobs."
+      end
+    end
+  end
+end
+
+require 'resque_cleaner/server'
+

data/lib/resque_cleaner/server.rb

@@ -0,0 +1,266 @@
+require 'yaml'
+
+# Extends Resque Web Based UI.
+# Structure has been borrowed from ResqueScheduler.
+module ResqueCleaner
+  module Server
+
+    def self.erb_path(filename)
+      File.join(File.dirname(__FILE__), 'server', 'views', filename)
+    end
+    def self.public_path(filename)
+      File.join(File.dirname(__FILE__), 'server', 'public', filename)
+    end
+
+    # Pagination helper for list page.
+    class Paginate
+      attr_accessor :page_size, :page, :jobs, :url
+      def initialize(jobs, url, page=1, page_size=20)
+        @jobs = jobs
+        @url = url
+        @page = (!page || page < 1) ? 1 : page
+        @page_size = 20
+      end
+
+      def first_index
+        @page_size * (@page-1)
+      end
+
+      def last_index
+        last = first_index + @page_size - 1
+        last > @jobs.size-1 ? @jobs.size-1 : last
+      end
+
+      def paginated_jobs
+        @jobs[first_index,@page_size]
+      end
+
+      def first_page?
+        @page <= 1
+      end
+
+      def last_page?
+        @page >= max_page
+      end
+
+      def page_url(page)
+        u = @url
+        u += @url.include?("?") ? "&" : "?"
+        if page.is_a?(Symbol)
+          page = @page - 1 if page==:prev
+          page = @page + 1 if page==:next
+        end
+        u += "p=#{page}"
+      end
+
+      def total_size
+        @jobs.size
+      end
+
+      def max_page
+        ((total_size-1) / @page_size) + 1
+      end
+    end
+
+    def self.included(base)
+
+      base.class_eval do
+        helpers do
+          def time_filter(id, name, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            [1, 3, 6, 12, 24].each do |h|
+              selected = h.to_s == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{h}\">#{h} #{h==1 ? "hour" : "hours"} ago</option>"
+            end
+            [3, 7, 14, 28].each do |d|
+              selected = (d*24).to_s == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{d*24}\">#{d} days ago</option>"
+            end
+            html += "</select>"
+          end
+
+          def class_filter(id, name, klasses, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            klasses.each do |k|
+              selected = k == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{k}\">#{k}</option>"
+            end
+            html += "</select>"
+          end
+
+          def exception_filter(id, name, exceptions, value)
+            html = "<select id=\"#{id}\" name=\"#{name}\">"
+            html += "<option value=\"\">-</option>"
+            exceptions.each do |ex|
+              selected = ex == value ? 'selected="selected"' : ''
+              html += "<option #{selected} value=\"#{ex}\">#{ex}</option>"
+            end
+            html += "</select>"
+          end
+
+          def show_job_args(args)
+            Array(args).map { |a| a.inspect }.join("\n")
+          end
+
+          def text_filter(id, name, value)
+            html = "<input id=\"#{id}\"  type=\"text\" name=\"#{name}\" value=\"#{value}\">"
+            html += "</input>"
+          end
+        end
+
+        mime_type :json, 'application/json'
+
+        get "/cleaner" do
+          load_library
+          load_cleaner_filter
+
+          @jobs = cleaner.select
+          @stats = { :klass => {}, :exception => {} }
+          @total = Hash.new(0)
+          @jobs.each do |job|
+            klass = job["payload"]["class"] || 'UNKNOWN'
+            exception = job["exception"] || 'UNKNOWN'
+            failed_at = Time.parse job["failed_at"]
+            @stats[:klass][klass] ||= Hash.new(0)
+            @stats[:exception][exception] ||= Hash.new(0)
+
+            [
+              @stats[:klass][klass],
+              @stats[:exception][exception],
+              @total
+            ].each do |stat|
+              stat[:total] += 1
+              stat[:h1] += 1 if failed_at >= hours_ago(1)
+              stat[:h3] += 1 if failed_at >= hours_ago(3)
+              stat[:d1] += 1 if failed_at >= hours_ago(24)
+              stat[:d3] += 1 if failed_at >= hours_ago(24*3)
+              stat[:d7] += 1 if failed_at >= hours_ago(24*7)
+            end
+          end
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner.erb'))
+        end
+
+        get "/cleaner_list" do
+          load_library
+          load_cleaner_filter
+          build_urls
+
+          block = filter_block
+
+          @failed = cleaner.select(&block).reverse
+
+          @paginate = Paginate.new(@failed, @list_url, params[:p].to_i)
+
+          @klasses = cleaner.stats_by_class.keys
+          @exceptions = cleaner.stats_by_exception.keys
+          @count = cleaner.select(&block).size
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner_list.erb'))
+        end
+
+        post "/cleaner_exec" do
+          load_library
+          load_cleaner_filter
+          build_urls
+
+          if params[:select_all_pages]!="1"
+            @sha1 = {}
+            params[:sha1].split(",").each {|s| @sha1[s] = true }
+          end
+
+          block = filter_block
+
+          @count =
+            case params[:action]
+            when "clear" then cleaner.clear(&block)
+            when "retry_and_clear" then cleaner.requeue(true,&block)
+            when "retry" then cleaner.requeue(false,{},&block)
+            end
+
+          erb File.read(ResqueCleaner::Server.erb_path('cleaner_exec.erb'))
+        end
+
+        get "/cleaner_dump" do
+          load_library
+          load_cleaner_filter
+
+          block = filter_block
+
+          content_type :json
+          JSON.pretty_generate(cleaner.select(&block))
+        end
+
+        post "/cleaner_stale" do
+          load_library
+          cleaner.clear_stale
+          redirect url_path(:cleaner)
+        end
+
+        get /cleaner\/public\/([a-z]+\.[a-z]+)/ do
+          send_file ResqueCleaner::Server.public_path(params[:captures].first)
+        end
+      end
+
+    end
+
+    def cleaner
+      @cleaner ||= Resque::Plugins::ResqueCleaner.new
+      @cleaner.print_message = false
+      @cleaner
+    end
+
+    def load_library
+      require 'digest/sha1'
+      begin
+        require 'yajl/json_gem' unless [].respond_to?(:to_json)
+      rescue Exception
+        require 'json'
+      end
+    end
+
+    def load_cleaner_filter
+      @from = params[:f]=="" ? nil : params[:f]
+      @to = params[:t]=="" ? nil : params[:t]
+      @klass = params[:c]=="" ? nil : params[:c]
+      @exception = params[:ex]=="" ? nil : params[:ex]
+      @regex = params[:regex]=="" ? nil : params[:regex]
+    end
+
+    def build_urls
+      params = {
+        c: @klass,
+        ex: @exception,
+        f: @from,
+        t: @to,
+        regex: @regex
+      }.map {|key,value| "#{key}=#{URI.encode(value.to_s)}"}.join("&")
+
+      @list_url = "cleaner_list?#{params}"
+      @dump_url = "cleaner_dump?#{params}"
+    end
+
+    def filter_block
+      block = lambda{|j|
+        (!@from || j.after?(hours_ago(@from))) &&
+        (!@to || j.before?(hours_ago(@to))) &&
+        (!@klass || j.klass?(@klass)) &&
+        (!@exception || j.exception?(@exception)) &&
+        (!@sha1 || @sha1[Digest::SHA1.hexdigest(j.to_json)]) &&
+        (!@regex || j.to_s =~ /#{@regex}/)
+      }
+    end
+
+    def hours_ago(h)
+      Time.now - h.to_i*60*60
+    end
+    Resque::Server.tabs << 'Cleaner'
+  end
+end
+
+Resque::Server.class_eval do
+  include ResqueCleaner::Server
+end
+