resque-cleanerer 0.5.0

data/lib/resque_cleaner.rb

@@ -0,0 +1,309 @@
+require 'time'
+require 'resque'
+require 'resque/server'
+
+module Resque
+  module Plugins
+    # ResqueCleaner class provides useful functionalities to retry or clean
+    # failed jobs. Let's clean up your failed list!
+    class ResqueCleaner
+      # ResqueCleaner fetches all elements from Redis and checks them
+      # by linear when filtering them. Since there is a performance concern,
+      # ResqueCleaner handles only the latest x(default 1000) jobs.
+      #
+      # You can change the value through limiter attribute.
+      # e.g. cleaner.limiter.maximum = 5000
+      attr_reader :limiter
+
+      # Set false if you don't show any message.
+      attr_accessor :print_message
+
+      # Initializes instance
+      def initialize
+        @failure = Resque::Failure.backend
+        @print_message = true
+        @limiter = Limiter.new self
+      end
+
+      # Returns redis instance.
+      def redis
+        Resque.redis
+      end
+
+      # Returns failure backend. Only supports redis backend.
+      def failure
+        @failure
+      end
+
+      # Stats by date.
+      def stats_by_date(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          date = job["failed_at"][0,10]
+          stats[date] ||= 0
+          stats[date] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Stats by class.
+      def stats_by_class(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          stats[job.klass_name] ||= 0
+          stats[job.klass_name] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Stats by exception.
+      def stats_by_exception(&block)
+        jobs, stats = select(&block), {}
+        jobs.each do |job|
+          exception = job["exception"]
+          stats[exception] ||= 0
+          stats[exception] += 1
+        end
+
+        print_stats(stats) if print?
+        stats
+      end
+
+      # Print stats
+      def print_stats(stats)
+        log too_many_message if @limiter.on?
+        stats.keys.sort.each do |k|
+          log "%15s: %4d" % [k,stats[k]]
+        end
+        log "%15s: %4d" % ["total", @limiter.count]
+      end
+
+      # Returns every jobs for which block evaluates to true.
+      def select(&block)
+        jobs = @limiter.jobs
+        block_given? ? @limiter.jobs.select(&block) : jobs
+      end
+      alias :failure_jobs :select
+
+      def select_by_regex(regex)
+        select do |job|
+          job.to_s =~ regex
+        end
+      end
+
+      # Clears every jobs for which block evaluates to true.
+      def clear(&block)
+        cleared = 0
+        @limiter.lock do
+          @limiter.jobs.each_with_index do |job,i|
+            if !block_given? || block.call(job)
+              index = @limiter.start_index + i - cleared
+              # fetches again since you can't ensure that it is always true:
+              # a == endode(decode(a))
+              value = redis.lindex(:failed, index)
+              redis.lrem(:failed, 1, value)
+              cleared += 1
+            end
+          end
+        end
+        cleared
+      end
+
+      # Retries every jobs for which block evaluates to true.
+      def requeue(clear_after_requeue=false, options={}, &block)
+        requeued = 0
+        queue = options["queue"] || options[:queue]
+        @limiter.lock do
+          @limiter.jobs.each_with_index do |job,i|
+            if !block_given? || block.call(job)
+              index = @limiter.start_index + i - requeued
+
+              value = redis.lindex(:failed, index)
+              redis.multi do
+                Job.create(queue||job['queue'], job['payload']['class'], *job['payload']['args'])
+
+                if clear_after_requeue
+                  # remove job
+                  # TODO: should use ltrim. not sure why i used lrem here...
+                  redis.lrem(:failed, 1, value)
+                else
+                  # mark retried
+                  job['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+                  redis.lset(:failed, @limiter.start_index+i, Resque.encode(job))
+                end
+              end
+
+              requeued += 1
+            end
+          end
+        end
+        requeued
+      end
+
+      # Clears all jobs except the last X jobs
+      def clear_stale
+        return 0 unless @limiter.on?
+        c = @limiter.maximum
+        redis.ltrim(:failed, -c, -1)
+        c
+      end
+
+      # Exntends job(Hash instance) with some helper methods.
+      module FailedJobEx
+        # Returns true if the job has been already retried. Otherwise returns
+        # false.
+        def retried?
+          !self['retried_at'].nil?
+        end
+        alias :requeued? :retried?
+
+        # Returns true if the job processed(failed) before the given time.
+        # Otherwise returns false.
+        # You can pass Time object or String.
+        def before?(time)
+          time = Time.parse(time) if time.is_a?(String)
+          Time.parse(self['failed_at']) < time
+        end
+
+        # Returns true if the job processed(failed) after the given time.
+        # Otherwise returns false.
+        # You can pass Time object or String.
+        def after?(time)
+          time = Time.parse(time) if time.is_a?(String)
+          Time.parse(self['failed_at']) >= time
+        end
+
+        # Returns true if the class of the job matches. Otherwise returns false.
+        def klass?(klass_or_name)
+          klass_name == klass_or_name
+        end
+
+        def klass_name
+          @klass_name ||=
+          (self["payload"]["args"] && self["payload"]["args"][0] && self["payload"]["args"][0]["job_class"]) || # ActiveJob
+          self["payload"]["class"] || # Rescue
+          "UNKNOWN"
+        end
+
+        # Returns true if the exception raised by the failed job matches. Otherwise returns false.
+        def exception?(exception)
+          self["exception"] == exception.to_s
+        end
+
+        # Returns true if the queue of the job matches. Otherwise returns false.
+        def queue?(queue)
+          self["queue"] == queue.to_s
+        end
+      end
+
+      # Through the Limiter class, you accesses only the last x(default 1000)
+      # jobs.
+      class Limiter
+        @@default_maximum ||= 1000
+
+        class << self
+          def default_maximum
+            @@default_maximum
+          end
+
+          def default_maximum=(v)
+            @@default_maximum = v
+          end
+        end
+
+        attr_accessor :maximum
+        def initialize(cleaner)
+          @cleaner = cleaner
+          @maximum = @@default_maximum
+          @locked = false
+        end
+
+        # Returns true if limiter is ON: number of failed jobs is more than
+        # maximum value.
+        def on?
+          @cleaner.failure.count > @maximum
+        end
+
+        # Returns limited count.
+        def count
+          if @locked
+            @jobs.size
+          else
+            on? ? @maximum : @cleaner.failure.count
+          end
+        end
+
+        # Returns jobs. If numbers of jobs is more than maximum, it returns only
+        # the maximum.
+        def jobs
+          if @locked
+            @jobs
+          else
+            all( - count, count)
+          end
+        end
+
+        # Wraps Resque's all and returns always array.
+        def all(index=0,count=1)
+          jobs = @cleaner.failure.all( index, count)
+          jobs = [] unless jobs
+          jobs = [jobs] unless jobs.is_a?(Array)
+          jobs.each{|j| j.extend FailedJobEx}
+          jobs
+        end
+
+        # Returns a start index of jobs in :failed list.
+        def start_index
+          if @locked
+            @start_index
+          else
+            on? ? @cleaner.failure.count-@maximum : 0
+          end
+        end
+
+        # Assuming new failures pushed while cleaner is dealing with failures,
+        # you need to lock the range.
+        def lock
+          old = @locked
+
+          unless @locked
+            total_count = @cleaner.failure.count
+            if total_count>@maximum
+              @start_index = total_count-@maximum
+              @jobs = all( @start_index, @maximum)
+            else
+              @start_index = 0
+              @jobs = all( 0, total_count)
+            end
+          end
+
+          @locked = true
+          yield
+        ensure
+          @locked = old
+        end
+      end
+
+      # Outputs message. Overrides this method when you want to change a output
+      # stream.
+      def log(msg)
+        puts msg if print?
+      end
+
+      def print?
+        @print_message
+      end
+
+      def too_many_message
+        "There are too many failed jobs(count=#{@failure.count}). This only looks at last #{@limiter.maximum} jobs."
+      end
+    end
+  end
+end
+
+require 'resque_cleaner/server'
+

data/test/redis-test.conf

@@ -0,0 +1,115 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+#glueoutputbuf yes

data/test/resque_cleaner_test.rb

@@ -0,0 +1,206 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+require 'time'
+describe "ResqueCleaner" do
+  before do
+    Resque.redis.flushall
+
+    @worker = Resque::Worker.new(:jobs,:jobs2)
+
+    # 3 BadJob at 2009-03-13
+    create_and_process_jobs :jobs, @worker, 3, Time.parse('2009-03-13'), BadJob
+    # 2 BadJob by Jason at 2009-03-13
+    create_and_process_jobs :jobs2, @worker, 2, Time.parse('2009-03-13'), BadJob, "Jason"
+
+    # 1 BadJob by Johnson at 2009-03-13
+    create_and_process_jobs :jobs, @worker, 1, Time.parse('2009-03-13'), BadJob, "Johnson"
+
+    # 7 BadJob at 2009-11-13
+    create_and_process_jobs :jobs, @worker, 7, Time.parse('2009-11-13'), BadJobWithSyntaxError
+    # 7 BadJob by Freddy at 2009-11-13
+    create_and_process_jobs :jobs2, @worker, 7, Time.parse('2009-11-13'), BadJob, "Freddy"
+
+    # 11 BadJob at 2010-08-13
+    create_and_process_jobs :jobs, @worker, 11, Time.parse('2010-08-13'), BadJob
+    # 11 BadJob by Jason at 2010-08-13
+    create_and_process_jobs :jobs2, @worker, 11, Time.parse('2010-08-13'), BadJob, "Jason"
+
+    @cleaner = Resque::Plugins::ResqueCleaner.new
+    @cleaner.print_message = false
+  end
+
+  it "#select_by_regex returns only Jason jobs" do
+    ret = @cleaner.select_by_regex(/Jason/)
+    assert_equal 13, ret.size
+  end
+
+  it "#select_by_regex returns an empty array if passed a non-regex" do
+    ['string', nil, 13, Class.new].each do |non_regex|
+      ret = @cleaner.select_by_regex(nil)
+      assert_equal 0, ret.size
+    end
+  end
+
+  it "#select returns failure jobs" do
+    ret = @cleaner.select
+    assert_equal 42, ret.size
+  end
+
+  it "#select works with a limit" do
+    @cleaner.limiter.maximum = 10
+    ret = @cleaner.select
+
+    # only maximum number
+    assert_equal 10, ret.size
+
+    # lait one
+    assert_equal Time.parse(ret[0]['failed_at']), Time.parse('2010-08-13')
+  end
+
+  it "#select with a block returns failure jobs which the block evaluates true" do
+    ret = @cleaner.select {|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, ret.size
+  end
+
+  it "#clear deletes failure jobs" do
+    cleared = @cleaner.clear
+    assert_equal 42, cleared
+    assert_equal 0, @cleaner.select.size
+  end
+
+  it "#clear with a block deletes failure jobs which the block evaluates true" do
+    cleared = @cleaner.clear{|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, cleared
+    assert_equal 42-13, @cleaner.select.size
+    assert_equal 0, @cleaner.select{|job| job["payload"]["args"][0]=="Jason"}.size
+  end
+
+  it "#requeue retries failure jobs" do
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    requeued = @cleaner.requeue
+    assert_equal 42, requeued
+    assert_equal 42, @cleaner.select.size # it doesn't clear jobs
+    assert_equal 42, queue_size(:jobs,:jobs2)
+  end
+
+  it "#requeue with a block retries failure jobs which the block evaluates true" do
+    requeued = @cleaner.requeue{|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, requeued
+    assert_equal 13, queue_size(:jobs,:jobs2)
+  end
+
+  it "#requeue with clear option requeues and deletes failure jobs" do
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    requeued = @cleaner.requeue(true)
+    assert_equal 42, requeued
+    assert_equal 42, queue_size(:jobs,:jobs2)
+    assert_equal 0, @cleaner.select.size
+  end
+
+  it "#requeue with :queue option requeues the jobs to the queue" do
+    assert_equal 0, queue_size(:jobs,:jobs2,:retry)
+    requeued = @cleaner.requeue false, :queue => :retry
+    assert_equal 42, requeued
+    assert_equal 42, @cleaner.select.size # it doesn't clear jobs
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    assert_equal 42, queue_size(:retry)
+  end
+
+  it "#clear_stale deletes failure jobs which is queued before the last x enqueued" do
+    @cleaner.limiter.maximum = 10
+    @cleaner.clear_stale
+    assert_equal 10, @cleaner.failure.count
+    assert_equal Time.parse(@cleaner.failure_jobs[0]['failed_at']), Time.parse('2010-08-13')
+  end
+
+  it "FailedJobEx module extends job and provides some useful methods" do
+    # before 2009-04-01
+    ret = @cleaner.select {|j| j.before?('2009-04-01')}
+    assert_equal 6, ret.size
+
+    # after 2010-01-01
+    ret = @cleaner.select {|j| j.after?('2010-01-01')}
+    assert_equal 22, ret.size
+
+    # filter by class
+    ret = @cleaner.select {|j| j.klass?(BadJobWithSyntaxError)}
+    assert_equal 7, ret.size
+
+    # filter by exception
+    ret = @cleaner.select {|j| j.exception?(SyntaxError)}
+    assert_equal 7, ret.size
+
+    # filter by queue
+    ret = @cleaner.select {|j| j.queue?(:jobs2)}
+    assert_equal 20, ret.size
+
+    # combination
+    ret = @cleaner.select {|j| j.queue?(:jobs2) && j.before?('2009-12-01')}
+    assert_equal 9, ret.size
+
+    # combination 2
+    ret = @cleaner.select {|j| j['payload']['args']==['Jason'] && j.queue?(:jobs2)}
+    assert_equal 13, ret.size
+
+    # retried?
+    requeued = @cleaner.requeue{|j| j["payload"]["args"][0]=="Johnson"}
+    ret = @cleaner.select {|j| j.retried?}
+    assert_equal 1, ret.size
+  end
+
+  it "#stats_by_date returns stats grouped by date" do
+    ret = @cleaner.stats_by_date
+    assert_equal 6, ret['2009/03/13']
+    assert_equal 14, ret['2009/11/13']
+
+    # with block
+    ret = @cleaner.stats_by_date{|j| j['payload']['args']==['Jason']}
+    assert_equal 2, ret['2009/03/13']
+    assert_equal nil, ret['2009/11/13']
+    assert_equal 11, ret['2010/08/13']
+  end
+
+  it "#stats_by_class returns stats grouped by class" do
+    ret = @cleaner.stats_by_class
+    assert_equal 35, ret['BadJob']
+    assert_equal 7, ret['BadJobWithSyntaxError']
+  end
+
+  it "#stats_by_class works with broken log" do
+    add_empty_payload_failure
+    ret = @cleaner.stats_by_class
+    assert_equal 1, ret['UNKNOWN']
+  end
+
+  it "#stats_by_exception returns stats grouped by exception" do
+    ret = @cleaner.stats_by_exception
+    assert_equal 35, ret['RuntimeError']
+    assert_equal 7, ret['SyntaxError']
+  end
+
+  it "#lock ensures that a new failure job doesn't affect in a limit mode" do
+    @cleaner.limiter.maximum = 23
+    @cleaner.limiter.lock do
+      first = @cleaner.select[0]
+      assert_equal "Freddy", first["payload"]["args"][0]
+
+      create_and_process_jobs :jobs, @worker, 30, Time.parse('2010-10-10'), BadJob, "Jack"
+
+      first = @cleaner.select[0]
+      assert_equal "Freddy", first["payload"]["args"][0]
+    end
+    first = @cleaner.select[0]
+    assert_equal "Jack", first["payload"]["args"][0]
+  end
+
+  it "allows you to configure limiter" do
+    c = Resque::Plugins::ResqueCleaner.new
+    refute_equal c.limiter.maximum, 10_000
+
+    module Resque::Plugins
+      ResqueCleaner::Limiter.default_maximum = 10_000
+    end
+
+    c = Resque::Plugins::ResqueCleaner.new
+    assert_equal c.limiter.maximum, 10_000
+  end
+end

data/test/resque_web_test.rb

@@ -0,0 +1,66 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+require 'digest/sha1'
+require 'json'
+require 'rack/test'
+
+class Minitest::Spec
+  include Rack::Test::Methods
+  def app
+    Resque::Server.new
+  end
+end
+
+def setup_some_failed_jobs
+  Resque.redis.flushall
+
+  @worker = Resque::Worker.new(:jobs,:jobs2)
+
+  create_and_process_jobs :jobs, @worker, 1, Time.now, BadJobWithSyntaxError, "great_args"
+
+  10.times {|i|
+    create_and_process_jobs :jobs, @worker, 1, Time.now, BadJob, "test_#{i}"
+  }
+
+  @cleaner = Resque::Plugins::ResqueCleaner.new
+  @cleaner.print_message = false
+end
+
+describe "resque-web" do
+  before do
+    setup_some_failed_jobs
+  end
+
+  it "#cleaner should respond with success" do
+    get "/cleaner"
+    assert last_response.body.include?('BadJob')
+    assert last_response.body =~ /\bException\b/
+  end
+
+  it "#cleaner_list should respond with success" do
+    get "/cleaner_list"
+    assert last_response.ok?, last_response.errors
+  end
+
+  it '#cleaner_list shows the failed jobs' do
+    get "/cleaner_list"
+    assert last_response.body.include?('BadJob')
+  end
+
+  it "#cleaner_list shows the failed jobs when we use a select_by_regex" do
+    get "/cleaner_list", :regex => "BadJob*"
+    assert last_response.body.include?('"BadJobWithSyntaxError"')
+    assert last_response.body.include?('"BadJob"')
+  end
+
+
+  it '#cleaner_exec clears job' do
+    post "/cleaner_exec", :action => "clear", :sha1 => Digest::SHA1.hexdigest(@cleaner.select[0].to_json)
+    assert_equal 10, @cleaner.select.size
+  end
+  it "#cleaner_dump should respond with success" do
+    get "/cleaner_dump"
+    assert last_response.ok?, last_response.errors
+  end
+end
+