misha-resque-cleaner 0.3.0

data/test/resque_cleaner_test.rb

@@ -0,0 +1,206 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+require 'time'
+describe "ResqueCleaner" do
+  before do
+    Resque.redis.flushall
+
+    @worker = Resque::Worker.new(:jobs,:jobs2)
+
+    # 3 BadJob at 2009-03-13
+    create_and_process_jobs :jobs, @worker, 3, Time.parse('2009-03-13'), BadJob
+    # 2 BadJob by Jason at 2009-03-13
+    create_and_process_jobs :jobs2, @worker, 2, Time.parse('2009-03-13'), BadJob, "Jason"
+
+    # 1 BadJob by Johnson at 2009-03-13
+    create_and_process_jobs :jobs, @worker, 1, Time.parse('2009-03-13'), BadJob, "Johnson"
+
+    # 7 BadJob at 2009-11-13
+    create_and_process_jobs :jobs, @worker, 7, Time.parse('2009-11-13'), BadJobWithSyntaxError
+    # 7 BadJob by Freddy at 2009-11-13
+    create_and_process_jobs :jobs2, @worker, 7, Time.parse('2009-11-13'), BadJob, "Freddy"
+
+    # 11 BadJob at 2010-08-13
+    create_and_process_jobs :jobs, @worker, 11, Time.parse('2010-08-13'), BadJob
+    # 11 BadJob by Jason at 2010-08-13
+    create_and_process_jobs :jobs2, @worker, 11, Time.parse('2010-08-13'), BadJob, "Jason"
+
+    @cleaner = Resque::Plugins::ResqueCleaner.new
+    @cleaner.print_message = false
+  end
+
+  it "#select_by_regex returns only Jason jobs" do
+    ret = @cleaner.select_by_regex(/Jason/)
+    assert_equal 13, ret.size
+  end
+
+  it "#select_by_regex returns an empty array if passed a non-regex" do
+    ['string', nil, 13, Class.new].each do |non_regex|
+      ret = @cleaner.select_by_regex(nil)
+      assert_equal 0, ret.size
+    end
+  end
+
+  it "#select returns failure jobs" do
+    ret = @cleaner.select
+    assert_equal 42, ret.size
+  end
+
+  it "#select works with a limit" do
+    @cleaner.limiter.maximum = 10
+    ret = @cleaner.select
+
+    # only maximum number
+    assert_equal 10, ret.size
+
+    # lait one
+    assert_equal Time.parse(ret[0]['failed_at']), Time.parse('2010-08-13')
+  end
+
+  it "#select with a block returns failure jobs which the block evaluates true" do
+    ret = @cleaner.select {|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, ret.size
+  end
+
+  it "#clear deletes failure jobs" do
+    cleared = @cleaner.clear
+    assert_equal 42, cleared
+    assert_equal 0, @cleaner.select.size
+  end
+
+  it "#clear with a block deletes failure jobs which the block evaluates true" do
+    cleared = @cleaner.clear{|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, cleared
+    assert_equal 42-13, @cleaner.select.size
+    assert_equal 0, @cleaner.select{|job| job["payload"]["args"][0]=="Jason"}.size
+  end
+
+  it "#requeue retries failure jobs" do
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    requeued = @cleaner.requeue
+    assert_equal 42, requeued
+    assert_equal 42, @cleaner.select.size # it doesn't clear jobs
+    assert_equal 42, queue_size(:jobs,:jobs2)
+  end
+
+  it "#requeue with a block retries failure jobs which the block evaluates true" do
+    requeued = @cleaner.requeue{|job| job["payload"]["args"][0]=="Jason"}
+    assert_equal 13, requeued
+    assert_equal 13, queue_size(:jobs,:jobs2)
+  end
+
+  it "#requeue with clear option requeues and deletes failure jobs" do
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    requeued = @cleaner.requeue(true)
+    assert_equal 42, requeued
+    assert_equal 42, queue_size(:jobs,:jobs2)
+    assert_equal 0, @cleaner.select.size
+  end
+
+  it "#requeue with :queue option requeues the jobs to the queue" do
+    assert_equal 0, queue_size(:jobs,:jobs2,:retry)
+    requeued = @cleaner.requeue false, :queue => :retry
+    assert_equal 42, requeued
+    assert_equal 42, @cleaner.select.size # it doesn't clear jobs
+    assert_equal 0, queue_size(:jobs,:jobs2)
+    assert_equal 42, queue_size(:retry)
+  end
+
+  it "#clear_stale deletes failure jobs which is queued before the last x enqueued" do
+    @cleaner.limiter.maximum = 10
+    @cleaner.clear_stale
+    assert_equal 10, @cleaner.failure.count
+    assert_equal Time.parse(@cleaner.failure_jobs[0]['failed_at']), Time.parse('2010-08-13')
+  end
+
+  it "FailedJobEx module extends job and provides some useful methods" do
+    # before 2009-04-01
+    ret = @cleaner.select {|j| j.before?('2009-04-01')}
+    assert_equal 6, ret.size
+
+    # after 2010-01-01
+    ret = @cleaner.select {|j| j.after?('2010-01-01')}
+    assert_equal 22, ret.size
+
+    # filter by class
+    ret = @cleaner.select {|j| j.klass?(BadJobWithSyntaxError)}
+    assert_equal 7, ret.size
+
+    # filter by exception
+    ret = @cleaner.select {|j| j.exception?(SyntaxError)}
+    assert_equal 7, ret.size
+
+    # filter by queue
+    ret = @cleaner.select {|j| j.queue?(:jobs2)}
+    assert_equal 20, ret.size
+
+    # combination
+    ret = @cleaner.select {|j| j.queue?(:jobs2) && j.before?('2009-12-01')}
+    assert_equal 9, ret.size
+
+    # combination 2
+    ret = @cleaner.select {|j| j['payload']['args']==['Jason'] && j.queue?(:jobs2)}
+    assert_equal 13, ret.size
+
+    # retried?
+    requeued = @cleaner.requeue{|j| j["payload"]["args"][0]=="Johnson"}
+    ret = @cleaner.select {|j| j.retried?}
+    assert_equal 1, ret.size
+  end
+
+  it "#stats_by_date returns stats grouped by date" do
+    ret = @cleaner.stats_by_date
+    assert_equal 6, ret['2009/03/13']
+    assert_equal 14, ret['2009/11/13']
+
+    # with block
+    ret = @cleaner.stats_by_date{|j| j['payload']['args']==['Jason']}
+    assert_equal 2, ret['2009/03/13']
+    assert_equal nil, ret['2009/11/13']
+    assert_equal 11, ret['2010/08/13']
+  end
+
+  it "#stats_by_class returns stats grouped by class" do
+    ret = @cleaner.stats_by_class
+    assert_equal 35, ret['BadJob']
+    assert_equal 7, ret['BadJobWithSyntaxError']
+  end
+
+  it "#stats_by_class works with broken log" do
+    add_empty_payload_failure
+    ret = @cleaner.stats_by_class
+    assert_equal 1, ret['UNKNOWN']
+  end
+
+  it "#stats_by_exception returns stats grouped by exception" do
+    ret = @cleaner.stats_by_exception
+    assert_equal 35, ret['RuntimeError']
+    assert_equal 7, ret['SyntaxError']
+  end
+
+  it "#lock ensures that a new failure job doesn't affect in a limit mode" do
+    @cleaner.limiter.maximum = 23
+    @cleaner.limiter.lock do
+      first = @cleaner.select[0]
+      assert_equal "Freddy", first["payload"]["args"][0]
+
+      create_and_process_jobs :jobs, @worker, 30, Time.parse('2010-10-10'), BadJob, "Jack"
+
+      first = @cleaner.select[0]
+      assert_equal "Freddy", first["payload"]["args"][0]
+    end
+    first = @cleaner.select[0]
+    assert_equal "Jack", first["payload"]["args"][0]
+  end
+
+  it "allows you to configure limiter" do
+    c = Resque::Plugins::ResqueCleaner.new
+    refute_equal c.limiter.maximum, 10_000
+
+    module Resque::Plugins
+      ResqueCleaner::Limiter.default_maximum = 10_000
+    end
+
+    c = Resque::Plugins::ResqueCleaner.new
+    assert_equal c.limiter.maximum, 10_000
+  end
+end

data/test/resque_web_test.rb

@@ -0,0 +1,66 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+require 'digest/sha1'
+require 'json'
+require 'rack/test'
+
+class Minitest::Spec
+  include Rack::Test::Methods
+  def app
+    Resque::Server.new
+  end
+end
+
+def setup_some_failed_jobs
+  Resque.redis.flushall
+
+  @worker = Resque::Worker.new(:jobs,:jobs2)
+
+  create_and_process_jobs :jobs, @worker, 1, Time.now, BadJobWithSyntaxError, "great_args"
+
+  10.times {|i|
+    create_and_process_jobs :jobs, @worker, 1, Time.now, BadJob, "test_#{i}"
+  }
+
+  @cleaner = Resque::Plugins::ResqueCleaner.new
+  @cleaner.print_message = false
+end
+
+describe "resque-web" do
+  before do
+    setup_some_failed_jobs
+  end
+
+  it "#cleaner should respond with success" do
+    get "/cleaner"
+    assert last_response.body.include?('BadJob')
+    assert last_response.body =~ /\bException\b/
+  end
+
+  it "#cleaner_list should respond with success" do
+    get "/cleaner_list"
+    assert last_response.ok?, last_response.errors
+  end
+
+  it '#cleaner_list shows the failed jobs' do
+    get "/cleaner_list"
+    assert last_response.body.include?('BadJob')
+  end
+
+  it "#cleaner_list shows the failed jobs when we use a select_by_regex" do
+    get "/cleaner_list", :regex => "BadJob*"
+    assert last_response.body.include?('"BadJobWithSyntaxError"')
+    assert last_response.body.include?('"BadJob"')
+  end
+
+
+  it '#cleaner_exec clears job' do
+    post "/cleaner_exec", :action => "clear", :sha1 => Digest::SHA1.hexdigest(@cleaner.select[0].to_json)
+    assert_equal 10, @cleaner.select.size
+  end
+  it "#cleaner_dump should respond with success" do
+    get "/cleaner_dump"
+    assert last_response.ok?, last_response.errors
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,131 @@
+# Mostly copied from Resque in order to have similar test environment.
+# https://github.com/defunkt/resque/blob/master/test/test_helper.rb
+
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+$TESTING = true
+require 'rubygems'
+require 'minitest'
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'resque'
+require 'timecop'
+
+begin
+  require 'leftright'
+rescue LoadError
+end
+require 'resque'
+require 'resque_cleaner'
+
+$TEST_PID = Process.pid
+
+#
+# make sure we can run redis
+#
+
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** try running `sudo rake install`"
+  abort ''
+end
+
+
+#
+# start our own redis when the tests start,
+# kill it when they end
+#
+MiniTest.after_run {
+  if Process.pid == $TEST_PID
+    processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
+    pids = processes.map { |process| process.split(" ")[0] }
+    puts "Killing test redis server..."
+    pids.each { |pid| Process.kill("TERM", pid.to_i) }
+    dump = "test/dump.rdb"
+    File.delete(dump) if File.exist?(dump)
+  end
+}
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+Resque.redis = 'localhost:9736'
+
+
+##
+# Helper to perform job classes
+#
+module PerformJob
+  def perform_job(klass, *args)
+    resque_job = Resque::Job.new(:testqueue, 'class' => klass, 'args' => args)
+    resque_job.perform
+  end
+end
+
+#
+# fixture classes
+#
+
+class SomeJob
+  def self.perform(repo_id, path)
+  end
+end
+
+class SomeIvarJob < SomeJob
+  @queue = :ivar
+end
+
+class SomeMethodJob < SomeJob
+  def self.queue
+    :method
+  end
+end
+
+class BadJob
+  def self.perform(name=nil)
+    msg = name ? "Bad job, #{name}" : "Bad job!"
+    raise msg
+  end
+end
+
+class GoodJob
+  def self.perform(name)
+    "Good job, #{name}"
+  end
+end
+
+class BadJobWithSyntaxError
+  def self.perform
+    raise SyntaxError, "Extra Bad job!"
+  end
+end
+
+#
+# helper methods
+#
+
+def create_and_process_jobs(queue,worker,num,date,job,*args)
+  Timecop.freeze(date) do
+    num.times do
+      Resque::Job.create(queue, job, *args)
+    end
+    worker.work(0)
+  end
+end
+
+def queue_size(*queues)
+  queues.inject(0){|sum,queue| sum + Resque.size(queue).to_i}
+end
+
+def add_empty_payload_failure
+  data = {
+    :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+    :payload   => nil,
+    :exception => "Resque::DirtyExit",
+    :error     => "Resque::DirtyExit",
+    :backtrace => [],
+    :worker    => "worker",
+    :queue     => "queue"
+  }
+  data = Resque.encode(data)
+  Resque.redis.rpush(:failed, data)
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: misha-resque-cleaner
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Tatsuya Ono
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+description: |2
+      resque-cleaner maintains the cleanliness of failed jobs on Resque.
+email: ononoma@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+- CHANGELOG.md
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- Rakefile
+- lib/resque-cleaner.rb
+- lib/resque_cleaner.rb
+- lib/resque_cleaner/server.rb
+- lib/resque_cleaner/server/public/cleaner.css
+- lib/resque_cleaner/server/views/_limiter.erb
+- lib/resque_cleaner/server/views/_paginate.erb
+- lib/resque_cleaner/server/views/_stats.erb
+- lib/resque_cleaner/server/views/cleaner.erb
+- lib/resque_cleaner/server/views/cleaner_exec.erb
+- lib/resque_cleaner/server/views/cleaner_list.erb
+- test/redis-test.conf
+- test/resque_cleaner_test.rb
+- test/resque_web_test.rb
+- test/test_helper.rb
+homepage: https://github.com/ono/resque-cleaner
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--charset=UTF-8"
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Resque plugin cleaning up failed jobs.
+test_files: []
+has_rdoc: