creeper 1.0.9 → 2.0.0

data/spec/lib/creeper/client_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+require 'creeper/client'
+require 'creeper/worker'
+
+describe Creeper::Client do
+
+  context 'with mock beanstalk and redis' do
+    before do
+      @beanstalk  = double('beanstalk')
+      @connection = double('connection')
+      def @beanstalk.on_tube(tube); yield @connection if block_given?; end
+      Creeper.instance_variable_set(:@beanstalk, @beanstalk)
+      @redis = double('redis')
+      def @redis.multi; [yield] * 2 if block_given?; end
+      def @redis.set(*); true; end
+      def @redis.sadd(*); true; end
+      def @redis.srem(*); true; end
+      def @redis.get(*); nil; end
+      def @redis.del(*); nil; end
+      def @redis.incrby(*); nil; end
+      def @redis.setex(*); true; end
+      def @redis.expire(*); true; end
+      def @redis.watch(*); true; end
+      def @redis.with_connection; yield self; end
+      def @redis.with; yield self; end
+      def @redis.exec; true; end
+      Creeper.instance_variable_set(:@redis, @redis)
+    end
+
+    after do
+      Creeper.instance_variable_set(:@beanstalk, nil)
+      Creeper.instance_variable_set(:@redis, nil)
+    end
+
+    it 'raises ArgumentError with invalid params' do
+      expect do
+        Creeper::Client.push('foo', 1)
+      end.to raise_error(ArgumentError)
+
+      expect do
+        Creeper::Client.push('foo', :class => 'Foo', :noargs => [1, 2])
+      end.to raise_error(ArgumentError)
+    end
+
+    it 'pushes messages to beanstalk' do
+      @beanstalk.should_receive(:on_tube).with('foo').and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["foo",{"retry":true,"queue":"foo","class":"MyWorker","args":[1,2]}])).and_return(24)
+      pushed = Creeper::Client.push('queue' => 'foo', 'class' => MyWorker, 'args' => [1, 2])
+      expect(pushed).to eq(24)
+    end
+
+    it 'has default options' do
+      expect(Creeper::Worker::ClassMethods::DEFAULT_OPTIONS).to eq(MyWorker.get_creeper_options)
+    end
+
+    it 'handles perform_async' do
+      @beanstalk.should_receive(:on_tube).with('default').and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["default",{"retry":true,"queue":"default","class":"MyWorker","args":[1,2]}])).and_return(24)
+      pushed = MyWorker.perform_async(1, 2)
+      expect(pushed).to eq(24)
+    end
+
+    it 'handles perform_async on failure' do
+      @beanstalk.should_receive(:on_tube).with('default').and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["default",{"retry":true,"queue":"default","class":"MyWorker","args":[1,2]}])).and_return(nil)
+      pushed = MyWorker.perform_async(1, 2)
+      expect(pushed).to be_nil
+    end
+
+    it 'enqueues messages to beanstalk' do
+      @beanstalk.should_receive(:on_tube).with('default').and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["default",{"retry":true,"queue":"default","class":"MyWorker","args":[1,2]}])).and_return(24)
+      pushed = Creeper::Client.enqueue(MyWorker, 1, 2)
+      expect(pushed).to eq(24)
+    end
+
+    it 'enqueues messages to beanstalk using legacy method' do
+      @beanstalk.should_receive(:on_tube).with('my.worker').and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["my.worker",{"retry":true,"queue":"my.worker","args":[1,2],"class":"MyWorker","delay":0,"priority":65536,"time_to_run":120}]), 65536, 0, 120).and_return(24)
+      pushed = Creeper.enqueue('my.worker', 1, 2)
+      expect(pushed).to eq(24)
+    end
+
+    it 'enqueues to the named queue' do
+      @beanstalk.should_receive(:on_tube).with(:flimflam).and_yield(@connection)
+      @connection.should_receive(:put).with(%q(["flimflam",{"retry":true,"queue":"flimflam","timeout":1,"class":"QueuedWorker","args":[1,2]}])).and_return(24)
+      pushed = QueuedWorker.perform_async(1, 2)
+      expect(pushed).to eq(24)
+    end
+
+    it 'retrieves queues' do
+      @redis.should_receive(:smembers).with('queues').and_return(['bob'])
+      expect(Creeper::Client.registered_queues).to eq(['bob'])
+    end
+
+    it 'retrieves workers' do
+      @redis.should_receive(:smembers).with('workers').and_return(['bob'])
+      expect(Creeper::Client.registered_workers).to eq(['bob'])
+    end
+
+  end
+
+  describe 'inheritance' do
+    it 'should inherit creeper options' do
+      expect(AWorker.get_creeper_options['retry']).to eq('base')
+      expect(BWorker.get_creeper_options['retry']).to eq('b')
+    end
+  end
+
+end

data/spec/lib/creeper/exception_handler_spec.rb

@@ -0,0 +1,110 @@
+require 'spec_helper'
+require 'creeper'
+require 'creeper/exception_handler'
+require 'stringio'
+require 'logger'
+
+describe Creeper::ExceptionHandler do
+
+  before do
+    stub_const('ExceptionHandlerTestException', Class.new(StandardError))
+    stub_const('TEST_EXCEPTION', ExceptionHandlerTestException.new("Something didn't work!"))
+  end
+
+  class Component
+    include Creeper::Util
+
+    def invoke_exception(args)
+      raise TEST_EXCEPTION
+    rescue ExceptionHandlerTestException => e
+      handle_exception(e,args)
+    end
+  end
+
+  describe "with mock logger" do
+    before do
+      @old_logger = Creeper.logger
+      @str_logger = StringIO.new
+      Creeper.logger = Logger.new(@str_logger)
+    end
+
+    after do
+      Creeper.logger = @old_logger
+    end
+
+    it "logs the exception to Creeper.logger" do
+      Component.new.invoke_exception(:a => 1)
+      @str_logger.rewind
+      log = @str_logger.readlines
+      expect(log[0]).to match(/a=>1/)                                           # didn't include the context
+      expect(log[1]).to match(/Something didn't work!/)                         # didn't include the exception message
+      expect(log[2]).to match(/spec\/lib\/creeper\/exception_handler_spec\.rb/) # didn't include the backtrace
+    end
+  end
+
+  describe "with fake Airbrake" do
+    before do
+      ::Airbrake = double('Airbrake')
+    end
+
+    after do
+      Object.send(:remove_const, "Airbrake") # HACK should probably inject Airbrake etc into this class in the future
+    end
+
+    it "notifies Airbrake" do
+      ::Airbrake.should_receive(:notify).with(TEST_EXCEPTION,:parameters => { :a => 1 }).and_return(nil)
+      Component.new.invoke_exception(:a => 1)
+    end
+  end
+
+  describe "with fake ExceptionNotifier" do
+    before do
+      ::ExceptionNotifier = Module.new
+      ::ExceptionNotifier::Notifier = double('ExceptionNotifier::Notifier')
+    end
+
+    after do
+      Object.send(:remove_const, "ExceptionNotifier")
+    end
+
+    it "notifies ExceptionNotifier" do
+      mail = double('mail')
+      mail.should_receive(:deliver).and_return(nil)
+      ::ExceptionNotifier::Notifier.should_receive(:background_exception_notification).with(TEST_EXCEPTION, :data => { :message => { :b => 2 } }).and_return(mail)
+      Component.new.invoke_exception(:b => 2)
+    end
+  end
+
+  describe "with fake Exceptional" do
+    before do
+      ::Exceptional = Class.new do
+
+        def self.context(msg)
+          @msg = msg
+        end
+
+        def self.check_context
+          @msg
+        end
+      end
+
+      ::Exceptional::Config = double('Exceptional::Config')
+      ::Exceptional::Remote = double('Exceptional::Remote')
+      ::Exceptional::ExceptionData = double('Exceptional::ExceptionData')
+    end
+
+    after do
+      Object.send(:remove_const, "Exceptional")
+    end
+
+    it "notifies Exceptional" do
+      ::Exceptional::Config.should_receive(:should_send_to_api?).and_return(true)
+      exception_data = double('exception_data')
+      ::Exceptional::Remote.should_receive(:error).with(exception_data).and_return(nil)
+      ::Exceptional::ExceptionData.should_receive(:new).with(TEST_EXCEPTION).and_return(exception_data)
+      Component.new.invoke_exception(:c => 3)
+      expect(::Exceptional.check_context).to eq({:c => 3}) # did not record arguments properly
+    end
+  end
+
+end

data/spec/lib/creeper/processor_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+require 'creeper/processor'
+
+describe Creeper::Processor do
+
+  context 'with mock setup' do
+
+    before do
+      stub_const('TestException', Class.new(StandardError))
+      stub_const('TEST_EXCEPTION', TestException.new("kerboom!"))
+    end
+
+    let(:boss)      { double('boss') }
+    let(:processor) { ::Creeper::Processor.new(boss) }
+
+    let(:job)  { double('job') }
+    let(:conn) { double('conn') }
+
+    before do
+      $invokes = 0
+      Celluloid.logger = nil
+      Creeper.redis = REDIS
+    end
+
+    class MockWorker
+      include Creeper::Worker
+      def perform(args)
+        raise TEST_EXCEPTION if args == 'boom'
+        args.pop if args.is_a? Array
+        $invokes += 1
+      end
+    end
+
+    it 'processes as expected' do
+      msg = Creeper.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+
+      conn.should_receive(:close).and_return(true)
+      job.should_receive(:id).and_return(1)
+      job.should_receive(:delete).and_return(true)
+      boss.should_receive(:processor_done!).with(processor).and_return(nil)
+
+      processor.process(msg, 'default', job, conn)
+      expect($invokes).to eq(1)
+    end
+
+    it 'passes exceptions to ExceptionHandler' do
+      msg = Creeper.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+
+      conn.should_receive(:close).and_return(true)
+      job.should_receive(:id).and_return(1)
+      job.should_receive(:bury).and_return(true)
+
+      expect do
+        processor.process(msg, 'default', job, conn)
+      end.to raise_error(TestException)
+
+      expect($invokes).to eq(0)
+    end
+
+    it 're-raises exceptions after handling' do
+      msg = Creeper.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+      re_raise = false
+
+      conn.should_receive(:close).and_return(true)
+      job.should_receive(:id).and_return(1)
+      job.should_receive(:bury).and_return(true)
+
+      begin
+        processor.process(msg, 'default', job, conn)
+      rescue TestException
+        re_raise = true
+      end
+
+      expect(re_raise).to be_true # does not re-raise exceptions after handling
+    end
+
+    it 'does not modify original arguments' do
+      msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
+      msgstr = Creeper.dump_json(msg)
+
+      conn.should_receive(:close).and_return(true)
+      job.should_receive(:id).and_return(1)
+      job.should_receive(:delete).and_return(true)
+      boss.should_receive(:processor_done!).with(processor).and_return(nil)
+
+      processor.process(msgstr, 'default', job, conn)
+      expect(msg['args']).to eq([['myarg']])
+    end
+
+  end
+
+end

data/spec/lib/creeper/testing_spec.rb

@@ -0,0 +1,105 @@
+require 'spec_helper'
+require 'creeper'
+require 'creeper/worker'
+require 'active_record'
+require 'action_mailer'
+require 'creeper/rails'
+require 'creeper/extensions/action_mailer'
+require 'creeper/extensions/active_record'
+
+Creeper.hook_rails!
+
+describe Creeper::Worker do
+
+  context 'creeper testing' do
+    class PerformError < RuntimeError; end
+
+    class DirectWorker
+      include Creeper::Worker
+      def perform(a, b)
+        a + b
+      end
+    end
+
+    class EnqueuedWorker
+      include Creeper::Worker
+      def perform(a, b)
+        a + b
+      end
+    end
+
+    class StoredWorker
+      include Creeper::Worker
+      def perform(error)
+        raise PerformError if error
+      end
+    end
+
+    class FooMailer < ActionMailer::Base
+      def bar(str)
+        str
+      end
+    end
+
+    class FooModel < ActiveRecord::Base
+      def bar(str)
+        str
+      end
+    end
+
+    before do
+      load 'creeper/testing.rb'
+    end
+
+    after do
+      # Undo override
+      Creeper::Worker::ClassMethods.class_eval do
+        remove_method :client_push
+        alias_method :client_push, :client_push_old
+        remove_method :client_push_old
+      end
+    end
+
+    it 'stubs the async call' do
+      expect(DirectWorker.jobs.size).to eq(0)
+      expect(DirectWorker.perform_async(1, 2)).to be_true
+      expect(DirectWorker.jobs.size).to eq(1)
+      expect(DirectWorker.perform_in(10, 1, 2)).to be_true
+      expect(DirectWorker.jobs.size).to eq(2)
+      expect(DirectWorker.perform_at(10, 1, 2)).to be_true
+      expect(DirectWorker.jobs.size).to eq(3)
+      expect(DirectWorker.jobs.last['at']).to be_within(0.1).of(10.seconds.from_now.to_f)
+    end
+
+    it 'stubs the delay call on mailers' do
+      expect(Creeper::Extensions::DelayedMailer.jobs.size).to eq(0)
+      FooMailer.delay.bar('hello!')
+      expect(Creeper::Extensions::DelayedMailer.jobs.size).to eq(1)
+    end
+
+    it 'stubs the delay call on models' do
+      expect(Creeper::Extensions::DelayedModel.jobs.size).to eq(0)
+      FooModel.delay.bar('hello!')
+      expect(Creeper::Extensions::DelayedModel.jobs.size).to eq(1)
+    end
+
+    it 'stubs the enqueue call' do
+      expect(EnqueuedWorker.jobs.size).to eq(0)
+      expect(Creeper::Client.enqueue(EnqueuedWorker, 1, 2)).to be_true
+      expect(EnqueuedWorker.jobs.size).to eq(1)
+    end
+
+    it 'executes all stored jobs' do
+      expect(StoredWorker.perform_async(false)).to be_true
+      expect(StoredWorker.perform_async(true)).to be_true
+
+      expect(StoredWorker.jobs.size).to eq(2)
+      expect do
+        StoredWorker.drain
+      end.to raise_error(PerformError)
+      expect(StoredWorker.jobs.size).to eq(0)
+    end
+
+  end
+
+end

data/spec/lib/creeper_spec.rb

@@ -1,131 +1,65 @@
 require 'spec_helper'
+require 'creeper/cli'
 
 describe Creeper do
 
-  after :each do
-    Creeper.clear!
-  end
-  
-  it "work a job and do it up" do
-    val = rand(999999)
-    Creeper.job('my.job') { |args| $result = args['val'] }
-    Creeper.enqueue('my.job', :val => val)
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    val.should == $result
-  end
-
-  it "invoke error handler when defined" do
-    with_an_error_handler
-    Creeper.job('my.job') { |args| fail }
-    Creeper.enqueue('my.job', :foo => 123)
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    $handled.should_not == nil
-    'my.job'.should == $job_name
-    {'foo' => 123}.should ==  $job_args
-  end
-
-  it "should be compatible with legacy error handlers" do
-    exception = StandardError.new("Oh my, the job has failed!")
-    Creeper.error { |e| $handled = e }
-    Creeper.job('my.job') { |args| raise exception }
-    Creeper.enqueue('my.job')
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    exception.should == $handled
-  end
-
-  it "continue working when error handler not defined" do
-    Creeper.error { |e| $handled = false }
-    Creeper.job('my.job') { fail }
-    Creeper.enqueue('my.job')
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    false.should == $handled
-  end
-
-  it "exception raised one second before beanstalk ttr reached" do
-    with_an_error_handler
-    Creeper.job('my.job') { sleep(3); $handled = "didn't time out" }
-    Creeper.enqueue('my.job', {}, :ttr => 2)
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    $handled.should == "didn't time out"
-  end
+  it 'is able to process a real job' do
+    pending 'still need to figure out how to test the daemon'
+    let(:cli) { Creeper::CLI.instance }
 
-  it "before filter gets run first" do
-    Creeper.before { |name| $flag = "i_was_here" }
-    Creeper.job('my.job') { |args| $handled = ($flag == 'i_was_here') }
-    Creeper.enqueue('my.job')
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    true.should == $handled
-  end
+    before do
+      $rd, $wr = IO.pipe
+      $rd.sync, $wr.sync = true
+      if @pid = fork
+        # parent
+        $wr.close
+      else
+        # child
+        $rd.close
+        cli.parse(['creeper', '-r', './spec/support/fake_env.rb'])
+        sleep 5
+        begin
+          cli.run
+        rescue SystemExit
+          Process.kill(:KILL, $$)
+        end
+      end
+    end
 
-  it "before filter passes the name of the job" do
-    Creeper.before { |name| $jobname = name }
-    Creeper.job('my.job') { true }
-    Creeper.enqueue('my.job')
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    'my.job'.should == $jobname
-  end
+    after do
+      if @pid
+        Process.kill(:TERM, @pid)
+        done = nil
+        # 8.times do
+        #   begin
+        #     Process.getpgid(@pid)
+        #   rescue Errno::ESRCH
+        #     done = true
+        #     break
+        #   end
+        #   sleep 1
+        # end
+        # expect(done).to be_true
+        sleep 1
+        Process.kill(:KILL, @pid)
+        child_pid, status = Process.waitpid2(-1, Process::WNOHANG)
+        expect(child_pid).to eq(@pid)
+        # expect(status).to be_success
 
-  it "before filter can pass an instance var" do
-    Creeper.before { |name| @foo = "hello" }
-    Creeper.job('my.job') { |args| $handled = (@foo == "hello") }
-    Creeper.enqueue('my.job')
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    true.should == $handled
-  end
-
-  it "before filter invokes error handler when defined" do
-    with_an_error_handler
-    Creeper.before { |name| fail }
-    Creeper.job('my.job') {  }
-    Creeper.enqueue('my.job', :foo => 123)
-    w = Creeper::Worker.new
-    w.stub(:exception_message)
-    w.stub(:log)
-    w.prepare
-    w.work_one_job
-    $handled.should_not == nil
-    'my.job'.should ==  $job_name
-    {'foo' => 123}.should == $job_args
-  end
+        $rd.close rescue nil
+        $wr.close rescue nil
+      end
+    end
 
-  def with_an_error_handler
-    Creeper.error do |e, job_name, args|
-      $handled = e.class
-      $job_name = job_name
-      $job_args = args
+    it 'is able to process a real job' do
+      pending 'still need to figure out how to test the daemon'
+      RealWorker.perform_async(1, 2)
+      result = nil
+      Timeout::timeout(10) do
+        result = $rd.read
+      end
+      expect(result).to eq('[1,2]')
     end
   end
-  
+
 end