stormtroopers 0.0.2 → 0.1.0

data/README.md

@@ -1,6 +1,10 @@
-# Stormtroopers [![Build Status](https://secure.travis-ci.org/socialreferral/stormtroopers.png)](http://travis-ci.org/socialreferral/stormtroopers)
+# Stormtroopers
 
-Stormtroopers is a jruby execution environment for delayed jobs, Work In Progress!
+Stormtroopers is a JRuby execution environment for background tasks using threads. It can use different backends, currently it comes with a DelayedJob (light) backend only. **NOTE: While the basic functionality works and is under test, Stormtroopers is still a Work In Progress!**
+
+## Prerequisites
+
+JRuby 1.7.0+ in 1.9 mode. Your code and the libraries you are using must be threadsafe. If you're running with Rails in development mode be sure that you do not run more than 1 thread at a time!
 
 ## Installation
 
@@ -18,7 +22,37 @@ Or install it yourself as:
 
 ## Usage
 
-configure using a config file 'stormtroopers.yml' and run with `bundle exec stormtroopers`
+Stormtroopers are organized in Armies, each Army has a Factory that produces Troopers. Each Trooper runs a job inside it's own Thread (which is disposed of after job completion or failure).
+
+Configure Stormtroopers using a config file 'config/stormtroopers.yml' and execute it using 'bundle exec stormtroopers' from the root directory of your application, here's an example that specifies workers for different DelayedJob queues:
+
+```yaml
+armies:
+  - factory:
+      type: :delayed_job
+      queues:
+        - emails
+        - posts
+    max_threads: 10
+    name: "Comms"
+  - factory:
+      type: :delayed_job
+      queues:
+        - calculation
+        - graphs
+    max_threads: 5
+    name: "Math"
+```
+
+The above configuration specifies two armies, each with a DelayedJob factory to produce its Troopers.
+
+Each Army can be given a set of options:
+
+- factory: options that are passed on to the Army's Trooper producing Factory
+- max_threads: the maximum number of Troopers for this Army that may be concurrently running a job at any time
+- name: the Army's name, this is used when logging
+
+You set the Army's Factory with the factory type or class option. When using type this is translated into a builtin class, when using class you can specify the name of whatever class you like (it needs to implement the appropriate interface to be able to work of course, have a look through the source code and specs to figure out what you need to do). Besides specifying the type or class, you can also specify the Factory's name (used for logging purposes), if you don't give the Factory a name then the Army's name is propagated to the Factory. The factory options also include specific settings for the chosen backend, for DelayedJob you can specify the queues that jobs may be picked up from. If you don't specify queues then the factory will pick up jobs from all queues.
 
 ## Contributing
 

data/lib/stormtroopers/army.rb

@@ -4,33 +4,32 @@ module Stormtroopers
 
     def initialize(config)
       @name = config[:name] || factory_class(config).name
-      @factory = factory_class(config).new(config[:factory])
+      @factory = factory_class(config).new({name: config[:name]}.merge(config[:factory]))
       @max_threads = config[:max_threads] || 1
       @threads = []
     end
 
     def factory_class(config)
-      @factory_class ||= begin
-        raise ArgumentError, "Factory class or type must be defined" if config[:factory][:class].blank? && config[:factory][:type].blank?
-        class_name ||= config[:factory].delete(:class)
-        class_name ||= "stormtroopers/#{config[:factory].delete(:type)}_factory".camelize
-        class_name.constantize
-      end
+      @factory_class ||= self.class.factory_class(config)
     end
 
     def manage
       cleanup
       if threads.count < max_threads
         if trooper = factory.produce
-          threads << Thread.new do
-            begin
-              trooper.run
-            ensure
-              if defined?(::Mongoid)
-                ::Mongoid::IdentityMap.clear
-                ::Mongoid.session(:default).disconnect
-              end
-            end
+          run_trooper(trooper)
+        end
+      end
+    end
+
+    def run_trooper(trooper)
+      threads << Thread.new do
+        begin
+          trooper.run
+        ensure
+          if defined?(::Mongoid)
+            ::Mongoid::IdentityMap.clear
+            ::Mongoid.session(:default).disconnect
           end
         end
       end
@@ -41,14 +40,21 @@ module Stormtroopers
       threads.each(&:join)
     end
 
+    def cleanup
+      threads.reject!{ |thread| !thread.alive? }
+    end
+
     def logger
       Stormtroopers::Manager.logger
     end
 
-    private
-
-    def cleanup
-      threads.reject!{ |thread| !thread.alive? }
+    class << self
+      def factory_class(config)
+        raise ArgumentError, "Factory class or type must be defined" if config[:factory][:class].blank? && config[:factory][:type].blank?
+        class_name ||= config[:factory].delete(:class)
+        class_name ||= "stormtroopers/#{config[:factory].delete(:type)}_factory".camelize
+        class_name.constantize
+      end
     end
   end
 end

data/lib/stormtroopers/trooper.rb

@@ -16,20 +16,18 @@ module Stormtroopers
     end
 
     def exception(exception)
-      raise
-      # Empty hook for overriding
+      # Hook for to override handling exceptions
+      raise exception
     end
 
     def run
-      before_execution
+      before_run
       task.call
-      after_execution
+      after_run
     rescue => e
       exception(e)
     end
 
-    private
-
     def logger
       Manager.logger
     end

data/lib/stormtroopers/trooper/delayed_job.rb

@@ -15,18 +15,19 @@ module Stormtroopers
       reschedule
     end
 
-    private
-
     def reschedule
       if (job.attempts += 1) < max_attempts(job)
-        time ||= job.reschedule_at
-        job.run_at = time
+        job.run_at = job.reschedule_at
         job.unlock
         job.save!
       else
-        logger.error "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+        logger.error("PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.")
         job.hook(:failure)
       end
     end
+
+    def max_attempts(job)
+      job.max_attempts || Delayed::Worker.max_attempts
+    end
   end
-end
+end

data/lib/stormtroopers/version.rb

@@ -1,3 +1,3 @@
 module Stormtroopers
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/spec/stormtroopers/army_spec.rb

@@ -0,0 +1,122 @@
+require "stormtroopers"
+
+describe Stormtroopers::Army do
+  let(:army) do
+    Stormtroopers::Army.new(name: "Dad's Army", max_threads: 2, factory: {name: "Dad's Factory", type: :dummy})
+  end
+
+  describe "#initialize" do
+    it "extracts the name from the options" do
+      army.name.should eq("Dad's Army")
+    end
+
+    it "extracts the max_threads from the options" do
+      army.max_threads.should eq(2)
+    end
+
+    it "instatiates a factory using #factory_class" do
+      factory_class = stub
+      Stormtroopers::Army.any_instance.should_receive(:factory_class).and_return(factory_class)
+      factory_instance = stub
+      factory_class.should_receive(:new).with(name: "Dad's Factory", type: :dummy).and_return(factory_instance)
+      army.factory.should equal(factory_instance)
+    end
+
+    it "when there is no factory name, the army name is used as the factory name" do
+      factory_class = stub
+      Stormtroopers::Army.any_instance.should_receive(:factory_class).and_return(factory_class)
+      factory_instance = stub
+      factory_class.should_receive(:new).with(name: "Dad's Army", type: :dummy).and_return(factory_instance)
+      army = Stormtroopers::Army.new(name: "Dad's Army", max_threads: 2, factory: {type: :dummy})
+      army.factory.should equal(factory_instance)
+    end
+  end
+
+  describe "#cleanup" do
+    it "removes threads that are not alive" do
+      live_thread_stub = stub(:alive? => true)
+      dead_thread_stub = stub(:alive? => false)
+      army.stub(:threads).and_return([live_thread_stub, dead_thread_stub])
+      army.cleanup
+      army.threads.should eq([live_thread_stub])
+    end
+  end
+
+  describe "#manage" do
+    it "cleans up" do
+      army.should_receive(:cleanup)
+      army.manage
+    end
+
+    it "produces a trooper and runs it if not at max_threads" do
+      trooper = stub
+      army.factory.should_receive(:produce).and_return(trooper)
+      army.should_receive(:run_trooper).with(trooper)
+      army.manage
+    end
+
+    it "does not produce a trooper if at max_threads" do
+      thread_stub = stub(:alive? => true)
+      army.stub(:threads).and_return([thread_stub, thread_stub])
+      army.factory.should_not_receive(:produce)
+      army.manage
+    end
+  end
+
+  describe "#run_trooper" do
+    it "creates a new thread and runs the trooper in it" do
+      trooper = mock
+      trooper.should_receive(:run)
+      Thread.should_receive(:new).and_yield
+      army.run_trooper(trooper)
+    end
+
+    it "cleans up the Mongoid environment if Mongoid is defined" do
+      stub_const("Mongoid", Class.new)
+      stub_const("Mongoid::IdentityMap", Class.new)
+      Mongoid::IdentityMap.should_receive(:clear)
+      mongoid_session = stub
+      Mongoid.should_receive(:session).with(:default).and_return(mongoid_session)
+      mongoid_session.should_receive(:disconnect)
+      trooper = mock
+      trooper.should_receive(:run)
+      Thread.should_receive(:new).and_yield
+      army.run_trooper(trooper)
+    end
+  end
+
+  describe "#finish" do
+    it "calls join on all threads" do
+      thread1 = stub
+      thread1.should_receive(:join)
+      thread2 = stub
+      thread2.should_receive(:join)
+      army.stub(:threads).and_return([thread1, thread2])
+      army.finish
+    end
+  end
+
+  describe "#logger" do
+    it "takes the logger from Stormtroopers::Manager" do
+      logger = stub
+      Stormtroopers::Manager.stub(:logger).and_return(logger)
+      army.logger.should equal(logger)
+    end
+  end
+
+  describe ".factory_class" do
+    it "returns the class specified in the options" do
+      stub_const("MyFactory", Class.new)
+      Stormtroopers::Army.factory_class(factory: {class: "MyFactory"}).should equal(MyFactory)
+    end
+
+    it "returns a builtin factory class when type is specified in the options" do
+      Stormtroopers::Army.factory_class(factory: {type: :dummy}).should equal(Stormtroopers::DummyFactory)
+      Stormtroopers::Army.factory_class(factory: {type: :delayed_job}).should equal(Stormtroopers::DelayedJobFactory)
+    end
+
+    it "raises an ArgumentError when no type or class is specified" do
+      expect { Stormtroopers::Army.factory_class(factory: {}) }.to raise_error(ArgumentError)
+    end
+  end
+end

data/spec/stormtroopers/factory/delayed_job_spec.rb

@@ -0,0 +1,42 @@
+require "stormtroopers/factory/delayed_job"
+
+describe Stormtroopers::DelayedJobFactory do
+  let(:factory) { Stormtroopers::DelayedJobFactory.new }
+  let(:worker) { stub }
+
+  before(:each) do
+    stub_const("Delayed::Worker", Class.new)
+    Delayed::Worker.stub(:new).and_return(worker)
+    worker.stub(:name=).and_return(nil)
+    worker.stub(:name).and_return(nil)
+    stub_const("Delayed::Job", Class.new)
+    Delayed::Job.stub(:reserve)
+  end
+
+  describe "#produce" do
+    it "instantiates a new worker" do
+      Delayed::Worker.should_receive(:new).and_return(worker)
+      factory.produce
+    end
+
+    it "assigns the worker a name with random value" do
+      Time.stub_chain(:now, :utc, :to_f).and_return("1234")
+      factory.should_receive(:rand).and_return(500)
+      worker.should_receive(:name=).with("rand 1234 500")
+      factory.produce
+    end
+
+    it "reserves a job and instantiates a trooper" do
+      job = stub.as_null_object
+      trooper = stub
+      Delayed::Job.should_receive(:reserve).and_return(job)
+      Stormtroopers::DelayedJobTrooper.should_receive(:new).and_return(trooper)
+      factory.produce.should equal(trooper)
+    end
+
+    it "does not instantiate a trooper if it doesn't find a job to reserve" do
+      Delayed::Job.should_receive(:reserve).and_return(nil)
+      factory.produce.should be_nil
+    end
+  end
+end

data/spec/stormtroopers/factory_spec.rb

@@ -0,0 +1,28 @@
+require "stormtroopers/factory"
+
+describe Stormtroopers::Factory do
+  describe "#options" do
+    it "exposes the options passed in during #initialize" do
+      options = {key1: "value1", key2: "value2"}
+      factory = Stormtroopers::Factory.new(options)
+      factory.options.should equal(options)
+    end
+  end
+
+  describe "#produce" do
+    it "is not implemented" do
+      factory = Stormtroopers::Factory.new
+      expect { factory.produce }.to raise_error(NotImplementedError)
+    end
+  end
+
+  describe "#logger" do
+    it "uses the Stormtroopers::Manager#logger" do
+      stub_const("Stormtroopers::Manager", Class.new)
+      logger = stub
+      Stormtroopers::Manager.stub(:logger).and_return(logger)
+      factory = Stormtroopers::Factory.new
+      factory.logger.should equal(logger)
+    end
+  end
+end

data/spec/stormtroopers/trooper/delayed_job_spec.rb

@@ -0,0 +1,56 @@
+require "stormtroopers/trooper/delayed_job"
+
+describe Stormtroopers::DelayedJobTrooper do
+  let(:job) { stub.as_null_object }
+  let(:trooper) { Stormtroopers::DelayedJobTrooper.new(job) }
+
+  describe "#initialize" do
+    it "accepts a job and exposes it as #job" do
+      job = stub.as_null_object
+      trooper = Stormtroopers::DelayedJobTrooper.new(job)
+      trooper.job.should equal(job)
+    end
+  end
+
+  describe "#run" do
+    context "a succesful job" do
+      it "calls invoke job.invoke_job and then job.destroy" do
+        job.should_receive(:invoke_job)
+        job.should_receive(:destroy)
+        trooper.run
+      end
+    end
+
+    context "a failing job" do
+      it "sets job.last_error and reschedules the job" do
+        job.should_receive(:invoke_job) { raise "Oops!" }
+        job.should_not_receive(:destroy)
+        job.should_receive(:last_error=).with(match("Oops!"))
+        trooper.should_receive(:reschedule)
+        trooper.run
+      end
+    end
+  end
+
+  describe "#reschedule" do
+    context "job has not reached max attempts" do
+      it "should reschedule and unlock" do
+        job.stub(:attempts).and_return(1)
+        job.stub(:max_attempts).and_return(3)
+        job.should_receive(:run_at=)
+        job.should_receive(:unlock)
+        job.should_receive(:save!)
+        trooper.reschedule
+      end
+    end
+
+    context "job has reached max attempts" do
+      it "should fail the job" do
+        job.stub(:attempts).and_return(3)
+        job.stub(:max_attempts).and_return(3)
+        job.should_receive(:hook).with(:failure)
+        trooper.reschedule
+      end
+    end
+  end
+end

data/spec/stormtroopers/trooper_spec.rb

@@ -0,0 +1,56 @@
+require "stormtroopers/trooper"
+
+describe Stormtroopers::Trooper do
+  before(:each) do
+    stub_const("Stormtroopers::Manager", Class.new)
+    logger = stub
+    Stormtroopers::Manager.stub(:logger).and_return(logger)
+  end
+
+  describe "#initialize" do
+    it "it accepts parameters and exposes them through #parameters" do
+      parameters = {key1: "value1", key2: "value2"}
+      trooper = Stormtroopers::Trooper.new(parameters)
+      trooper.parameters.should equal(parameters)
+    end
+
+    it "accepts a task block and exposes it through #task" do
+      task = lambda { puts "This is a task" }
+      trooper = Stormtroopers::Trooper.new({}, &task)
+      trooper.task.should equal(task)
+    end
+  end
+
+  describe "#run" do
+    let(:task) { lambda { puts "This is a task" } }
+    let(:trooper) { Stormtroopers::Trooper.new({}, &task) }
+
+    it "calls the before_run hook" do
+      trooper.should_receive(:before_run)
+      trooper.run
+    end
+
+    it "calls call on the task" do
+      task.should_receive(:call)
+      trooper.run
+    end
+
+    it "calls the after_run hook" do
+      trooper.should_receive(:after_run)
+      trooper.run
+    end
+
+    it "when the task raises an exception the exception hook is called" do
+      task.stub(:call) { raise "Oops" }
+      trooper.should_receive(:exception)
+      trooper.run
+    end
+  end
+
+  describe "#logger" do
+    it "uses the Stormtroopers::Manager#logger" do
+      trooper = Stormtroopers::Trooper.new
+      trooper.logger.should equal(Stormtroopers::Manager.logger)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stormtroopers
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease:
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-12 00:00:00.000000000 Z
+date: 2012-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -76,7 +76,12 @@ files:
 - lib/stormtroopers/trooper/delayed_job.rb
 - lib/stormtroopers/trooper/dummy.rb
 - lib/stormtroopers/version.rb
+- spec/stormtroopers/army_spec.rb
+- spec/stormtroopers/factory/delayed_job_spec.rb
+- spec/stormtroopers/factory_spec.rb
 - spec/stormtroopers/manager_spec.rb
+- spec/stormtroopers/trooper/delayed_job_spec.rb
+- spec/stormtroopers/trooper_spec.rb
 - stormtroopers.gemspec
 - vagrant-provision
 homepage: http://github.com/socialreferral/stormtroopers
@@ -106,4 +111,9 @@ signing_key:
 specification_version: 3
 summary: Execute delayed jobs in a threaded jruby environment
 test_files:
+- spec/stormtroopers/army_spec.rb
+- spec/stormtroopers/factory/delayed_job_spec.rb
+- spec/stormtroopers/factory_spec.rb
 - spec/stormtroopers/manager_spec.rb
+- spec/stormtroopers/trooper/delayed_job_spec.rb
+- spec/stormtroopers/trooper_spec.rb