girl_friday 0.9.4 → 0.9.5

data/.gitignore

@@ -2,4 +2,5 @@
 .bundle
 Gemfile.lock
 pkg/*
-.rbxdb/
+rbxdb/
+*.rdb

data/.rvmrc

@@ -1,3 +1,3 @@
 rvm use jruby@gf --create
-export RBXOPT=-Xrbc.db=~/.rbxdb
-export JRUBY_OPTS='--1.9 -X+O -J-Djruby.launch.inproc=false'
+export RBXOPT=-Xrbc.db=".rbxdb"
+export JRUBY_OPTS='--1.9'

data/.travis.yml

@@ -5,4 +5,4 @@ rvm:
 branches:
   only:
     - master
-env: JRUBY_OPTS='--1.9 -X+O -J-Djruby.launch.inproc=false'
+env: JRUBY_OPTS='--1.9'

data/Gemfile

@@ -6,4 +6,5 @@ gemspec
 # Needed for testing only!
 gem 'minitest'
 gem 'redis'
+gem 'connection_pool'
 gem 'flexmock-minitest'

data/History.md

@@ -1,6 +1,16 @@
 Changes
 ================
 
+0.9.5
+---------
+
+* Refactor thread usage so Queues can be properly shutdown and GC'd [GH-30]
+* Use WeakRefs instead of ObjectSpace, as that plays better on JRuby.
+* Can now pass a [connection\_pool](https://github.com/mperham/connection_pool) in as a Redis instance.
+* Switch Redis.new to Redis.connect so a :url option can be passed in.
+  Nice for using on Heroku with Redis To Go.
+* Allow stacking of error handlers, fixes GH-11
+
 0.9.4
 ---------
 

data/README.md

@@ -32,20 +32,30 @@ In your Rails app, create a `config/initializers/girl_friday.rb` which defines y
     EMAIL_QUEUE = GirlFriday::WorkQueue.new(:user_email, :size => 3) do |msg|
       UserMailer.registration_email(msg).deliver
     end
+
     IMAGE_QUEUE = GirlFriday::WorkQueue.new(:image_crawler, :size => 7) do |msg|
       ImageCrawler.process(msg)
     end
 
-    SCRAPE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :host => 'host' }] do |msg|
+    SCRAPE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :host => 'host' }]) do |msg|
       Page.scrape(msg)
     end
 
-    TRANSCODE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => $redis }] do |msg|
+    TRANSCODE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => $redis }]) do |msg|
       VideoProcessor.transcode(msg)
     end
 
 :size is the number of workers to spin up and defaults to 5.  Keep in mind, ActiveRecord defaults to a connection pool size of 5 so if your workers are accessing the database you'll want to ensure that the connection pool is large enough by modifying `config/database.yml`.
 
+You can use a connection pool to share a set of Redis connections with
+other threads and GirlFriday queues using the `connection\_pool` gem:
+
+    require 'connection_pool'
+    redis_pool = ConnectionPool.new(:size => 5, :timeout => 5) { Redis.new }
+    CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis_pool}]) do |msg|
+      Filter.clean(msg)
+    end
+
 In your controller action or model, you can call `#push(msg)`
 
     EMAIL_QUEUE.push(:email => @user.email, :name => @user.name)
@@ -56,6 +66,11 @@ Your message processing block should **not** access any instance data or variabl
 
 You can call `GirlFriday::WorkQueue.immediate!` to process jobs immediately, which is helpful when testing. `GirlFriday::WorkQueue.queue!` will revert this & jobs will be processed by actors.
 
+Queues are not garbage collected until they are shutdown, even if you
+have no reference to them.  Make sure you call `WorkQueue#shutdown` if you are
+dynamically creating them so you don't leak memory.  `GirlFriday.shutdown!` will shut down all
+running queues in the process.
+
 More Detail
 --------------------
 

data/Rakefile

@@ -5,4 +5,4 @@ Rake::TestTask.new(:test) do |test|
   test.pattern = 'test/**/test_*.rb'
 end
 
-task :default => :test
+task :default => :test

data/examples/backends/mongo_persistence.rb

@@ -0,0 +1,55 @@
+require 'mongo'
+
+# Add index on created_at
+
+module GirlFriday
+  module Store
+    class Mongo
+      def initialize(name, options)
+        @opts = options
+        @key = "girl_friday-#{name}-#{environment}"
+      end
+
+      def push(work)
+        val = Marshal.dump(work)
+        collection.insert({ "work" => val })
+      end
+      alias_method :<<, :push
+
+      def pop
+        begin
+          val = collection.find_and_modify(:sort => [['$natural', 1]],
+                                           :remove => true)
+          Marshal.load(val["work"]) if val
+        rescue
+          # rescuing
+          # Database command 'findandmodify' failed: {"errmsg"=>"No matching object found", "ok"=>0.0}
+        end
+      end
+
+      def size
+        collection.size
+      end
+
+      private
+
+      def environment
+        ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'none'
+      end
+
+      def connection
+        @connection ||= (@opts.delete(:mongo) || ::Mongo::Connection.new("localhost", 27017, :pool_size => 5))
+      end
+
+      def db
+        db = @opts.delete(:db) || "girl_friday"
+        @db ||= connection[db]
+      end
+
+      def collection
+        @collection ||= db[@key]
+      end
+
+    end
+  end
+end

data/examples/backends/using_mongo.rb

@@ -0,0 +1,22 @@
+require 'rubygems' # for rbx
+here = File.dirname(__FILE__)
+$LOAD_PATH.unshift File.expand_path(here + '/../../lib')
+$LOAD_PATH.unshift here
+require 'girl_friday'
+require 'mongo_persistence'
+
+class Foo
+  def initialize(msg)
+    puts msg.inspect
+  end
+end
+
+PUTS_QUEUE = GirlFriday::WorkQueue.new(:puts_mongo, :store => GirlFriday::Store::Mongo) do |msg|
+  Foo.new(msg)
+end
+
+1.upto(100) do |i|
+  PUTS_QUEUE << { :what => i }
+end
+
+sleep 2

data/lib/girl_friday/batch.rb

@@ -25,6 +25,7 @@ module GirlFriday
     def results(timeout=nil)
       @lock.synchronize do
         @condition.wait(@lock, timeout) if @complete != @size
+        @queue.shutdown
         @results
       end
     end

data/lib/girl_friday/error_handler.rb

@@ -1,22 +1,25 @@
 module GirlFriday
-  class ErrorHandler
-    def handle(ex)
-      $stderr.puts(ex)
-      $stderr.puts(ex.backtrace.join("\n"))
-    end
-    
+  module ErrorHandler
+
     def self.default
-      defined?(HoptoadNotifier) ? Hoptoad : self
+      handlers = [Stderr]
+      handlers << Hoptoad if defined?(HoptoadNotifier)
+      handlers
+    end
+
+    class Stderr
+      def handle(ex)
+        $stderr.puts(ex)
+        $stderr.puts(ex.backtrace.join("\n"))
+      end
     end
-  end
-end
 
-module GirlFriday
-  class ErrorHandler
     class Hoptoad
       def handle(ex)
         HoptoadNotifier.notify_or_ignore(ex)
       end
     end
+    Airbrake = Hoptoad
+
   end
-end
+end

data/lib/girl_friday/persistence.rb

@@ -48,7 +48,7 @@ module GirlFriday
       end
 
       def redis
-        @redis ||= (@opts.delete(:redis) || ::Redis.new(*@opts))
+        @redis ||= (@opts.delete(:redis) || ::Redis.connect(*@opts))
       end
     end
   end

data/lib/girl_friday/version.rb

@@ -1,3 +1,3 @@
 module GirlFriday
-  VERSION = "0.9.4"
+  VERSION = "0.9.5"
 end

data/lib/girl_friday/work_queue.rb

@@ -11,14 +11,16 @@ module GirlFriday
       @name = name.to_s
       @size = options[:size] || 5
       @processor = block
-      @error_handler = (options[:error_handler] || ErrorHandler.default).new
+      @error_handlers = (Array(options[:error_handler]) || ErrorHandler.default).map(&:new)
 
       @shutdown = false
       @busy_workers = []
       @created_at = Time.now.to_i
       @total_processed = @total_errors = @total_queued = 0
       @persister = (options[:store] || Store::InMemory).new(name, (options[:store_config] || []))
+      @weakref = WeakRef.new(self)
       start
+      GirlFriday.add_queue @weakref
     end
 
     def self.immediate!
@@ -37,13 +39,8 @@ module GirlFriday
       result
     end
 
-    if defined?(Rails) && Rails.env.development?
-      Rails.logger.debug "[girl_friday] Starting in single-threaded mode for Rails autoloading compatibility" if Rails.logger
-      alias_method :push_async, :push_immediately
-    else
-      def push_async(work, &block)
-        @supervisor << Work[work, block]
-      end
+    def push_async(work, &block)
+      @supervisor << Work[work, block]
     end
     alias_method :push, :push_async
     alias_method :<<, :push_async
@@ -88,18 +85,17 @@ module GirlFriday
       else
         @busy_workers.delete(who.this)
         ready_workers << who.this
-        shutdown_complete if @shutdown && @busy_workers.size == 0
       end
     rescue => ex
       # Redis network error?  Log and ignore.
-      @error_handler.handle(ex)
+      @error_handlers.each { |handler| handler.handle(ex) }
     end
 
     def shutdown_complete
       begin
         @when_shutdown.call(self) if @when_shutdown
       rescue Exception => ex
-        @error_handler.handle(ex)
+        @error_handlers.each { |handler| handler.handle(ex) }
       end
     end
 
@@ -115,7 +111,7 @@ module GirlFriday
       end
     rescue => ex
       # Redis network error?  Log and ignore.
-      @error_handler.handle(ex)
+      @error_handlers.each { |handler| handler.handle(ex) }
     end
 
     def ready_workers
@@ -133,39 +129,19 @@ module GirlFriday
       @supervisor = Actor.spawn do
         supervisor = Actor.current
         @work_loop = Proc.new do
-          loop do
+          while !@shutdown do
             work = Actor.receive
-            result = @processor.call(work.msg)
-            work.callback.call(result) if work.callback
-            supervisor << Ready[Actor.current]
+            if !@shutdown
+              result = @processor.call(work.msg)
+              work.callback.call(result) if work.callback
+              supervisor << Ready[Actor.current]
+            end
           end
         end
 
         Actor.trap_exit = true
         begin
-          loop do
-            Actor.receive do |f|
-              f.when(Ready) do |who|
-                on_ready(who)
-              end
-              f.when(Work) do |work|
-                on_work(work)
-              end
-              f.when(Shutdown) do |stop|
-                @shutdown = true
-                @when_shutdown = stop.callback
-                shutdown_complete if @shutdown && @busy_workers.size == 0
-              end
-              f.when(Actor::DeadActorError) do |exit|
-                # TODO Provide current message contents as error context
-                @total_errors += 1
-                @busy_workers.delete(exit.actor)
-                ready_workers << Actor.spawn_link(&@work_loop)
-                @error_handler.handle(exit.reason)
-              end
-            end
-          end
-
+          supervisor_loop
         rescue Exception => ex
           $stderr.print "Fatal error in girl_friday: supervisor for #{name} died.\n"
           $stderr.print("#{ex}\n")
@@ -185,6 +161,37 @@ module GirlFriday
       end
     end
 
+    def supervisor_loop
+      loop do
+        Actor.receive do |f|
+          f.when(Ready) do |who|
+            on_ready(who)
+          end
+          f.when(Work) do |work|
+            on_work(work)
+          end
+          f.when(Shutdown) do |stop|
+            @shutdown = true
+            @when_shutdown = stop.callback
+            @busy_workers.each { |w| w << stop }
+            ready_workers.each { |w| w << stop }
+            shutdown_complete
+            GirlFriday.remove_queue @weakref
+            return
+          end
+          f.when(Actor::DeadActorError) do |ex|
+            if !@shutdown
+              # TODO Provide current message contents as error context
+              @total_errors += 1
+              @busy_workers.delete(ex.actor)
+              ready_workers << Actor.spawn_link(&@work_loop)
+              @error_handlers.each { |handler| handler.handle(ex.reason) }
+            end
+          end
+        end
+      end
+    end
+
   end
   Queue = WorkQueue
 end

data/lib/girl_friday.rb

@@ -1,3 +1,4 @@
+require 'weakref'
 require 'thread'
 begin
   # Rubinius
@@ -16,12 +17,34 @@ require 'girl_friday/batch'
 
 module GirlFriday
 
+  @@lock = Mutex.new
+
+  def self.add_queue(ref)
+    @@lock.synchronize do
+      @queues ||= []
+      @queues.reject! { |q| !q.weakref_alive? }
+      @queues << ref
+    end
+  end
+
+  def self.remove_queue(ref)
+    @@lock.synchronize do
+      @queues.delete ref
+    end
+  end
+
   def self.queues
-    ObjectSpace.each_object(GirlFriday::WorkQueue).to_a
+    @queues || []
   end
 
   def self.status
-    queues.inject({}) { |memo, queue| memo.merge(queue.status) }
+    queues.inject({}) do |memo, queue|
+      begin
+        memo = memo.merge(queue.__getobj__.status)
+      rescue WeakRef::RefError
+      end
+      memo
+    end
   end
 
   ##
@@ -33,7 +56,7 @@ module GirlFriday
   # Note that shutdown! just works with existing queues.  If you create a
   # new queue, it will act as normal.
   def self.shutdown!(timeout=30)
-    qs = queues
+    qs = queues.select { |q| q.weakref_alive? }
     count = qs.size
 
     if count > 0
@@ -41,7 +64,15 @@ module GirlFriday
       var = ConditionVariable.new
 
       qs.each do |q|
-        q.shutdown do |queue|
+        next if !q.weakref_alive?
+        begin
+          q.__getobj__.shutdown do |queue|
+            m.synchronize do
+              count -= 1
+              var.signal if count == 0
+            end
+          end
+        rescue WeakRef::RefError
           m.synchronize do
             count -= 1
             var.signal if count == 0
@@ -52,19 +83,15 @@ module GirlFriday
       m.synchronize do
         var.wait(m, timeout)
       end
-      #puts "girl_friday shutdown complete"
     end
     count
   end
 
 end
 
-begin
-  ObjectSpace.each_object(GirlFriday).to_a
+
+unless defined?($testing)
   at_exit do
     GirlFriday.shutdown!
   end
-rescue RuntimeError
-  $stderr.puts "[warn] girl_friday will not shut down cleanly, pass -X+O to JRuby to enable ObjectSpace"
 end
-

data/test/helper.rb

@@ -1,19 +1,29 @@
 $testing = true
+puts RUBY_DESCRIPTION
+
+at_exit do
+  if Thread.list.size > 1
+    Thread.list.each do |thread|
+      next if thread.status == 'run'
+      puts "WARNING: lingering threads found.  All threads should be shutdown and garbage collected."
+      p [thread, thread['name']]
+#      puts thread.backtrace.join("\n")
+    end
+  end
+end
 
 # require 'simplecov'
 # SimpleCov.start do
 #   add_filter "/actor.rb"
 # end
 
-# rbx is 1.8-mode for another month...
 require 'rubygems'
+require 'minitest/spec'
 require 'minitest/autorun'
-require 'timed_queue'
+require 'connection_pool'
 require 'girl_friday'
 require 'flexmock/minitest'
 
-puts RUBY_DESCRIPTION
-
 class MiniTest::Unit::TestCase
 
   def async_test(time=0.5)

data/test/test_batch.rb

@@ -3,7 +3,7 @@ require 'helper'
 class TestBatch < MiniTest::Unit::TestCase
 
   def test_simple_batch_operation
-    work = [1] * 10
+    work = [0.5] * 10
     a = Time.now
     batch = GirlFriday::Batch.new(work, :size => 10) do |msg|
       sleep msg
@@ -14,10 +14,29 @@ class TestBatch < MiniTest::Unit::TestCase
     assert_in_delta(0.0, (b - a), 0.1)
 
     # asking for the results should block
-    results = batch.results(2.0)
+    results = batch.results(1.0)
     c = Time.now
-    assert_in_delta(1.0, (c - b), 0.1)
+    assert_in_delta(0.5, (c - b), 0.1)
+
     assert_equal 10, results.size
     assert_kind_of Time, results[0]
   end
+
+  def test_batch_timeout
+    work = [0.1] * 4
+    work[2] = 0.4
+    batch = GirlFriday::Batch.new(work, :size => 4) do |msg|
+      sleep msg
+      'x'
+    end
+    results = batch.results(0.3)
+    assert_equal 'x', results[0]
+    assert_equal 'x', results[1]
+    assert_nil results[2]
+    assert_equal 'x', results[3]
+
+    # Necessary to work around a Ruby 1.9.2 bug
+    # http://redmine.ruby-lang.org/issues/5342
+    sleep 0.1
+  end
 end

data/test/test_girl_friday.rb

@@ -2,198 +2,36 @@ require 'helper'
 
 class TestGirlFriday < MiniTest::Unit::TestCase
 
-  class TestErrorHandler
-    include MiniTest::Assertions
-  end
-
-  def test_should_process_messages
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('test') do |msg|
-        assert_equal 'foo', msg[:text]
-        cb.call
-      end
-      queue.push(:text => 'foo')
-    end
-  end
-
-  def test_should_handle_worker_error
-    async_test do |cb|
-      TestErrorHandler.send(:define_method, :handle) do |ex|
-        assert_equal 'oops', ex.message
-        assert_equal 'RuntimeError', ex.class.name
-        cb.call
-      end
-
-      queue = GirlFriday::WorkQueue.new('test', :error_handler => TestErrorHandler) do |msg|
-        raise 'oops'
-      end
-      queue.push(:text => 'foo')
-    end
-  end
-
-  def test_should_call_callback_when_complete
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('test', :size => 1) do |msg|
-        assert_equal 'foo', msg[:text]
-        'camel'
-      end
-      queue.push(:text => 'foo') do |result|
-        assert_equal 'camel', result
-        cb.call
-      end
-    end
-  end
-
-  def test_should_provide_status
-    mutex = Mutex.new
-    total = 200
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
-      end
-    end
-
-    actual = nil
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('image_crawler', :size => 3) do |msg|
-        mycount = incr.call
-        actual = queue.status if mycount == 100
-        cb.call if mycount == total
-      end
-      total.times do |idx|
-        queue.push(:text => 'foo')
-      end
-    end
-
-    refute_nil actual
-    refute_nil actual['image_crawler']
-    metrics = actual['image_crawler']
-    assert metrics[:total_queued] > 0
-    assert metrics[:total_queued] <= total
-    assert_equal 3, metrics[:pool_size]
-    assert_equal 3, metrics[:busy]
-    assert_equal 0, metrics[:ready]
-    assert(metrics[:backlog] > 0)
-    assert(metrics[:total_processed] > 0)
-  end
-
-  def test_should_persist_with_redis
-    begin
-      require 'redis'
-      redis = Redis.new
-      redis.flushdb
-    rescue LoadError
-      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
-    rescue Errno::ECONNREFUSED
-      return puts 'Skipping redis test, not running locally'
+  describe 'GirlFriday' do
+    after do
+      GirlFriday.shutdown!
     end
 
-    mutex = Mutex.new
-    total = 100
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
+    describe '.status' do
+      before do
+        q1 = GirlFriday::Queue.new(:q1) do; end
+        q2 = GirlFriday::Queue.new(:q2) do; end
       end
-    end
-
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('test', :size => 2, :store => GirlFriday::Store::Redis) do |msg|
-        incr.call
-        cb.call if count == total
-      end
-      total.times do
-        queue.push(:text => 'foo')
-      end
-    end
-  end
-
-  def test_should_persist_with_redis_instance
-    begin
-      require 'redis'
-      redis = Redis.new
-      redis.flushdb
-    rescue LoadError
-      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
-    rescue Errno::ECONNREFUSED
-      return puts 'Skipping redis test, not running locally accepting connections over UNIX socket'
-    end
-
-    mutex = Mutex.new
-    total = 100
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
-      end
-    end
-
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('test', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
-        incr.call
-        cb.call if count == total
-      end
-      total.times do
-        queue.push(:text => 'foo')
-      end
-    end
-  end
-
-  def test_should_allow_graceful_shutdown
-    mutex = Mutex.new
-    total = 100
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
+      it 'provides a status structure for each live queue' do
+        hash = GirlFriday.status
+        assert_kind_of Hash, hash
+        assert_equal 2, GirlFriday.queues.size
+        assert_equal 2, hash.size
       end
     end
 
-    async_test do |cb|
-      queue = GirlFriday::WorkQueue.new('shutdown', :size => 2) do |msg|
-        incr.call
-        cb.call if count == total
+    describe '.shutdown!' do
+      before do
+        q1 = GirlFriday::Queue.new(:q1) do; end
+        q2 = GirlFriday::Queue.new(:q2) do; end
       end
-      total.times do
-        queue.push(:text => 'foo')
+      it 'provides a status structure for each live queue' do
+        a = Time.now
+        assert_equal 0, GirlFriday.shutdown!
+        assert_in_delta 0, Time.now - a, 0.1
+        assert_equal 0, GirlFriday.queues.size
       end
-
-      count = GirlFriday.shutdown!
-      assert_equal 0, count
-      s = queue.status
-      assert_equal 0, s['shutdown'][:busy]
-      assert_equal 2, s['shutdown'][:ready]
-      assert(s['shutdown'][:backlog] > 0)
-      cb.call
-    end
-  end
-
-  def test_should_create_workers_lazily
-    async_test do |cb|
-      queue = GirlFriday::Queue.new('shutdown', :size => 2) do |msg|
-        assert_equal 1, queue.instance_variable_get(:@ready_workers).size
-        cb.call
-      end
-      refute queue.instance_variable_defined?(:@ready_workers)
-      # don't instantiate the worker threads until we actually put
-      # work onto the queue.
-      queue << 'empty msg'
-    end
-  end
-
-  def test_stubbing_girl_friday_with_flexmock
-    expected = Thread.current.to_s
-    actual = nil
-    processor = Proc.new do |msg|
-      actual = Thread.current.to_s
-    end
-    queue = GirlFriday::Queue.new('shutdown', :size => 2, &processor)
-    flexmock(queue).should_receive(:push).zero_or_more_times.and_return do |msg|
-      processor.call(msg)
     end
-    queue.push 'hello world!'
-    assert_equal expected, actual
   end
 
 end

data/test/test_girl_friday_immediately.rb

@@ -16,6 +16,7 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
     end
     assert_equal 42, queue.push(:start => 41)
     assert_equal 42, queue << { :start => 41 }
+    queue.shutdown
   end
 
   def test_should_process_immediately_with_callback
@@ -23,6 +24,7 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
       msg[:start] + 1
     end
     assert_equal 43, queue.push(:start => 41) { |r| r + 1 }
+    queue.shutdown
   end
 
   def test_should_process_style_idempotently
@@ -45,35 +47,6 @@ class TestGirlFridayImmediately < MiniTest::Unit::TestCase
     GirlFriday::WorkQueue.immediate!
     assert_equal 3, queue.push(:start => 2)
     assert_equal 3, queue << {:start => 2}
-  end
-
-  def test_should_process_immediately_when_rails_dev
-    # remove WorkQueue definitions first, since #push_async is defined
-    # as the class definition is executed, and Rails constant is not defined yet
-    [:Queue, :WorkQueue].each {|c| GirlFriday.send(:remove_const, c)}
-
-    # now top-level Rails constant will exist
-    Object.send(:include, RailsEnvironment)
-
-    # re-load class now that Rails constant is defined
-    $".delete_if {|f| f =~ %r{work_queue.rb}}
-    require 'girl_friday/work_queue'
-
-    # verify that we process immediately
-    queue = GirlFriday::WorkQueue.new('now') do |msg|
-      msg[:start] + 1
-    end
-    assert_equal 4, queue.push(:start => 3)
-    assert_equal 4, queue << {:start => 3}
+    queue.shutdown
   end
 end
-
-module RailsEnvironment
-  class Rails
-    def self.method_missing(m, *args, &block)
-      # swallow every call and return self so that WorkQueue
-      # thinks Rails dev environment is loaded
-      self
-    end
-  end
-end

data/test/test_girl_friday_queue.rb

@@ -0,0 +1,250 @@
+require 'helper'
+
+class TestGirlFriday < MiniTest::Unit::TestCase
+
+  class TestErrorHandler
+    include MiniTest::Assertions
+  end
+
+  def test_should_process_messages
+    async_test do |cb|
+      queue = GirlFriday::WorkQueue.new('process') do |msg|
+        assert_equal 'foo', msg[:text]
+        queue.shutdown do
+          cb.call
+        end
+      end
+      queue.push(:text => 'foo')
+    end
+  end
+
+  def test_should_handle_worker_error
+    async_test do |cb|
+      queue = nil
+      TestErrorHandler.send(:define_method, :handle) do |ex|
+        assert_equal 'oops', ex.message
+        assert_equal 'RuntimeError', ex.class.name
+        queue.shutdown do
+          cb.call
+        end
+      end
+
+      queue = GirlFriday::WorkQueue.new('error', :error_handler => TestErrorHandler) do |msg|
+        raise 'oops'
+      end
+      queue.push(:text => 'foo')
+    end
+  end
+
+  def test_should_call_callback_when_complete
+    async_test do |cb|
+      queue = GirlFriday::WorkQueue.new('callback', :size => 1) do |msg|
+        assert_equal 'foo', msg[:text]
+        'camel'
+      end
+      queue.push(:text => 'foo') do |result|
+        assert_equal 'camel', result
+        queue.shutdown do
+          cb.call
+        end
+      end
+    end
+  end
+
+  def test_should_provide_status
+    mutex = Mutex.new
+    total = 200
+    count = 0
+    incr = Proc.new do
+      mutex.synchronize do
+        count += 1
+      end
+    end
+
+    actual = nil
+    async_test do |cb|
+      queue = GirlFriday::WorkQueue.new('status', :size => 3) do |msg|
+        mycount = incr.call
+        actual = queue.status if mycount == 100
+        queue.shutdown do
+          cb.call
+        end if mycount == total
+      end
+      total.times do |idx|
+        queue.push(:text => 'foo')
+      end
+    end
+
+    refute_nil actual
+    refute_nil actual['status']
+    metrics = actual['status']
+    assert metrics[:total_queued] > 0
+    assert metrics[:total_queued] <= total
+    assert_equal 3, metrics[:pool_size]
+    assert_equal 3, metrics[:busy]
+    assert_equal 0, metrics[:ready]
+    assert(metrics[:backlog] > 0)
+    assert(metrics[:total_processed] > 0)
+  end
+
+  def test_should_persist_with_redis
+    begin
+      require 'redis'
+      redis = Redis.new
+      redis.flushdb
+    rescue LoadError
+      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
+    rescue Errno::ECONNREFUSED
+      return puts 'Skipping redis test, not running locally'
+    end
+
+    mutex = Mutex.new
+    total = 100
+    count = 0
+    incr = Proc.new do
+      mutex.synchronize do
+        count += 1
+      end
+    end
+
+    async_test(1.0) do |cb|
+      queue = GirlFriday::WorkQueue.new('redis', :size => 2, :store => GirlFriday::Store::Redis) do |msg|
+        incr.call
+        queue.shutdown do
+          cb.call
+        end if count == total
+      end
+      total.times do
+        queue.push(:text => 'foo')
+      end
+    end
+  end
+
+  def test_should_persist_with_redis_instance
+    begin
+      require 'redis'
+      redis = Redis.new
+      redis.flushdb
+    rescue LoadError
+      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
+    rescue Errno::ECONNREFUSED
+      return puts 'Skipping redis test, not running locally'
+    end
+
+    mutex = Mutex.new
+    total = 100
+    count = 0
+    incr = Proc.new do
+      mutex.synchronize do
+        count += 1
+      end
+    end
+
+    async_test(1.0) do |cb|
+      queue = GirlFriday::WorkQueue.new('redis-instance', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
+        incr.call
+        queue.shutdown do
+          cb.call
+        end if count == total
+      end
+      total.times do
+        queue.push(:text => 'foo')
+      end
+    end
+  end
+
+  def test_should_persist_with_redis_connection_pool
+    begin
+      require 'redis'
+      require 'connection_pool'
+      redis = ConnectionPool.new(:size => 5, :timeout => 5){ Redis.new }
+      redis.flushdb
+    rescue LoadError
+      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
+    rescue Errno::ECONNREFUSED
+      return puts 'Skipping redis test, not running locally'
+    end
+
+    mutex = Mutex.new
+    total = 100
+    count = 0
+    incr = Proc.new do
+      mutex.synchronize do
+        count += 1
+      end
+    end
+
+    async_test(1.0) do |cb|
+      queue = GirlFriday::WorkQueue.new('redis-pool', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
+        incr.call
+        queue.shutdown do
+          cb.call
+        end if count == total
+      end
+      total.times do
+        queue.push(:text => 'foo')
+      end
+    end
+  end
+
+  def test_should_allow_graceful_shutdown
+    mutex = Mutex.new
+    total = 100
+    count = 0
+    incr = Proc.new do
+      mutex.synchronize do
+        count += 1
+      end
+    end
+
+    async_test do |cb|
+      queue = GirlFriday::WorkQueue.new('shutdown', :size => 2) do |msg|
+        incr.call
+        cb.call if count == total
+      end
+      total.times do
+        queue.push(:text => 'foo')
+      end
+
+      assert_equal 1, GirlFriday.queues.size
+      count = GirlFriday.shutdown!
+      assert_equal 0, count
+      cb.call
+    end
+  end
+
+  def test_should_create_workers_lazily
+    async_test do |cb|
+      queue = GirlFriday::Queue.new('lazy', :size => 2) do |msg|
+        assert_equal 1, queue.instance_variable_get(:@ready_workers).size
+        queue.shutdown do
+          cb.call
+        end
+      end
+      refute queue.instance_variable_defined?(:@ready_workers)
+      # don't instantiate the worker threads until we actually put
+      # work onto the queue.
+      queue << 'empty msg'
+    end
+  end
+
+  def test_stubbing_girl_friday_with_flexmock
+    expected = Thread.current.to_s
+    actual = nil
+    processor = Proc.new do |msg|
+      actual = Thread.current.to_s
+    end
+    async_test do |cb|
+      queue = GirlFriday::Queue.new('flexmock', :size => 2, &processor)
+      flexmock(queue).should_receive(:push).zero_or_more_times.and_return do |msg|
+        processor.call(msg)
+      end
+      queue.push 'hello world!'
+      assert_equal expected, actual
+      queue.shutdown do
+        cb.call
+      end
+    end
+  end
+
+end

metadata

@@ -1,38 +1,39 @@
 --- !ruby/object:Gem::Specification
 name: girl_friday
 version: !ruby/object:Gem::Version
-  version: 0.9.4
-  prerelease: 
+  prerelease:
+  version: 0.9.5
 platform: ruby
 authors:
 - Mike Perham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-08-23 00:00:00.000000000Z
+date: 2011-09-22 00:00:00.000000000 -07:00
+default_executable:
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &13466240 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: &2086 !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+    none: false
+  requirement: *2086
   prerelease: false
-  version_requirements: *13466240
+  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &13465820 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: &2104 !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+    none: false
+  requirement: *2104
   prerelease: false
-  version_requirements: *13465820
+  type: :development
 description: Background processing, simplified
 email:
 - mperham@gmail.com
@@ -49,6 +50,8 @@ files:
 - Rakefile
 - TODO.md
 - config.ru
+- examples/backends/mongo_persistence.rb
+- examples/backends/using_mongo.rb
 - examples/batch.rb
 - examples/pipeline.rb
 - girl_friday.gemspec
@@ -59,7 +62,6 @@ files:
 - lib/girl_friday/monkey_patches.rb
 - lib/girl_friday/persistence.rb
 - lib/girl_friday/server.rb
-- lib/girl_friday/timed_queue.rb
 - lib/girl_friday/version.rb
 - lib/girl_friday/work_queue.rb
 - server/public/css/style.css
@@ -70,29 +72,36 @@ files:
 - test/test_batch.rb
 - test/test_girl_friday.rb
 - test/test_girl_friday_immediately.rb
-- test/timed_queue.rb
+- test/test_girl_friday_queue.rb
+has_rdoc: true
 homepage: http://github.com/mperham/girl_friday
 licenses: []
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: girl_friday
-rubygems_version: 1.8.6
-signing_key: 
+rubygems_version: 1.5.1
+signing_key:
 specification_version: 3
 summary: Background processing, simplified
-test_files: []
+test_files:
+- test/helper.rb
+- test/test_batch.rb
+- test/test_girl_friday.rb
+- test/test_girl_friday_immediately.rb
+- test/test_girl_friday_queue.rb
+...

data/lib/girl_friday/timed_queue.rb

@@ -1,48 +0,0 @@
-require 'thread'
-require 'timeout'
-
-class TimedQueue
-  def initialize
-    @que = []
-    @waiting = []
-    @mutex = Mutex.new
-    @resource = ConditionVariable.new
-  end
-
-  def push(obj)
-    @mutex.synchronize do
-      @que.push obj
-      @resource.signal
-    end
-  end
-  alias << push
-
-  def timed_pop(timeout=0.5)
-    while true
-      @mutex.synchronize do
-        @waiting.delete(Thread.current)
-        if @que.empty?
-          @waiting.push Thread.current
-          @resource.wait(@mutex, timeout)
-          raise TimeoutError if @que.empty?
-        else
-          retval = @que.shift
-          @resource.signal
-          return retval
-        end
-      end
-    end
-  end
-
-  def empty?
-    @que.empty?
-  end
-
-  def clear
-    @que.clear
-  end
-
-  def length
-    @que.length
-  end
-end

data/test/timed_queue.rb

@@ -1,48 +0,0 @@
-require 'thread'
-require 'timeout'
-
-class TimedQueue
-  def initialize
-    @que = []
-    @waiting = []
-    @mutex = Mutex.new
-    @resource = ConditionVariable.new
-  end
-
-  def push(obj)
-    @mutex.synchronize do
-      @que.push obj
-      @resource.signal
-    end
-  end
-  alias << push
-
-  def timed_pop(timeout=0.5)
-    while true
-      @mutex.synchronize do
-        @waiting.delete(Thread.current)
-        if @que.empty?
-          @waiting.push Thread.current
-          @resource.wait(@mutex, timeout)
-          raise TimeoutError if @que.empty?
-        else
-          retval = @que.shift
-          @resource.signal
-          return retval
-        end
-      end
-    end
-  end
-
-  def empty?
-    @que.empty?
-  end
-
-  def clear
-    @que.clear
-  end
-
-  def length
-    @que.length
-  end
-end