girl_friday 0.9.5 → 0.9.6

data/.travis.yml

@@ -1,7 +1,7 @@
 rvm:
   - 1.9.2
+  - 1.9.3
   - jruby
-  - rbx-2.0
 branches:
   only:
     - master

data/History.md

@@ -1,6 +1,13 @@
 Changes
 ================
 
+0.9.6
+---------
+
+* Completely switch to connection\_pool for Redis backend.
+  The Redis store no longer connects for you automatically, so it's mandatory to use
+  connection\_pool.
+
 0.9.5
 ---------
 

data/README.md

@@ -1,11 +1,7 @@
 girl\_friday
 ====================
 
-Have a task you want to get done sometime soon but don't want to do it yourself?  Give it to girl\_friday!  From wikipedia:
-
-> The term Man Friday has become an idiom, still in mainstream usage, to describe an especially faithful servant or
-> one's best servant or right-hand man. The female equivalent is Girl Friday. The title of the movie His Girl Friday
-> alludes to it and may have popularized it.
+[![Travis-CI build status](https://secure.travis-ci.org/mperham/girl_friday.png)](http://travis-ci.org/mperham/girl\_friday)
 
 girl\_friday is a Ruby library for performing asynchronous tasks.  Often times you don't want to block a web response by performing some task, like sending an email, so you can just use this gem to perform it in the background.  It works with any Ruby application, including Rails 3 applications.
 
@@ -37,22 +33,16 @@ In your Rails app, create a `config/initializers/girl_friday.rb` which defines y
       ImageCrawler.process(msg)
     end
 
-    SCRAPE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :host => 'host' }]) do |msg|
-      Page.scrape(msg)
-    end
-
-    TRANSCODE_QUEUE = GirlFriday::WorkQueue.new(:scrape_sites, :size => 4, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => $redis }]) do |msg|
-      VideoProcessor.transcode(msg)
-    end
-
 :size is the number of workers to spin up and defaults to 5.  Keep in mind, ActiveRecord defaults to a connection pool size of 5 so if your workers are accessing the database you'll want to ensure that the connection pool is large enough by modifying `config/database.yml`.
 
-You can use a connection pool to share a set of Redis connections with
+In order to use the Redis backend, you must use a connection pool to share a set of Redis connections with
 other threads and GirlFriday queues using the `connection\_pool` gem:
 
     require 'connection_pool'
+
     redis_pool = ConnectionPool.new(:size => 5, :timeout => 5) { Redis.new }
-    CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis_pool}]) do |msg|
+
+    CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => [{ :pool => redis_pool}]) do |msg|
       Filter.clean(msg)
     end
 
@@ -71,11 +61,18 @@ have no reference to them.  Make sure you call `WorkQueue#shutdown` if you are
 dynamically creating them so you don't leak memory.  `GirlFriday.shutdown!` will shut down all
 running queues in the process.
 
+
 More Detail
 --------------------
 
 Please see the [girl\_friday wiki](https://github.com/mperham/girl_friday/wiki) for more detail and advanced options and tuning.  You'll find details on queue persistence with Redis, implementing clean shutdown, querying runtime metrics and SO MUCH MORE!
 
+From wikipedia:
+
+> The term Man Friday has become an idiom, still in mainstream usage, to describe an especially faithful servant or
+> one's best servant or right-hand man. The female equivalent is Girl Friday. The title of the movie His Girl Friday
+> alludes to it and may have popularized it.
+
 
 Thanks
 --------------------

data/TODO.md

@@ -1,5 +1,5 @@
 TODO
-===============
+==============
 
  - rufus-scheduler integration for scheduled tasks
  - web admin UI to surface status() metrics

data/girl_friday.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
-  s.add_development_dependency 'sinatra', '~> 1.0'
-  s.add_development_dependency 'rake'
+  s.add_dependency              'connection_pool', '~> 0.1.0'
+  s.add_development_dependency  'sinatra', '~> 1.0'
+  s.add_development_dependency  'rake'
 end

data/lib/girl_friday.rb

@@ -17,10 +17,10 @@ require 'girl_friday/batch'
 
 module GirlFriday
 
-  @@lock = Mutex.new
+  @lock = Mutex.new
 
   def self.add_queue(ref)
-    @@lock.synchronize do
+    @lock.synchronize do
       @queues ||= []
       @queues.reject! { |q| !q.weakref_alive? }
       @queues << ref
@@ -28,7 +28,7 @@ module GirlFriday
   end
 
   def self.remove_queue(ref)
-    @@lock.synchronize do
+    @lock.synchronize do
       @queues.delete ref
     end
   end
@@ -81,7 +81,7 @@ module GirlFriday
       end
 
       m.synchronize do
-        var.wait(m, timeout)
+        var.wait(m, timeout) if count != 0
       end
     end
     count

data/lib/girl_friday/persistence.rb

@@ -23,17 +23,20 @@ module GirlFriday
     class Redis
       def initialize(name, options)
         @opts = options
+        unless @opts[:pool]
+          raise ArgumentError, "you must pass in a :pool"
+        end
         @key = "girl_friday-#{name}-#{environment}"
       end
 
       def push(work)
         val = Marshal.dump(work)
-        redis.rpush(@key, val)
+        redis{ |r| r.rpush(@key, val) }
       end
       alias_method :<<, :push
 
       def pop
-        val = redis.lpop(@key)
+        val = redis{ |r| r.lpop(@key) }
         Marshal.load(val) if val
       end
 
@@ -48,8 +51,12 @@ module GirlFriday
       end
 
       def redis
-        @redis ||= (@opts.delete(:redis) || ::Redis.connect(*@opts))
+        @redis ||= @opts.delete(:pool)
+        @redis.with do |pooled|
+          yield pooled
+        end
       end
     end
+
   end
 end

data/lib/girl_friday/version.rb

@@ -1,3 +1,3 @@
 module GirlFriday
-  VERSION = "0.9.5"
+  VERSION = "0.9.6"
 end

data/lib/girl_friday/work_queue.rb

@@ -15,9 +15,10 @@ module GirlFriday
 
       @shutdown = false
       @busy_workers = []
+      @ready_workers = nil
       @created_at = Time.now.to_i
       @total_processed = @total_errors = @total_queued = 0
-      @persister = (options[:store] || Store::InMemory).new(name, (options[:store_config] || []))
+      @persister = (options[:store] || Store::InMemory).new(name, (options[:store_config] || {}))
       @weakref = WeakRef.new(self)
       start
       GirlFriday.add_queue @weakref
@@ -49,7 +50,7 @@ module GirlFriday
       { @name => {
           :pid => $$,
           :pool_size => @size,
-          :ready => ready_workers.size,
+          :ready => @ready_workers ? @ready_workers.size : 0,
           :busy => @busy_workers.size,
           :backlog => @persister.size,
           :total_queued => @total_queued,
@@ -127,8 +128,10 @@ module GirlFriday
 
     def start
       @supervisor = Actor.spawn do
+        Thread.current[:label] = "#{name}-supervisor"
         supervisor = Actor.current
         @work_loop = Proc.new do
+          Thread.current[:label] = "#{name}-worker"
           while !@shutdown do
             work = Actor.receive
             if !@shutdown

data/test/helper.rb

@@ -6,7 +6,7 @@ at_exit do
     Thread.list.each do |thread|
       next if thread.status == 'run'
       puts "WARNING: lingering threads found.  All threads should be shutdown and garbage collected."
-      p [thread, thread['name']]
+      p [thread, thread[:label], thread.object_id]
 #      puts thread.backtrace.join("\n")
     end
   end

data/test/test_girl_friday_queue.rb

@@ -87,44 +87,12 @@ class TestGirlFriday < MiniTest::Unit::TestCase
     assert(metrics[:total_processed] > 0)
   end
 
-  def test_should_persist_with_redis
-    begin
-      require 'redis'
-      redis = Redis.new
-      redis.flushdb
-    rescue LoadError
-      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
-    rescue Errno::ECONNREFUSED
-      return puts 'Skipping redis test, not running locally'
-    end
-
-    mutex = Mutex.new
-    total = 100
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
-      end
-    end
-
-    async_test(1.0) do |cb|
-      queue = GirlFriday::WorkQueue.new('redis', :size => 2, :store => GirlFriday::Store::Redis) do |msg|
-        incr.call
-        queue.shutdown do
-          cb.call
-        end if count == total
-      end
-      total.times do
-        queue.push(:text => 'foo')
-      end
-    end
-  end
-
-  def test_should_persist_with_redis_instance
+  def test_should_persist_with_redis_connection_pool
     begin
       require 'redis'
-      redis = Redis.new
-      redis.flushdb
+      require 'connection_pool'
+      pool = ConnectionPool.new(:size => 5, :timeout => 2){ Redis.new }
+      pool.flushdb
     rescue LoadError
       return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
     rescue Errno::ECONNREFUSED
@@ -140,8 +108,8 @@ class TestGirlFriday < MiniTest::Unit::TestCase
       end
     end
 
-    async_test(1.0) do |cb|
-      queue = GirlFriday::WorkQueue.new('redis-instance', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
+    async_test(2.0) do |cb|
+      queue = GirlFriday::WorkQueue.new('redis-pool', :size => 2, :store => GirlFriday::Store::Redis, :store_config => { :pool => pool }) do |msg|
         incr.call
         queue.shutdown do
           cb.call
@@ -153,36 +121,10 @@ class TestGirlFriday < MiniTest::Unit::TestCase
     end
   end
 
-  def test_should_persist_with_redis_connection_pool
-    begin
-      require 'redis'
-      require 'connection_pool'
-      redis = ConnectionPool.new(:size => 5, :timeout => 5){ Redis.new }
-      redis.flushdb
-    rescue LoadError
-      return puts "Skipping redis test, 'redis' gem not found: #{$!.message}"
-    rescue Errno::ECONNREFUSED
-      return puts 'Skipping redis test, not running locally'
-    end
-
-    mutex = Mutex.new
-    total = 100
-    count = 0
-    incr = Proc.new do
-      mutex.synchronize do
-        count += 1
-      end
-    end
-
-    async_test(1.0) do |cb|
-      queue = GirlFriday::WorkQueue.new('redis-pool', :size => 2, :store => GirlFriday::Store::Redis, :store_config => [{ :redis => redis }]) do |msg|
-        incr.call
-        queue.shutdown do
-          cb.call
-        end if count == total
-      end
-      total.times do
-        queue.push(:text => 'foo')
+  def test_should_raise_if_no_store_config_passed_in_for_redis_backend
+    assert_raises(ArgumentError) do
+      GirlFriday::WorkQueue.new('raise-test', :store => GirlFriday::Store::Redis) do |msg|
+        # doing work
       end
     end
   end
@@ -221,7 +163,8 @@ class TestGirlFriday < MiniTest::Unit::TestCase
           cb.call
         end
       end
-      refute queue.instance_variable_defined?(:@ready_workers)
+      assert queue.instance_variable_defined?(:@ready_workers)
+      assert_nil queue.instance_variable_get(:@ready_workers)
       # don't instantiate the worker threads until we actually put
       # work onto the queue.
       queue << 'empty msg'

metadata

@@ -1,39 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: girl_friday
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 0.9.5
+  version: 0.9.6
+  prerelease: 
 platform: ruby
 authors:
 - Mike Perham
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-22 00:00:00.000000000 -07:00
-default_executable:
+date: 2011-09-26 00:00:00.000000000Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: &22271180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *22271180
 - !ruby/object:Gem::Dependency
   name: sinatra
-  version_requirements: &2086 !ruby/object:Gem::Requirement
+  requirement: &22270680 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-    none: false
-  requirement: *2086
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *22270680
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: &2104 !ruby/object:Gem::Requirement
+  requirement: &22270300 !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
-  requirement: *2104
-  prerelease: false
   type: :development
+  prerelease: false
+  version_requirements: *22270300
 description: Background processing, simplified
 email:
 - mperham@gmail.com
@@ -73,35 +83,28 @@ files:
 - test/test_girl_friday.rb
 - test/test_girl_friday_immediately.rb
 - test/test_girl_friday_queue.rb
-has_rdoc: true
 homepage: http://github.com/mperham/girl_friday
 licenses: []
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
 rubyforge_project: girl_friday
-rubygems_version: 1.5.1
-signing_key:
+rubygems_version: 1.8.6
+signing_key: 
 specification_version: 3
 summary: Background processing, simplified
-test_files:
-- test/helper.rb
-- test/test_batch.rb
-- test/test_girl_friday.rb
-- test/test_girl_friday_immediately.rb
-- test/test_girl_friday_queue.rb
-...
+test_files: []