girl_friday 0.9.6 → 0.9.7

data/.gitignore

@@ -4,3 +4,5 @@ Gemfile.lock
 pkg/*
 rbxdb/
 *.rdb
+.idea/*
+

data/History.md

@@ -1,6 +1,13 @@
 Changes
 ================
 
+0.9.7
+---------
+
+* Fix error handling (xshay)
+* Add streaming batch support for adding elements to a batch one at a
+  time rather than all at once.
+
 0.9.6
 ---------
 

data/README.md

@@ -36,13 +36,13 @@ In your Rails app, create a `config/initializers/girl_friday.rb` which defines y
 :size is the number of workers to spin up and defaults to 5.  Keep in mind, ActiveRecord defaults to a connection pool size of 5 so if your workers are accessing the database you'll want to ensure that the connection pool is large enough by modifying `config/database.yml`.
 
 In order to use the Redis backend, you must use a connection pool to share a set of Redis connections with
-other threads and GirlFriday queues using the `connection\_pool` gem:
+other threads and GirlFriday queues using the `connection_pool` gem:
 
     require 'connection_pool'
 
     redis_pool = ConnectionPool.new(:size => 5, :timeout => 5) { Redis.new }
 
-    CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => [{ :pool => redis_pool}]) do |msg|
+    CLEAN_FILTER_QUEUE = GirlFriday::WorkQueue.new(:clean_filter, :store => GirlFriday::Store::Redis, :store_config => { :pool => redis_pool }) do |msg|
       Filter.clean(msg)
     end
 

data/examples/batch.rb

@@ -7,11 +7,14 @@ class UrlProcessor
   URLS = %w(http://www.bing.com http://www.google.com http://www.yahoo.com)
 
   def parallel
-    batch = GirlFriday::Batch.new(URLS, :size => 3) do |url|
+    batch = GirlFriday::Batch.new(nil, :size => 3) do |url|
       html = open(url)
       doc = Nokogiri::HTML(html.read)
       doc.css('span').count
     end
+    URLS.each do |url|
+      batch << url
+    end
     p URLS.zip(batch.results)
   end
 

data/girl_friday.gemspec

@@ -17,6 +17,6 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
   s.add_dependency              'connection_pool', '~> 0.1.0'
-  s.add_development_dependency  'sinatra', '~> 1.0'
+  s.add_development_dependency  'sinatra', '~> 1.3'
   s.add_development_dependency  'rake'
 end

data/lib/girl_friday/batch.rb

@@ -12,17 +12,23 @@ module GirlFriday
   #
   # TODO Errors are not handled well at all.
   class Batch
-    def initialize(enumerable, options, &block)
+    def initialize(enumerable=nil, options={}, &block)
       @queue = GirlFriday::Queue.new(:batch, options, &block)
       @complete = 0
-      @size = enumerable.count
-      @results = Array.new(@size)
+      @size = 0
+      @results = []
+      if enumerable
+        @size = enumerable.count
+        @results = Array.new(@size)
+      end
       @lock = Mutex.new
       @condition = ConditionVariable.new
+      @frozen = false
       start(enumerable)
     end
 
     def results(timeout=nil)
+      @frozen = true
       @lock.synchronize do
         @condition.wait(@lock, timeout) if @complete != @size
         @queue.shutdown
@@ -30,17 +36,34 @@ module GirlFriday
       end
     end
 
+    def push(msg)
+      raise ArgumentError, "Batch is frozen, you cannot push more items into it" if @frozen
+      @lock.synchronize do
+        @results << nil
+        @size += 1
+        index = @results.size - 1
+        @queue.push(msg) do |result|
+          completion(result, index)
+        end
+      end
+    end
+    alias_method :<<, :push
+
     private
 
     def start(operations)
       operations.each_with_index do |packet, index|
         @queue.push(packet) do |result|
-          @lock.synchronize do
-            @complete += 1
-            @results[index] = result
-            @condition.signal if @complete == @size
-          end
+          completion(result, index)
         end
+      end if operations
+    end
+
+    def completion(result, index)
+      @lock.synchronize do
+        @complete += 1
+        @results[index] = result
+        @condition.signal if @complete == @size
       end
     end
 

data/lib/girl_friday/persistence.rb

@@ -23,25 +23,23 @@ module GirlFriday
     class Redis
       def initialize(name, options)
         @opts = options
-        unless @opts[:pool]
-          raise ArgumentError, "you must pass in a :pool"
-        end
+        raise ArgumentError, "you must pass in a :pool" unless @opts[:pool]
         @key = "girl_friday-#{name}-#{environment}"
       end
 
       def push(work)
         val = Marshal.dump(work)
-        redis{ |r| r.rpush(@key, val) }
+        redis { |r| r.rpush(@key, val) }
       end
       alias_method :<<, :push
 
       def pop
-        val = redis{ |r| r.lpop(@key) }
+        val = redis { |r| r.lpop(@key) }
         Marshal.load(val) if val
       end
 
       def size
-        redis.llen(@key)
+        redis { |r| r.llen(@key) }
       end
 
       private

data/lib/girl_friday/server.rb

@@ -14,7 +14,7 @@ module GirlFriday
     basedir = File.expand_path(File.dirname(__FILE__) + '/../../server')
 
     set :views,  "#{basedir}/views"
-    set :public, "#{basedir}/public"
+    set :public_folder, "#{basedir}/public"
     set :static, true
 
     helpers do

data/lib/girl_friday/version.rb

@@ -1,3 +1,3 @@
 module GirlFriday
-  VERSION = "0.9.6"
+  VERSION = "0.9.7"
 end

data/lib/girl_friday/work_queue.rb

@@ -11,7 +11,7 @@ module GirlFriday
       @name = name.to_s
       @size = options[:size] || 5
       @processor = block
-      @error_handlers = (Array(options[:error_handler]) || ErrorHandler.default).map(&:new)
+      @error_handlers = (Array(options[:error_handler] || ErrorHandler.default)).map(&:new)
 
       @shutdown = false
       @busy_workers = []
@@ -70,17 +70,26 @@ module GirlFriday
       end
     end
 
-    def shutdown
+    def shutdown(&block)
       # Runtime state should never be modified by caller thread,
       # only the Supervisor thread.
-      @supervisor << Shutdown[block_given? ? Proc.new : nil]
+      @supervisor << Shutdown[block]
     end
 
     private
 
+    def running?
+      !@shutdown
+    end
+
+    def handle_error(ex)
+      # Redis network error?  Log and ignore.
+      @error_handlers.each { |handler| handler.handle(ex) }
+    end
+
     def on_ready(who)
       @total_processed += 1
-      if !@shutdown && work = @persister.pop
+      if running? && work = @persister.pop
         who.this << work
         drain
       else
@@ -88,22 +97,21 @@ module GirlFriday
         ready_workers << who.this
       end
     rescue => ex
-      # Redis network error?  Log and ignore.
-      @error_handlers.each { |handler| handler.handle(ex) }
+      handle_error(ex)
     end
 
     def shutdown_complete
       begin
         @when_shutdown.call(self) if @when_shutdown
       rescue Exception => ex
-        @error_handlers.each { |handler| handler.handle(ex) }
+        handle_error(ex)
       end
     end
 
     def on_work(work)
       @total_queued += 1
 
-      if !@shutdown && worker = ready_workers.pop
+      if running? && worker = ready_workers.pop
         @busy_workers << worker
         worker << work
         drain
@@ -111,19 +119,12 @@ module GirlFriday
         @persister << work
       end
     rescue => ex
-      # Redis network error?  Log and ignore.
-      @error_handlers.each { |handler| handler.handle(ex) }
+      handle_error(ex)
     end
 
     def ready_workers
-      @ready_workers ||= begin
-        workers = []
-        @size.times do
-          # start N workers
-          workers << Actor.spawn_link(&@work_loop)
-        end
-        workers
-      end
+      # start N workers
+      @ready_workers ||= Array.new(@size) { Actor.spawn_link(&@work_loop) }
     end
 
     def start
@@ -132,9 +133,9 @@ module GirlFriday
         supervisor = Actor.current
         @work_loop = Proc.new do
           Thread.current[:label] = "#{name}-worker"
-          while !@shutdown do
+          while running? do
             work = Actor.receive
-            if !@shutdown
+            if running?
               result = @processor.call(work.msg)
               work.callback.call(result) if work.callback
               supervisor << Ready[Actor.current]
@@ -155,8 +156,7 @@ module GirlFriday
 
     def drain
       # give as much work to as many ready workers as possible
-      ps = @persister.size
-      todo = ready_workers.size < ps ? ready_workers.size : ps
+      todo = [@persister.size, ready_workers.size].min
       todo.times do
         worker = ready_workers.pop
         @busy_workers << worker
@@ -183,18 +183,18 @@ module GirlFriday
             return
           end
           f.when(Actor::DeadActorError) do |ex|
-            if !@shutdown
+            if running?
               # TODO Provide current message contents as error context
               @total_errors += 1
               @busy_workers.delete(ex.actor)
               ready_workers << Actor.spawn_link(&@work_loop)
-              @error_handlers.each { |handler| handler.handle(ex.reason) }
+              handle_error(ex.reason)
             end
           end
         end
       end
     end
-
   end
+
   Queue = WorkQueue
 end

data/test/helper.rb

@@ -2,12 +2,16 @@ $testing = true
 puts RUBY_DESCRIPTION
 
 at_exit do
+  # queue.shutdown is async so sleep a little to minimize
+  # race conditions between us and other threads
+  # that haven't yet been GC'd.
+  sleep 0.1
   if Thread.list.size > 1
     Thread.list.each do |thread|
       next if thread.status == 'run'
       puts "WARNING: lingering threads found.  All threads should be shutdown and garbage collected."
-      p [thread, thread[:label], thread.object_id]
-#      puts thread.backtrace.join("\n")
+      puts "This is normal if a test failed so a queue did not get shutdown."
+      p [thread, thread[:label]]
     end
   end
 end
@@ -30,6 +34,9 @@ class MiniTest::Unit::TestCase
     q = TimedQueue.new
     yield Proc.new { q << nil }
     q.timed_pop(time)
+  ensure
+    count = GirlFriday.shutdown!(1)
+    puts "Unable to shutdown queue (#{count})" if count != 0
   end
 
 end

data/test/test_batch.rb

@@ -16,7 +16,7 @@ class TestBatch < MiniTest::Unit::TestCase
     # asking for the results should block
     results = batch.results(1.0)
     c = Time.now
-    assert_in_delta(0.5, (c - b), 0.1)
+    assert_in_delta(0.5, (c - b), 0.3)
 
     assert_equal 10, results.size
     assert_kind_of Time, results[0]
@@ -39,4 +39,35 @@ class TestBatch < MiniTest::Unit::TestCase
     # http://redmine.ruby-lang.org/issues/5342
     sleep 0.1
   end
+
+  def test_empty_batch
+    batch = GirlFriday::Batch.new(nil, :size => 4) do |msg|
+      sleep msg
+      'x'
+    end
+    values = batch.results
+    values.must_be_kind_of Array
+    values.must_equal []
+  end
+
+  def test_streaming_batch_api
+    batch = GirlFriday::Batch.new(nil, :size => 4) do |msg|
+      sleep msg
+      'x'
+    end
+    a = Time.now
+    batch << 0.1
+    batch << 0.1
+    batch << 0.1
+    batch << 0.1
+    values = batch.results
+    b = Time.now
+    values.must_be_kind_of Array
+    values.must_equal %w(x x x x)
+    assert_in_delta 0.2, (b - a), 0.1
+
+    assert_raises ArgumentError do
+      batch << 0.1
+    end
+  end
 end

data/test/test_girl_friday_queue.rb

@@ -1,6 +1,6 @@
 require 'helper'
 
-class TestGirlFriday < MiniTest::Unit::TestCase
+class TestGirlFridayQueue < MiniTest::Unit::TestCase
 
   class TestErrorHandler
     include MiniTest::Assertions
@@ -36,6 +36,20 @@ class TestGirlFriday < MiniTest::Unit::TestCase
     end
   end
 
+  def test_should_use_a_default_error_handler_when_none_specified
+    async_test do |cb|
+      queue = GirlFriday::WorkQueue.new('error') do |msg|
+      end
+      queue.shutdown do
+        cb.call
+      end
+      queue.push(:text => 'foo') # Redundant
+
+      # Not an ideal method, but I can't see a better way without complex stubbing.
+      assert queue.instance_eval { @error_handlers }.length > 0
+    end
+  end
+
   def test_should_call_callback_when_complete
     async_test do |cb|
       queue = GirlFriday::WorkQueue.new('callback', :size => 1) do |msg|
@@ -118,6 +132,7 @@ class TestGirlFriday < MiniTest::Unit::TestCase
       total.times do
         queue.push(:text => 'foo')
       end
+      refute_nil queue.status['redis-pool'][:backlog]
     end
   end
 

metadata

@@ -1,49 +1,50 @@
 --- !ruby/object:Gem::Specification
 name: girl_friday
 version: !ruby/object:Gem::Version
-  version: 0.9.6
-  prerelease: 
+  prerelease:
+  version: 0.9.7
 platform: ruby
 authors:
 - Mike Perham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-09-26 00:00:00.000000000Z
+date: 2011-11-28 00:00:00.000000000 -08:00
+default_executable:
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &22271180 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: &2078 !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.0
-  type: :runtime
+    none: false
+  requirement: *2078
   prerelease: false
-  version_requirements: *22271180
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &22270680 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: &2096 !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
+        version: '1.3'
+    none: false
+  requirement: *2096
   prerelease: false
-  version_requirements: *22270680
+  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &22270300 !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: &2114 !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+    none: false
+  requirement: *2114
   prerelease: false
-  version_requirements: *22270300
+  type: :development
 description: Background processing, simplified
 email:
 - mperham@gmail.com
@@ -83,28 +84,35 @@ files:
 - test/test_girl_friday.rb
 - test/test_girl_friday_immediately.rb
 - test/test_girl_friday_queue.rb
+has_rdoc: true
 homepage: http://github.com/mperham/girl_friday
 licenses: []
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: girl_friday
-rubygems_version: 1.8.6
-signing_key: 
+rubygems_version: 1.6.2
+signing_key:
 specification_version: 3
 summary: Background processing, simplified
-test_files: []
+test_files:
+- test/helper.rb
+- test/test_batch.rb
+- test/test_girl_friday.rb
+- test/test_girl_friday_immediately.rb
+- test/test_girl_friday_queue.rb
+...