resque-batched-logger 0.0.9

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+coverage

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in resque-batched-logger.gemspec
+gemspec

data/README

@@ -0,0 +1,75 @@
+Resque Batched Logger
+=====================
+
+ResqueBatchedLogger is an extension to the fantastic background work gem 'resque'. This extension is intended for when you have large volumes of work you process in logical groups, or batches, and want timing, and logging information on the group as a whole. For example we have a daily import process which spawns a large amount of several types of jobs, running a couple of hours. This grouped logging lets us track when those jobs finish, and how long each took (by type)
+
+Installation:
+=============
+
+  gem install resque-batched-logger
+
+Usage:
+======
+
+  # include the module in your job classes
+  class MyJob
+    include Resque::Plugins::BatchedLogging
+    def self.perform(*args)
+      # your job implementation
+    end
+  end
+
+  # enqueue your jobs by calling #enqueue from within a batched block on your job classes
+  # This will call any custom #self.enqueue or #self.create methods defined on your job class, defaulting to the standard Resque.enqueue after wrapping your job in it's logging code
+  # jobs will be batched grouped by the job class name, and must be unique so for running multiple batches at once see below on how to do so with subclassing.
+  MyJob.batched do
+    enqueue(1,2,3,{:my => :options})
+  end
+
+You'll need to run at least one resque-worker on the queue named 'batched_logger', (recommended only one for consistency of log ordering) this worker can be reasonably low priority, depending on how quickly you need output to the logfile.
+
+The logging of each batch of jobs will be written to a log file at `log/batched_jobs.log` by the BatchedLogger resque job.
+
+Sample Output:
+==============
+
+  ==== Batched jobs 'Daily MyJob run' : logged at Mon Mar 28 16:25:04 +1300 2011 ====
+    batch started processing at: Mon Mar 28 16:23:00 +1300 2011
+    batch finished processing at: Mon Mar 28 16:25:02 +1300 2011
+    Total run time for batch: 122 seconds
+    Jobs Enqueued: 220
+    Jobs Processed: 220
+    Average time per job: 0.527 seconds
+    Total time spent processing jobs: 116 seconds
+  ==== Batched jobs 'Daily MyJob run' completed at Mon Mar 28 16:25:02 +1300 2011 took 122 seconds ====
+
+Advanced Usage
+==============
+
+For batching jobs of the same type in multiple groups, or if your application might enqueue jobs of the same type while your batched jobs are running, it is recommended that you subclass your jobs for a batched class, to allow an exclusive batch scope
+
+example:
+
+  class BackendLifting
+    @queue = :standard
+    def self.perform(user_id)
+      # do something
+    end
+  end
+  class BatchedBackendLifting < BackendLifting
+    # This is simply to provide an exclusive scope
+  end
+
+This allows you to batch your jobs via:
+
+  BatchedBackendLifting.batched do
+    user_ids.each do |id|
+      enqueue(id)
+    end
+  end
+
+Which will prevent any `BackendLifting` jobs your application enqueues simultaneously from being inadvertently logged as a batched job. This is only necessary if you want to guarantee there are no additional jobs added to the logging of your batch, or the ability to enqueue multiple batches at once.
+
+How it works
+============
+Log information is stored in redis until all jobs have been processed, and once all the jobs present in the batch have been performed, the `BatchedJobsLogger` pulls this information out of redis, aggregates it and outputs it to the logfile 'log/batched_jobs.log'.

data/Rakefile

@@ -0,0 +1,19 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.rcov_opts << '--exclude /gems/,/Library/,/usr/,spec,lib/tasks' # exclude external gems/libraries
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/lib/resque/plugins/batched_logger.rb

@@ -0,0 +1,111 @@
+module Resque
+  module Plugins
+    class BatchedLogger < Resque::Job
+
+      require 'time'
+      require 'fileutils'
+
+      @queue = :batched_logger
+      LOG_FILE = "log/batched_jobs.log"
+      @@logger = nil
+
+      # Requeue's all loggers
+      def self.requeue_all
+        Resque.redis.keys("*:jobcount").each do |key|
+          Resque.enqueue(self, key.gsub(":jobcount", ""))
+        end
+      end
+
+      def self.perform(batch_name)
+        begin
+          FileUtils.mkdir_p(File.dirname(LOG_FILE))
+          FileUtils.touch(LOG_FILE)
+
+          if !batch_to_log?(batch_name)
+            logger.puts("No jobs to log for '#{batch_name}'")
+          elsif jobs_pending?(batch_name)
+            sleep 5                          # Wait 5 seconds
+            Resque.enqueue(self, batch_name) # Requeue, to check again
+          else
+            # Start processing the logs
+            # Pull in the info stored in redis
+            job_stats = {
+              :processing_time => 0,      :longest_processing_time => 0,
+              :processed_job_count => 0,  :enqueued_job_count => Resque.redis.get("#{batch_name}:jobcount"),
+              :start_time => nil,         :finish_time => nil
+            }
+            Resque.redis.del("#{batch_name}:jobcount") # Stop any other logging processes picking up the same job
+            # lpop pops the first element off the list in redis
+            while job = Resque.redis.lpop("batch_stats:#{batch_name}")
+              job = decode_job(job) # Decode from string format
+              # job => [run_time, start_time, end_time]
+              job_stats[:processing_time] += job[0]                                                           # run_time
+              job_stats[:longest_processing_time] = job[0] if job[0] > job_stats[:longest_processing_time]
+              job_stats[:start_time] ||= job[1]                                                               # start_time
+              job_stats[:finish_time] ||= job[2]
+              job_stats[:finish_time] = job[2] if job[2] > job_stats[:finish_time]                            # end_time
+              job_stats[:processed_job_count] += 1
+            end
+            Resque.redis.del("batch_stats:#{batch_name}") # Cleanup the array of stats we've just processed (it should be empty now)
+
+            if job_stats[:processed_job_count].zero?
+              log_empty_batch(batch_name)
+            else
+              log_stats(batch_name, job_stats)
+            end
+          end
+        ensure
+          logger.close
+          @@logger = nil
+        end
+      end
+
+      private
+
+      def self.logger
+        @@logger ||= File.open(LOG_FILE, "a")
+      end
+
+      def self.log_stats(batch_name, job_stats)
+        job_stats[:total_time] = job_stats[:finish_time] - job_stats[:start_time]
+
+        # Aggregate stats
+        job_stats[:average_time] = job_stats[:total_time] / job_stats[:processed_job_count].to_f
+
+        logger.puts "==== Batched jobs '#{batch_name}' : logged at #{Time.now.to_s} ===="
+        logger.puts "  batch started processing at: #{job_stats[:start_time]}"
+        logger.puts "  batch finished processing at: #{job_stats[:finish_time]}"
+        logger.puts "  Total run time for batch: #{job_stats[:total_time]} seconds"
+
+        logger.puts "  Jobs Enqueued: #{job_stats[:enqueued_job_count]}"
+        logger.puts "  Jobs Processed: #{job_stats[:processed_job_count]}"
+        logger.puts "  Average time per job: #{job_stats[:average_time]} seconds"
+        logger.puts "  Total time spent processing jobs: #{job_stats[:processing_time]} seconds"
+
+        logger.puts "==== Batched jobs '#{batch_name}' completed at #{job_stats[:finish_time]} took #{job_stats[:total_time]} seconds ===="
+      end
+
+      def self.log_empty_batch(batch_name)
+        logger.puts "==== Batched jobs '#{batch_name}' : logged at #{Time.now.to_s} ===="
+        logger.puts "==== Batched jobs '#{batch_name}' completed, 0 jobs enqueued ===="
+      end
+
+      # It is possible that there might be more jobs processed than the number specified, in those cases we will process them all anyway
+      def self.jobs_pending?(batch_name)
+        jobs = Resque.redis.get("#{batch_name}:jobcount")
+        jobs && Resque.redis.llen("batch_stats:#{batch_name}") < jobs.to_i
+      end
+      def self.batch_to_log?(batch_name)
+        jobs = Resque.redis.get("#{batch_name}:jobcount")
+      end
+
+      def self.decode_job(job)
+        job = JSON.parse(job)
+        job[1] = Time.parse(job[1])
+        job[2] = Time.parse(job[2])
+        job
+      end
+
+    end
+  end
+end

data/lib/resque/plugins/batched_logging.rb

@@ -0,0 +1,85 @@
+module Resque
+  module Plugins
+    module BatchedLogging
+
+      require 'benchmark'
+
+      class BatchExists < StandardError; end;
+
+      def self.included(base)
+        base.class_eval do
+          class << self
+
+            # MyJob.batched(:batch_name => "MyCustomBatchName") do
+            #   enqueue(1,2,3,{:my => :options})
+            # end
+            def batched(&block)
+              batch_name = self.to_s # Group the batch of jobs by the job's class name
+              raise Resque::Plugins::BatchedLogging::BatchExists.new("Batch for '#{batch_name}' exists already") if Resque.redis.get("#{batch_name}:jobcount")
+
+              job_count = 0
+              Resque.redis.set("#{batch_name}:jobcount", job_count) # Set the job count right away, because the workers will check for it's existence
+              if block_given?
+                proxy_obj = Resque::Plugins::BatchedLogging::BatchLoggerProxy.new(self, batch_name)
+                proxy_obj.run(&block)
+                job_count = proxy_obj.job_count
+              else
+                raise "Must pass a block through to a batched group of jobs"
+              end
+              Resque.redis.set("#{batch_name}:jobcount", job_count)
+              Resque.enqueue(Resque::Plugins::BatchedLogger, batch_name) # Queue a job to proccess the log information that is stored in redis
+            end
+
+            # Plugin.around_hook for wrapping the job in logging code
+            def around_perform_log_as_batched(*args)
+              batch_name = self.to_s
+              # Presence of the jobcount variable means that batched logging is enabled for this job type
+              if Resque.redis.get("#{batch_name}:jobcount")
+                # Perform our logging
+                start_time = Time.now
+                run_time = Benchmark.realtime do
+                  yield
+                end
+                end_time = Time.now
+                # Store, [run_time, start_time, end_time] as an entry into redis under in an array for this specific job type & queue
+                # rpush appends to the end of a list in redis
+                Resque.redis.rpush("batch_stats:#{batch_name}", [run_time, start_time, end_time].to_json) # Push the values onto the end of the list (will create the list if it doesn't exist)
+                # End of our logging
+              else
+                yield # Just perform the standard job without benchmarking
+              end
+            end
+
+          end
+        end
+      end
+
+      # For enabling MyJob.in_batches with block syntax
+      class BatchLoggerProxy
+        attr_reader :job_type, :batch_name
+        attr_accessor :job_count
+        def initialize(job_type, batch_name)
+          @job_type = job_type
+          @batch_name = batch_name || @job_type.to_s
+          @job_count = 0
+        end
+        def run(&block)
+          instance_eval(&block) if block_given?
+        end
+        # Capture #create, and #enqueue calls that are done within the scope of a MyJob.in_batches block and enqueue the original job
+        def enqueue(*args)
+          if @job_type.respond_to?(:enqueue)
+            @job_type.enqueue(*args)
+          elsif @job_type.respond_to?(:create)
+            @job_type.create(*args)
+          else
+            Resque.enqueue(@job_type, *args)
+          end
+          @job_count += 1
+        end
+        alias :create :enqueue
+      end
+
+    end
+  end
+end

data/lib/resque-batched-logger/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Batched
+    module Logger
+      VERSION = "0.0.9"
+    end
+  end
+end

data/lib/resque-batched-logger.rb

@@ -0,0 +1,3 @@
+lib_dir = File.dirname(__FILE__)
+require File.join(lib_dir, 'resque/plugins/batched_logging')
+require File.join(lib_dir, 'resque/plugins/batched_logger')

data/resque-batched-logger.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "resque-batched-logger/version"
+
+Gem::Specification.new do |s|
+  s.name        = "resque-batched-logger"
+  s.version     = Resque::Batched::Logger::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Jeremy Olliver"]
+  s.email       = ["jeremy.olliver@gmail.com"]
+  s.homepage    = "http://github.com/heaps/resque-batched-logger"
+  s.summary     = %q{Allows resque jobs to be run in batches with aggregate logging}
+  s.description = %q{Allows resque jobs to be run in batches with aggregate logging, timing and statistics}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'resque'
+  # redis and json, are already dependencies of resque, but declaring these explicitly since these libraries are directly called from within this gem
+  s.add_dependency 'redis'
+  s.add_dependency 'json'
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'rcov'
+end

data/test/helper.rb

@@ -0,0 +1,39 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'minitest/unit'
+require 'resque'
+
+# Namespace our tests
+Resque.redis = Redis::Namespace.new('batched-logger/test:', :redis => Resque.redis)
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'resque-batched-logger'
+require 'shared_utilities'
+
+class MiniTest::Unit::TestCase
+  # def setup
+  #   global_teardown
+  # end
+end
+
+def global_teardown
+  # Don't do a flushdb on redis, that doesn't respect the namespace
+  Resque.redis.keys("*:jobcount").each {|k| Resque.redis.del("#{k.to_s}") }     # Clear our job count
+  Resque.redis.keys("batch_stats:*").each {|k| Resque.redis.del("#{k.to_s}") }  # Clear the lists of job stats
+  Resque.clear_test_jobs
+  SampleJob.clear_history
+  SampleModuleJob.clear_history
+  FileUtils.rm(Resque::Plugins::BatchedLogger::LOG_FILE) if File.exist?(Resque::Plugins::BatchedLogger::LOG_FILE)
+end
+global_teardown
+
+
+MiniTest::Unit.autorun

data/test/shared_utilities.rb

@@ -0,0 +1,72 @@
+# Test Helper classes
+class SampleJob
+  @@job_history = []
+  @queue = 'sample_job_queue'
+  include Resque::Plugins::BatchedLogging
+
+  def self.perform(*args)
+    @@job_history << args
+    sleep(0.5)
+  end
+
+  def self.job_history
+    @@job_history
+  end
+  def self.clear_history
+    @@job_history = []
+  end
+end
+
+class BatchedSampleJob < SampleJob
+  # This subclass is simply for namespacing batched 'SampleJob's
+end
+
+module SampleModuleJob
+  @@job_history = []
+  include Resque::Plugins::BatchedLogging
+
+  def self.perform(*args)
+    @@job_history << args
+    sleep(0.5)
+  end
+  def self.job_history
+    @@job_history
+  end
+  def self.clear_history
+    @@job_history = []
+  end
+end
+
+# Test Overrides
+module Resque
+  @@test_jobs = []
+  class TestJob
+    attr_accessor :args
+    def initialize(*args)
+      @args = args
+    end
+    def perform
+      local_payload = {'class' => @args.shift, 'args' => @args }
+      # Perform using Resque::Job, because that's what implements the hooks we need to test
+      Resque::Job.new('test_queue', local_payload).perform
+    end
+  end
+  def self.enqueue(*args)
+    @@test_jobs << Resque::TestJob.new(*args)
+  end
+  def self.test_jobs
+    @@test_jobs
+  end
+  def self.clear_test_jobs
+    @@test_jobs = []
+  end
+  def self.perform_test_jobs(options = {})
+    number_processed = 0
+    while job = @@test_jobs.shift # Pop from the front of the array of pending jobs
+      job.perform
+      number_processed +=1
+      break if number_processed == options[:limit]
+    end
+    number_processed
+  end
+end

data/test/test_batched_logger.rb

@@ -0,0 +1,72 @@
+require 'helper'
+
+class TestBatchedLogger < MiniTest::Unit::TestCase
+
+  require 'parsedate'
+
+  def test_log_format
+    arguments = [[1,2,3], [5,6,{:custom => :options}]]
+    batch_name = "SampleJob"
+    SampleJob.batched do
+      arguments.each do |arg|
+        enqueue(*arg)
+      end
+    end
+    Resque.perform_test_jobs
+    logged_data = File.read(Resque::Plugins::BatchedLogger::LOG_FILE)
+    assert_message_logged_with_valid_times(logged_data, Regexp.new("==== Batched jobs \'#{batch_name}\' : logged at (.*) ===="))
+    assert_message_logged_with_valid_times(logged_data, /batch started processing at: (.*)/)
+    assert_message_logged_with_valid_times(logged_data, /batch finished processing at: (.*)/)
+    assert logged_data.match(/Total run time for batch: [\d\.]+ seconds/), "log should include total run time"
+    assert logged_data.match(/Jobs Enqueued: 2/), "log should include number of jobs enqueued"
+    assert logged_data.match(/Jobs Processed: 2/), "log should include number of jobs processed"
+    assert logged_data.match(/Average time per job: [\d\.]+ seconds/), "log should include average time per job"
+    assert logged_data.match(/Total time spent processing jobs: [\d\.]+ seconds/), "log should total time spent processing"
+    assert_message_logged_with_valid_times(logged_data, Regexp.new("==== Batched jobs \'#{batch_name}\' completed at (.*) took [\\d\\.]+ seconds ===="))
+
+    assert total_run_time = logged_data.match(/Total run time for batch: ([\d\.]+) seconds/)[1]
+    assert final_completion_time = logged_data.match(Regexp.new("==== Batched jobs \'#{batch_name}\' completed at .* took ([\\d\\.]+) seconds ===="))[1]
+    assert_equal(total_run_time, final_completion_time, "Final completion length should match total time for batch")
+
+    assert_empty Resque.redis.keys("*:jobcount")
+    assert_empty Resque.redis.keys("batch_stats:*")
+  end
+
+  def test_logger_running_before_jobs_finished
+    Resque::Plugins::BatchedLogger.perform("SampleJob") # Should do no work
+    assert_empty Resque.test_jobs
+    arguments = [[1,2,3], [5,6,{:custom => :options}]]
+    SampleJob.batched do
+      arguments.each do |arg|
+        enqueue(*arg)
+      end
+    end
+    assert_equal 3, Resque.test_jobs.size
+    Resque.test_jobs.pop.perform # Try doing the batch logger first, should be requeued
+    assert_equal 3, Resque.test_jobs.size, "The batch logger should have been requeued"
+    Resque.perform_test_jobs
+    assert_empty Resque.test_jobs
+  end
+
+  def teardown
+    global_teardown
+  end
+
+  protected
+
+  # Line must match overall Regexp 'format', & any selected options with an () in the regexp must be a valid time string
+  def assert_message_logged_with_valid_times(string, format)
+    matches = string.match(format)
+    assert matches, "input #{string} did not match the specified format: #{format.inspect}"
+    matches[1..-1].each do |m|
+      assert valid_date?(m), "Expected #{m} to be a valid date"
+    end
+  end
+
+  def valid_date?(string)
+    # Returns true if the string's Year, Month and Day can be correctly determined
+    # parsedate returns an array of values for a date/time starting with year, decreasing in size
+    (d = ParseDate.parsedate(string)) && d[0..2].compact.size == 3
+  end
+
+end

data/test/test_batched_logging.rb

@@ -0,0 +1,107 @@
+require 'helper'
+
+class TestBatchedLogging < MiniTest::Unit::TestCase
+
+  def test_class_extensions
+    defined_methods = [:batched, :around_perform_log_as_batched]
+    [SampleJob, SampleModuleJob, BatchedSampleJob].each do |klass|
+      defined_methods.each do |meth_name|
+        assert klass.respond_to?(meth_name), "##{meth_name} should be defined on #{klass.to_s}"
+      end
+    end
+  end
+
+  def test_enqueueing_a_batch
+    arguments = [[1,2,3], [5,6,{:custom => :options}]]
+    SampleJob.batched do
+      arguments.each do |arg|
+        enqueue(*arg)
+      end
+    end
+    expected_job_list = arguments.collect {|j| [SampleJob] + j } # Jobs should be enqueued with correct Job class, and batch group
+    expected_job_list << [Resque::Plugins::BatchedLogger, "SampleJob"] # We expect the BatchedLogger to have been enqueued as well
+    assert_equal expected_job_list, Resque.test_jobs.collect(&:args), "Enqueued arguments should have a batch name hash appened"
+
+    Resque.perform_test_jobs
+    assert_equal arguments, SampleJob.job_history, "The processing job should have recieved the arguments without the :batched_log_group options hash"
+    assert_empty Resque.test_jobs, "Queue should be empty"
+  end
+
+  def test_enqueueing_without_batched
+    arguments = [[1,2,3], [5,6,{:custom => :options}]]
+    arguments.each do |args|
+      Resque.enqueue(SampleJob, *args)
+    end
+    expected_job_list = arguments.collect {|j| [SampleJob] + j }
+    assert_equal expected_job_list, Resque.test_jobs.collect(&:args), "Enqueued arguments should be unmodified"
+
+    Resque.perform_test_jobs
+    assert_equal arguments, SampleJob.job_history, "The arguments recieved by the actual job should have been unmodified"
+    assert_empty Resque.test_jobs, "Queue should be empty"
+  end
+
+  def test_queuing_multiple_batches
+    arguments = [[1,2,3], [4,5], [5,6,{:custom => :options}]]
+    # Should be able to do this twice without raising an error
+    2.times do
+      SampleJob.batched do
+        arguments.each do |args|
+          enqueue(*args)
+        end
+      end
+      assert_equal 4, Resque.test_jobs.size, "3 jobs + the logger job should have been queued"
+      Resque.perform_test_jobs
+      assert_empty Resque.test_jobs
+    end
+  end
+
+  def test_queueing_nothing
+    SampleJob.batched do
+      [].each do |args|
+        enqueue(*args)
+      end
+    end
+    Resque.perform_test_jobs
+  end
+
+  def test_single_jobs_dont_interrupt_batch
+    arguments = [[1,2,3], [4,5], [5,6,{:custom => :options}]]
+    SampleJob.batched do
+      enqueue(*arguments[0]) # Enqueue the first job as a batch
+      Resque.enqueue(SampleJob, *arguments[1]) # Enqueueing a non batched job in the middle of the batch being queued up (should be processed independently)
+      enqueue(*arguments[2]) # Enqueueing a second batched job
+    end
+    assert_equal 4, Resque.test_jobs.size, "3 jobs + the logger job should have been queued"
+    assert_equal 2, Resque.redis.get("SampleJob:jobcount").to_i, "should have listed 2 jobs on the queue"
+    Resque.perform_test_jobs(:limit => 3) # Process just the first 3 jobs (2 batched, 1 individual, don't process the logs yet)
+    assert_equal arguments, SampleJob.job_history
+    assert_equal 3, Resque.redis.llen("batch_stats:SampleJob"), "All 3 jobs will have been processed as batched"
+    Resque.perform_test_jobs # Do the log processing
+    assert_empty Resque.test_jobs
+  end
+
+  # Same test as 'test_single_jobs_dont_interrupt_batch', except we'll batch with the subclass, and hence the enqueueing of the superclas won't effect our logged job count
+  def test_sub_classed_batch_jobs
+    assert BatchedSampleJob.respond_to?(:perform)
+    assert_equal SampleJob, BatchedSampleJob.superclass
+
+    arguments = [[1,2,3], [4,5], [5,6,{:custom => :options}]]
+    BatchedSampleJob.batched do
+      enqueue(*arguments[0]) # Enqueue the first job as a batch
+      Resque.enqueue(SampleJob, *arguments[1]) # Enqueueing a non batched job in the middle of the batch being queued up (should be processed independently)
+      enqueue(*arguments[2]) # Enqueueing a second batched job
+    end
+    assert_equal 4, Resque.test_jobs.size, "3 jobs + the logger job should have been queued"
+    assert_equal 2, Resque.redis.get("BatchedSampleJob:jobcount").to_i, "should have listed 2 jobs on the queue"
+    Resque.perform_test_jobs(:limit => 3) # Process just the first 3 jobs (2 batched, 1 individual, don't process the logs yet)
+    assert_equal arguments, SampleJob.job_history
+    assert_equal 2, Resque.redis.llen("batch_stats:BatchedSampleJob"), "Only the 2 batched jobs should have been processed as batched"
+    Resque.perform_test_jobs # Do the log processing
+    assert_empty Resque.test_jobs
+  end
+
+  def teardown
+    global_teardown
+  end
+
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification 
+name: resque-batched-logger
+version: !ruby/object:Gem::Version 
+  hash: 13
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 9
+  version: 0.0.9
+platform: ruby
+authors: 
+- Jeremy Olliver
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-31 00:00:00 +13:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: minitest
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+description: Allows resque jobs to be run in batches with aggregate logging, timing and statistics
+email: 
+- jeremy.olliver@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README
+- Rakefile
+- lib/resque-batched-logger.rb
+- lib/resque-batched-logger/version.rb
+- lib/resque/plugins/batched_logger.rb
+- lib/resque/plugins/batched_logging.rb
+- resque-batched-logger.gemspec
+- test/helper.rb
+- test/shared_utilities.rb
+- test/test_batched_logger.rb
+- test/test_batched_logging.rb
+has_rdoc: true
+homepage: http://github.com/heaps/resque-batched-logger
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Allows resque jobs to be run in batches with aggregate logging
+test_files: 
+- test/helper.rb
+- test/shared_utilities.rb
+- test/test_batched_logger.rb
+- test/test_batched_logging.rb