resque-clues 0.1.0

data/lib/resque/plugins/clues/queue_extension.rb

@@ -0,0 +1,69 @@
+require 'digest/md5'
+require 'time'
+
+module Resque
+  module Plugins
+    module Clues
+      # Module capable of redefining the Resque#push and Resque#pop methods so
+      # that:
+      #
+      # * metadata will be stored in redis.
+      # * The metadata can be injected with arbitrary data by a configured item
+      # preprocessor.
+      # * That event data (including its metadata) will be published, provided
+      # an event publisher has been configured.
+      module QueueExtension
+        # push an item onto the queue.  If resque-clues is configured, this
+        # will First create the metadata associated with the event and adds it
+        # to the item.  This will include:
+        #
+        # * event_hash: a unique item identifying the job, will be included
+        # with other events arising from that job.
+        # * hostname: the hostname of the machine where the event occurred.
+        # * process:  The process id of the ruby process where the event
+        # occurred.
+        # * plus any items injected into the item via a configured
+        # item_preprocessor.
+        #
+        # After that, an enqueued event is published and the original push
+        # operation is invoked.
+        #
+        # queue:: The queue to push onto
+        # orig:: The original item to push onto the queue.
+        def _clues_push(queue, item)
+          return _base_push(queue, item) unless Clues.clues_configured?
+          item['clues_metadata'] = {
+            'event_hash' => Clues.event_hash,
+            'hostname' => Clues.hostname,
+            'process' => Clues.process,
+            'enqueued_time' => Time.now.utc.to_f
+          }
+          if Resque::Plugins::Clues.item_preprocessor
+            Resque::Plugins::Clues.item_preprocessor.call(queue, item)
+          end
+          Clues.event_publisher.publish(:enqueued, Clues.now, queue, item['clues_metadata'], item[:class], *item[:args])
+          _base_push(queue, item)
+        end
+
+        # pops an item off the head of the queue.  This will use the original
+        # pop operation to get the item, then calculate the time in queue and
+        # broadcast a dequeued event.
+        #
+        # queue:: The queue to pop from.
+        def _clues_pop(queue)
+          _base_pop(queue).tap do |item|
+            # TODO refactor
+            unless item.nil?
+              if Clues.clues_configured? and item['clues_metadata']
+                item['clues_metadata']['hostname'] = Clues.hostname
+                item['clues_metadata']['process'] = Clues.process
+                item['clues_metadata']['time_in_queue'] = Clues.time_delta_since(item['clues_metadata']['enqueued_time'])
+                Clues.event_publisher.publish(:dequeued, Clues.now, queue, item['clues_metadata'], item['class'], *item['args'])
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/clues/util.rb

@@ -0,0 +1,45 @@
+require 'time'
+require 'delegate'
+
+module Resque
+  module Plugins
+    module Clues
+      # A unique event hash crafted from the hostname, process and time.
+      def self.event_hash
+        Digest::MD5.hexdigest("#{hostname}#{process}#{Time.now.utc.to_f}")
+      end
+
+      # The hostname Resque is running on.
+      def self.hostname
+        @hostname ||= Socket.gethostname
+      end
+
+      # The process id.
+      def self.process
+        $$
+      end
+
+      # The current UTC time in iso 8601 format.
+      def self.now
+        Time.now.utc.iso8601
+      end
+
+      # The delta of time between the passed time and now.
+      def self.time_delta_since(start)
+        result = Time.now.utc.to_f - start.to_f
+        result >= 0.0 ? result : 0.0
+      end
+
+      # Convenience method to determine if resque-clues is properly
+      # configured.
+      def self.clues_configured?
+        Resque::Plugins::Clues.configured?
+      end
+
+      # Convenience method to access the resque-clues event publisher.
+      def self.event_publisher
+        Resque::Plugins::Clues.event_publisher
+      end
+    end
+  end
+end

data/lib/resque/plugins/clues/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  module Clues
+    VERSION = "0.1.0"
+  end
+end

data/resque-clues.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque/plugins/clues/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "resque-clues"
+  gem.version       = Resque::Clues::VERSION
+  gem.authors       = ["Lance Woodson"]
+  gem.email         = ["lance.woodson@peopleadmin.com"]
+  gem.description   = %q{Adds event publishing and job tracking ability to Resque}
+  gem.summary       = %q{Adds event publishing and job tracking}
+  gem.homepage      = "https://github.com/PeopleAdmin/resque-clues"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency 'resque', '>= 1.20.0'
+  gem.add_dependency 'multi_json', '~> 1.7.4'
+  gem.add_development_dependency 'rake', '~> 0.9.2.2'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'pry-debugger'
+  gem.add_development_dependency 'cane'
+end

data/spec/event_publisher_spec.rb

@@ -0,0 +1,193 @@
+require 'spec_helper'
+require 'stringio'
+require 'json'
+require 'fileutils'
+require 'time'
+require 'tmpdir'
+
+describe 'event publishers' do
+
+  before do
+    @current_time = Time.now.utc.iso8601
+  end
+
+  def publish_event_type(type)
+    @publisher.publish(type, @current_time, :test_queue, {}, "FooBar", "a", "b")
+  end
+
+  describe Resque::Plugins::Clues::StreamPublisher do
+    before do
+      @stream = StreamIO.new
+      @publisher = Resque::Plugins::Clues::StreamPublisher.new(@stream)
+    end
+  end
+
+  describe Resque::Plugins::Clues::StreamPublisher do
+    def verify_output_for_event_type(type)
+      @stream.rewind
+      event = MultiJson.load(@stream.readlines[-1].chomp)
+      event["queue"].should == "test_queue"
+      event["metadata"].should == {}
+      event["timestamp"].should_not be_nil
+      event["event_type"].should == type.to_s
+      event["worker_class"].should == "FooBar"
+      event["args"].should == ["a", "b"]
+    end
+
+    before do
+      @stream = StringIO.new
+      @publisher = Resque::Plugins::Clues::StreamPublisher.new(@stream)
+    end
+
+    it "should pass Resque lint detection" do
+      Resque::Plugin.lint(Resque::Plugins::Clues::StandardOutPublisher)
+    end
+
+    it "should send enqueued event to STDOUT" do
+      publish_event_type :enqueued
+      verify_output_for_event_type :enqueued
+    end
+
+    it "should send dequeued event to STDOUT" do
+      publish_event_type :dequeued
+      verify_output_for_event_type :dequeued
+    end
+
+    it "should send perform_started event to STDOUT" do
+      publish_event_type :perform_started
+      verify_output_for_event_type :perform_started
+    end
+
+    it "should send perform_finished event to STDOUT" do
+      publish_event_type :perform_finished
+      verify_output_for_event_type :perform_finished
+    end
+
+    it "should send failed event to STDOUT" do
+      publish_event_type :failed
+      verify_output_for_event_type :failed
+    end
+
+    it "should send destroyed event to STDOUT" do
+      publish_event_type :destroyed
+      verify_output_for_event_type :destroyed
+    end
+  end
+
+
+  describe Resque::Plugins::Clues::LogPublisher do
+    def verify_event_written_to_log(event_type)
+      last_event["event_type"].should == event_type.to_s
+      last_event["timestamp"].should == @current_time.to_s
+      last_event["queue"].should == 'test_queue'
+      last_event["metadata"].should == {}
+      last_event["worker_class"].should == "FooBar"
+      last_event["args"].should == ["a", "b"]
+    end
+
+    def last_event
+      MultiJson.load(File.readlines(@log_path)[-1].chomp)
+    end
+
+    before do
+      @log_path = File.join(Dir.tmpdir, "test_log.log")
+      FileUtils.rm(@log_path) if File.exists?(@log_path)
+      @publisher = Resque::Plugins::Clues::LogPublisher.new(@log_path)
+    end
+
+    it "should pass Resque lint detection" do
+      Resque::Plugin.lint(Resque::Plugins::Clues::LogPublisher)
+    end
+
+    it "should write enqueued event to file" do
+      publish_event_type :enqueued
+      verify_event_written_to_log :enqueued
+    end
+
+    it "should write dequeued event to file" do
+      publish_event_type :dequeued
+      verify_event_written_to_log :dequeued
+    end
+
+    it "should write destroyed event to file" do
+      publish_event_type :destroyed
+      verify_event_written_to_log :destroyed
+    end
+
+    it "should write perform_started event to file" do
+      publish_event_type :perform_started
+      verify_event_written_to_log :perform_started
+    end
+
+    it "should write perform_finished event to file" do
+      publish_event_type :perform_finished
+      verify_event_written_to_log :perform_finished
+    end
+
+    it "should write failed event to file" do
+      publish_event_type :failed
+      verify_event_written_to_log :failed
+    end
+
+    it "should write 1 event per line in the file" do
+      publish_event_type :enqueued
+      publish_event_type :dequeued
+      File.readlines(@log_path)[1..-1].size.should == 2
+    end
+  end
+
+  describe Resque::Plugins::Clues::CompositePublisher do
+    before do
+      @publisher = Resque::Plugins::Clues::CompositePublisher.new
+      @publisher << Resque::Plugins::Clues::StandardOutPublisher.new
+      @publisher << Resque::Plugins::Clues::StandardOutPublisher.new
+    end
+
+    def verify_event_delegated_to_children(event_type)
+      @publisher.each do |child|
+        child.should_receive(:publish).with(
+          event_type, @current_time, :test_queue, {}, "FooBar", "a", "b")
+      end
+    end
+
+    it "should pass Resque lint detection" do
+      Resque::Plugin.lint(Resque::Plugins::Clues::CompositePublisher)
+    end
+
+    it "should delegate enqueued event to children" do
+      verify_event_delegated_to_children :enqueued
+      publish_event_type :enqueued
+    end
+
+    it "should delegate dequeued event to children" do
+      verify_event_delegated_to_children :dequeued
+      publish_event_type :dequeued
+    end
+
+    it "should delegate destroyed event to children" do
+      verify_event_delegated_to_children :destroyed
+      publish_event_type :destroyed
+    end
+
+    it "should delegate perform_started event to children" do
+      verify_event_delegated_to_children :perform_started
+      publish_event_type :perform_started
+    end
+
+    it "should delegate perform_finished event to children" do
+      verify_event_delegated_to_children :perform_finished
+      publish_event_type :perform_finished
+    end
+
+    it "should delegate failed event to children" do
+      verify_event_delegated_to_children :failed
+      publish_event_type :failed
+    end
+
+    it "all children should be invoked when the first child throws an exception" do
+      @publisher[0].stub(:enqueued) {raise 'YOU SHALL NOT PASS'}
+      verify_event_delegated_to_children :enqueued
+      publish_event_type :enqueued
+    end
+  end
+end

data/spec/integration_spec.rb

@@ -0,0 +1,183 @@
+require 'spec_helper'
+require 'stringio'
+require 'timeout'
+require 'json'
+
+# Setting $TESTING to true will cause Resque to not fork a child to perform the
+# job, doing all the work within the test process and allowing us to test
+# results for all aspects of the job performance.
+$TESTING = true
+
+class DummyWorker
+  @queue = :test_queue
+
+  def self.invoked?
+    @invoked
+  end
+
+  def self.reset!
+    @invoked = false
+  end
+
+  def self.perform(msg)
+    @invoked = true
+  end
+end
+
+class FailingDummyWorker
+  @queue = :test_queue
+
+  def self.perform(msg)
+    raise msg
+  end
+end
+
+describe 'end-to-end integration' do
+  before(:each) do
+    DummyWorker.reset!
+    @stream = StringIO.new
+    @worker = Resque::Worker.new(:test_queue)
+    #@worker.very_verbose = true
+    Resque::Plugins::Clues.event_publisher = Resque::Plugins::Clues::StreamPublisher.new(@stream)
+  end
+
+  def stream_size
+    @stream.rewind
+    @stream.readlines.size
+  end
+
+  def enqueue_then_verify(klass, *args, &block)
+    Resque.enqueue(klass, *args)
+    work_and_verify(&block)
+  end
+
+  def work
+    timeout(0.2) do
+      @worker.work(0.1)
+    end
+  end
+
+  def work_and_verify(&block)
+    work
+    @stream.rewind
+    block.call(@stream.readlines.map{|line| MultiJson.decode(line)})
+  end
+
+  def verify_event(event, type, klass, *args)
+    event["worker_class"].should == klass.to_s
+    event["args"].should == args
+    event["event_type"].should == type.to_s
+    event["timestamp"].should_not be_nil
+    event["metadata"]["event_hash"].should_not be_nil
+    event["metadata"]["hostname"].should == `hostname`.strip
+    event["metadata"]["process"].should == $$
+    yield(event) if block_given?
+  end
+
+  context "for job that finishes normally" do
+    it "should publish enqueued, dequeued, perform_started and perform_finished events" do
+      enqueue_then_verify(DummyWorker, 'test') do |events|
+        events.size.should == 4
+        verify_event(events[0], :enqueued, DummyWorker, "test")
+        verify_event(events[1], :dequeued, DummyWorker, "test") do |event|
+          event["metadata"]["time_in_queue"].should_not be_nil
+        end
+        verify_event(events[2], :perform_started, DummyWorker, "test")
+        verify_event(events[3], :perform_finished, DummyWorker, "test") do |event|
+          event["metadata"]["time_to_perform"].should_not be_nil
+        end
+      end
+    end
+  end
+
+  context "for job that fails" do
+    it "should publish enqueued, dequeued, perform_started and failed events" do
+      enqueue_then_verify(FailingDummyWorker, 'test') do |events|
+        events.size.should == 4
+        verify_event(events[0], :enqueued, FailingDummyWorker, "test")
+        verify_event(events[1], :dequeued, FailingDummyWorker, "test") do |event|
+          event["metadata"]["time_in_queue"].should_not be_nil
+        end
+        verify_event(events[2], :perform_started, FailingDummyWorker, "test")
+        verify_event(events[3], :failed, FailingDummyWorker, "test") do |event|
+          event["metadata"]["time_to_perform"].should_not be_nil
+          event["metadata"]["exception"].should == RuntimeError.to_s
+          event["metadata"]["message"].should == 'test'
+          event["metadata"]["backtrace"].should_not be_nil
+        end
+      end
+    end
+  end
+
+  context "for job enqueued prior to use of resque-clues gem" do
+    def enqueue_unpatched(worker, *args)
+      unpatch_resque
+      begin
+        Resque.enqueue(worker, *args)
+      ensure
+        repatch_resque
+      end
+    end
+
+    context "job that performs normally" do
+      before do
+        enqueue_unpatched(DummyWorker, "test")
+      end
+
+      it "should succeed without failures" do
+        work
+        DummyWorker.invoked?.should == true
+        Resque::Failure.all.should == nil
+      end
+    end
+
+    context "job failures" do
+      before do
+        enqueue_unpatched(FailingDummyWorker, "test")
+      end
+
+      it "should report failure normally" do
+        work
+        Resque::Failure.count.should == 1
+      end
+    end
+  end
+
+  context "where job enqueued with resque-clues gem but worker performing job is not" do
+    def unpatch_and_work
+      unpatch_resque
+      work
+      yield if block_given?
+    ensure
+      repatch_resque
+    end
+
+    context "for job that performs normally" do
+      before {Resque.enqueue DummyWorker, "test"}
+
+      it "should succeed without failures" do
+        unpatch_and_work {Resque::Failure.all.should == nil}
+      end
+    end
+
+    context "for job failures" do
+      before {Resque.enqueue FailingDummyWorker, "test"}
+
+      it "should report failure normally" do
+        unpatch_and_work {Resque::Failure.count.should == 1}
+      end
+    end
+  end
+
+  context "Resque internals assumptions" do
+    describe "Resque#push" do
+      it "should receive item with class and args as symbols" do
+        received_item = nil
+        Resque.stub(:push) {|queue, item| received_item = item}
+        Resque.enqueue(DummyWorker, 'test')
+        received_item[:class].should == "DummyWorker"
+        received_item[:args].should == ['test']
+      end
+    end
+  end
+end