dispatch-rider 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 865cf06c42c11dd3d75306800df557467d80a114
-  data.tar.gz: c86261b609d01cc2db299bf8cd5a74be20de1a95
+  metadata.gz: 20ecab8e18af62e4795b431f0860148f5929277c
+  data.tar.gz: b87e0c608b4ee99a1e85279bf9d42afc5b4e8b53
 SHA512:
-  metadata.gz: ac475def2ee3a4fa688d4dd693da7da42e29f6826f880e9142532e5026533af96eec09d3d8bd79e735fbb09e7ce63d6a028a3a3c1bd8639e150d76e200ea5243
-  data.tar.gz: 28def6f13e59297cc5269443c94f22121fa87bae86d0fc6f02e436fe285ffb3d6605b245da91a56728b860c579238d7337d2a7beedc21aa60c7ffe536fc4e620
+  metadata.gz: 9e645f3c4257d14f8d17e1402ec5ef73ceee737b380ffa531391a010ad28e11fba30baa806af34d1d21f518583fd6bc44651fc65e5bc36a7f698f50c4edc1158
+  data.tar.gz: 8aa77e404a4caa1680def056db848c6c124c1e1ee935278770b9a35c8e278100a23c5b347b80c2bdadc5c62c3e2b8a494b8f2f5850af531abc46299b824811ea

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Change Log
 
+## [v1.6.0](https://github.com/payrollhero/dispatch-rider/tree/v1.6.0) (2015-06-20)
+
+[Full Changelog](https://github.com/payrollhero/dispatch-rider/compare/v1.5.3...v1.6.0)
+
+**Closed issues:**
+
+- ActiveJob Support [\#50](https://github.com/payrollhero/dispatch-rider/issues/50)
+
+**Merged pull requests:**
+
+- Adding "long polling" to filesystem queue driver [\#64](https://github.com/payrollhero/dispatch-rider/pull/64) ([piotrb](https://github.com/piotrb))
+
+- Rspec upgrade [\#63](https://github.com/payrollhero/dispatch-rider/pull/63) ([piotrb](https://github.com/piotrb))
+
+- Add JSON log formatter [\#62](https://github.com/payrollhero/dispatch-rider/pull/62) ([mykola-kyryk](https://github.com/mykola-kyryk))
+
+- Extract logging from Demultiplexer into a separate module. Add log fo… [\#61](https://github.com/payrollhero/dispatch-rider/pull/61) ([mykola-kyryk](https://github.com/mykola-kyryk))
+
+## [v1.5.3](https://github.com/payrollhero/dispatch-rider/tree/v1.5.3) (2015-05-21)
+
+[Full Changelog](https://github.com/payrollhero/dispatch-rider/compare/v1.4.2...v1.5.3)
+
+**Merged pull requests:**
+
+- Allow setting of message guid [\#59](https://github.com/payrollhero/dispatch-rider/pull/59) ([more-ron](https://github.com/more-ron))
+
+- Make the overriden function "publish to channel" be the only thing overriden [\#58](https://github.com/payrollhero/dispatch-rider/pull/58) ([more-ron](https://github.com/more-ron))
+
+- Add publishing callbacks [\#57](https://github.com/payrollhero/dispatch-rider/pull/57) ([more-ron](https://github.com/more-ron))
+
+- Updating rspec and some of the depracations [\#56](https://github.com/payrollhero/dispatch-rider/pull/56) ([piotrb](https://github.com/piotrb))
+
 ## [v1.4.2](https://github.com/payrollhero/dispatch-rider/tree/v1.4.2) (2015-04-10)
 
 [Full Changelog](https://github.com/payrollhero/dispatch-rider/compare/v1.4.1...v1.4.2)
@@ -266,8 +298,6 @@
 
 - Added process naming to dispatcher. [\#23](https://github.com/payrollhero/dispatch-rider/pull/23) ([link664](https://github.com/link664))
 
-- Fail job after running error handler. [\#27](https://github.com/payrollhero/dispatch-rider/pull/27) ([link664](https://github.com/link664))
-
 ## [v0.0.5](https://github.com/payrollhero/dispatch-rider/tree/v0.0.5) (2013-08-20)
 
 [Full Changelog](https://github.com/payrollhero/dispatch-rider/compare/2013-06-06-13-08-PST...v0.0.5)
@@ -314,10 +344,6 @@
 
 - Fix double serialize on file system bug [\#15](https://github.com/payrollhero/dispatch-rider/pull/15) ([more-ron](https://github.com/more-ron))
 
-- Add publisher configure method [\#16](https://github.com/payrollhero/dispatch-rider/pull/16) ([more-ron](https://github.com/more-ron))
-
--  Initial implementation of hash based configuration to satisfy issue \#10 [\#11](https://github.com/payrollhero/dispatch-rider/pull/11) ([jimbomt](https://github.com/jimbomt))
-
 ## [2013-05-28-15-13+08-00](https://github.com/payrollhero/dispatch-rider/tree/2013-05-28-15-13+08-00) (2013-05-28)
 
 [Full Changelog](https://github.com/payrollhero/dispatch-rider/compare/2013-05-28...2013-05-28-15-13+08-00)

data/README.md

@@ -212,6 +212,12 @@ DispatchRider.config do |config|
   end
 
   config.logger = Rails.logger
+  # Log Formatter
+  # Text Formatter (default)
+  # config.log_formatter = DispatchRider::Logging::TextFormatter.new
+  # JSON Formatter
+  # config.log_formatter = DispatchRider::Logging::JsonFormatter.new
+
   config.default_retry_timeout = 300
 
   config.error_handler = DispatchRider::DefaultErrorHandler # an object that responds to .call(message, exception)
@@ -226,7 +232,7 @@ end
 Options:
 
  * `logger` : what logger to use to send messages to (responds to the standard ruby Logger protocol), defaults to a new Logger sending messages to STDERR
-
+ * `log_formatter` : what log formatter to use with logger, defaults to DispatchRider::Logging::TextFormatter.new
 
 ### Callbacks
 

data/lib/dispatch-rider.rb

@@ -20,6 +20,10 @@ module DispatchRider
       @configuration ||= Configuration.new
     end
     alias_method :config, :configuration
+
+    def clear_configuration!
+      @configuration = nil
+    end
   end
 end
 

data/lib/dispatch-rider/configuration.rb

@@ -14,7 +14,7 @@ module DispatchRider
       @logger = Logger.new(STDERR)
       @debug = false
 
-      @callbacks.around(:handle_message) do |job, message|
+      @callbacks.around(:dispatch_message) do |job, message|
         Logging::LifecycleLogger.wrap_handling(message) do
           job.call
         end

data/lib/dispatch-rider/demultiplexer.rb

@@ -55,13 +55,17 @@ module DispatchRider
 
     def do_loop
       catch(:done) do
-        loop do
+        while keep_going?
           throw :done unless @continue
           yield
         end
       end
     end
 
+    def keep_going?
+      true
+    end
+
     def handle_next_queue_item
       queue.pop do |message|
         dispatch_message(message)

data/lib/dispatch-rider/logging/json_formatter.rb

@@ -34,21 +34,27 @@ module DispatchRider
       # @return [String] JSON representation of the log item
       def format_handling(kind, message, exception: nil, duration: nil)
         as_json do
-          case kind
-          when :start
-            message_info_fragment(message)
-          when :success
-            message_info_fragment(message)
-          when :fail
-            exception_info_fragment(message, exception)
-          when :complete
-            { duration: format_duration(duration) }.merge message_info_fragment(message)
-          end.merge(phase: kind)
+          fragment = case kind
+                     when :start
+                       message_info_fragment(message)
+                     when :success
+                       message_info_fragment(message)
+                     when :fail
+                       exception_info_fragment(message, exception)
+                     when :complete
+                       duration_fragment = { duration: format_duration(duration) }
+                       message_info_fragment(message).merge duration_fragment
+                     end
+          { phase: kind }.merge fragment
         end
       end
 
       private
 
+      def format_duration(duration)
+        duration
+      end
+
       def as_json
         JSON.generate yield
       end

data/lib/dispatch-rider/logging/lifecycle_logger.rb

@@ -11,8 +11,8 @@ module DispatchRider
         end
 
         def wrap_handling(message)
-          log_start(message)
           start_time = Time.now
+          log_start(message)
           yield
           log_success(message)
         rescue => exception

data/lib/dispatch-rider/runner.rb

@@ -16,7 +16,7 @@ module DispatchRider
 
     def process
       callbacks.invoke(:process) do
-        puts "Running..."
+        logger.info "Running..."
         @subscriber.process
       end
     end

data/lib/dispatch-rider/version.rb

@@ -1,4 +1,4 @@
 # This file specifies the current version of the gem.
 module DispatchRider
-  VERSION = "1.6.0"
+  VERSION = "1.6.1"
 end

data/spec/integration/logging_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+
+describe "Logging" do
+  let(:publisher) { setup_publisher }
+  let(:subscriber) { setup_subscriber }
+
+  let(:mock_logger_class) do
+    Class.new {
+      def initialize
+        @log = []
+      end
+
+      attr_reader :log
+
+      def info(message)
+        @log << { level: :info, message: message }
+      end
+
+      def error(message)
+        @log << { level: :error, message: message }
+      end
+    }
+  end
+
+  let(:mock_logger) { mock_logger_class.new }
+
+  before do
+    purge_test_queue
+    DispatchRider.config.log_formatter = DispatchRider::Logging::JsonFormatter.new
+    DispatchRider.config.logger = mock_logger
+    DispatchRider.config.debug = true
+  end
+
+  after do
+    DispatchRider.clear_configuration!
+  end
+
+  context "successful handler" do
+    before do
+      publisher.publish message: { subject: 'sample_handler', body: {} }, destinations: :dst
+      work_off_jobs(subscriber)
+    end
+
+    example "full logging integration in json mode" do
+      expected1 = {
+        'phase' => 'start',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'sample_handler',
+        'body' => {},
+      }
+
+      expected2 = {
+        'phase' => 'success',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'sample_handler',
+        'body' => {},
+      }
+
+      expected3 = {
+        'phase' => 'complete',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'sample_handler',
+        'body' => {},
+      }
+
+      expect(mock_logger.log.count).to eq(3)
+
+      expect(mock_logger.log[0]).to eq(level: :info, message: expected1.to_json)
+      expect(mock_logger.log[1]).to eq(level: :info, message: expected2.to_json)
+
+      # last one is a bit harder since it has a relative 'duration' value
+      entry = mock_logger.log[2]
+      expect(entry[:level]).to eq(:info)
+      payload = JSON.load(entry[:message])
+      expect(payload['duration']).to be_a(Numeric)
+      payload.delete 'duration'
+      expect(payload).to eq(expected3)
+    end
+  end
+
+  context "failing handler" do
+    before do
+      publisher.publish message: { subject: 'crashing_handler', body: {} }, destinations: :dst
+      work_off_jobs(subscriber, fail_on_error: false)
+    end
+
+    example "full logging integration in json mode" do
+      expected1 = {
+        'phase' => 'start',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'crashing_handler',
+        'body' => {},
+      }
+
+      expected2 = {
+        'phase' => 'fail',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'crashing_handler',
+        'body' => {},
+        'expection' => {
+          'class' => 'RuntimeError',
+          'message' => 'I crashed!',
+        },
+      }
+
+      expected3 = {
+        'phase' => 'complete',
+        'guid' => 'test-mode-not-random-guid',
+        'subject' => 'crashing_handler',
+        'body' => {},
+      }
+
+      expect(mock_logger.log.count).to eq(3)
+
+      expect(mock_logger.log[0]).to eq(level: :info, message: expected1.to_json)
+      expect(mock_logger.log[1]).to eq(level: :error, message: expected2.to_json)
+
+      # last one is a bit harder since it has a relative 'duration' value
+      entry = mock_logger.log[2]
+      expect(entry[:level]).to eq(:info)
+      payload = JSON.load(entry[:message])
+      expect(payload['duration']).to be_a(Numeric)
+      payload.delete 'duration'
+      expect(payload).to eq(expected3)
+    end
+  end
+end

data/spec/lib/dispatch-rider/demultiplexer_spec.rb

@@ -59,7 +59,6 @@ describe DispatchRider::Demultiplexer, nodb: true do
         demultiplexer_thread.run
         sleep 0.01 # give it a chance to process the job async before killing the demux
       end
-
       # THIS ALSO TESTS THAT THE JOB IS NOT RUN MULTIPLE TIMES
       # IF THIS FAILS, BE CAREFUL NOT TO INTRODUCE BUGS
       it "should call the correct handler" do

data/spec/lib/dispatch-rider/dispatcher_spec.rb

@@ -14,12 +14,18 @@ describe DispatchRider::Dispatcher, :nodb => true do
   end
 
   describe "#dispatch" do
-    let(:message) { DispatchRider::Message.new(subject: "handle_something", body: { do_throw_something: true }) }
+    let(:message_subject) { "handle_something" }
+    let(:message_body) { { do_throw_something: true } }
+    let(:fs_message) do
+      DispatchRider::Message.new(subject: message_subject, body: message_body.merge('guid' => 123))
+    end
+    let(:item) { double :item }
+    let(:queue) { double :queue }
+    let(:message) { DispatchRider::QueueServices::FileSystem::FsReceivedMessage.new(fs_message, item, queue) }
 
     describe "callbacks" do
       let(:dummy) { double(:dummy) }
       let(:storage) { DispatchRider::Callbacks::Storage.new }
-      let(:message) { DispatchRider::Message.new(subject: "handle_something", body: { do_throw_something: true }) }
 
       before do
         allow(DispatchRider.config).to receive(:callbacks) { storage }
@@ -55,9 +61,7 @@ describe DispatchRider::Dispatcher, :nodb => true do
     end
 
     context "when the handler returns false" do
-      let(:message) do
-        DispatchRider::Message.new(subject: "handler_that_returns_false", body: { do_throw_something: true })
-      end
+      let(:message_subject) { "handler_that_returns_false" }
 
       before :each do
         subject.register('handler_that_returns_false')

data/spec/lib/dispatch-rider/logging/json_formatter_spec.rb

@@ -56,7 +56,7 @@ describe DispatchRider::Logging::JsonFormatter do
     end
 
     context "complete" do
-      let(:formatted_message) { result_object.merge("phase" => "complete", "duration" => "2.00") }
+      let(:formatted_message) { result_object.merge("phase" => "complete", "duration" => 2.0) }
       let(:result_message) { JSON.parse subject.format_handling(:complete, message, duration: 2.0) }
 
       example { expect(result_message).to eq(formatted_message) }

data/spec/lib/dispatch-rider/runner_spec.rb

@@ -11,6 +11,10 @@ describe DispatchRider::Runner do
       end
     end
 
+    after do
+      DispatchRider.clear_configuration!
+    end
+
     example do
       expect(subscriber).to receive(:new).once.and_return(subscriber)
       expect(subscriber).to receive(:register_queue).once

data/spec/lib/dispatch-rider/subscriber_spec.rb

@@ -1,6 +1,14 @@
 require "spec_helper"
 
 describe DispatchRider::Subscriber do
+  let(:message_subject) { "handle_something" }
+  let(:message_body) { { do_throw_something: true } }
+  let(:fs_message) do
+    DispatchRider::Message.new(subject: message_subject, body: message_body.merge('guid' => 123))
+  end
+  let(:item) { double :item }
+  let(:queue) { double :queue }
+  let(:message) { DispatchRider::QueueServices::FileSystem::FsReceivedMessage.new(fs_message, item, queue) }
 
   before do
     allow(DispatchRider::Handlers::Base).to receive(:subclasses) { Set.new }
@@ -27,20 +35,19 @@ describe DispatchRider::Subscriber do
   end
 
   describe "#register_handler" do
+    let(:message_subject) { :foo_bar }
+    let(:message_body) { { 'foo' => 'bar' } }
+
     it "should register a handler" do
       subject.register_handler(:foo_bar)
       expect {
-        message = DispatchRider::Message.new(subject: :foo_bar, body: { 'foo' => 'bar' })
         subject.dispatcher.dispatch(message)
       }.to throw_symbol(:process_was_called)
     end
-  end
 
-  describe "#register_handlers" do
     it "should register all the handlers" do
       subject.register_handlers(:foo_bar)
       expect {
-        message = DispatchRider::Message.new(subject: :foo_bar, body: { 'foo' => 'bar' })
         subject.dispatcher.dispatch(message)
       }.to throw_symbol(:process_was_called)
     end
@@ -82,8 +89,10 @@ describe DispatchRider::Subscriber do
     # kills travis sometimes so leaving it here as tested documentation
     describe "process termination", if: false do
       before { allow(subject.demultiplexer).to receive(:stop) { throw :got_stopped } }
+      let(:message_body) { { 'foo' => 'bar' } }
 
       context "when process quits" do
+        let(:message_subject) { :quiter }
         before do
           konst = Class.new(DispatchRider::Handlers::Base) do
             def process(_options)
@@ -93,7 +102,6 @@ describe DispatchRider::Subscriber do
           stub_const("Quiter", konst)
 
           subject.register_handler(:quiter)
-          message = DispatchRider::Message.new(subject: :quiter, body: {})
           subject.queue_service_registrar.fetch(:simple).push(message)
         end
 
@@ -101,6 +109,7 @@ describe DispatchRider::Subscriber do
       end
 
       context "when process terminates" do
+        let(:message_subject) { :terminator }
         before do
           konst = Class.new(DispatchRider::Handlers::Base) do
             def process(_options)
@@ -109,7 +118,6 @@ describe DispatchRider::Subscriber do
           end
           stub_const("Terminator", konst)
           subject.register_handler(:terminator)
-          message = DispatchRider::Message.new(subject: :terminator, body: {})
           subject.queue_service_registrar.fetch(:simple).push(message)
         end
 
@@ -117,6 +125,7 @@ describe DispatchRider::Subscriber do
       end
 
       context "when process is interupted" do
+        let(:message_subject) { :interupter }
         before do
           konst = Class.new(DispatchRider::Handlers::Base) do
             def process(_options)
@@ -125,7 +134,6 @@ describe DispatchRider::Subscriber do
           end
           stub_const("Interupter", konst)
           subject.register_handler(:interupter)
-          message = DispatchRider::Message.new(subject: :interupter, body: {})
           subject.queue_service_registrar.fetch(:simple).push(message)
         end
 
@@ -133,6 +141,7 @@ describe DispatchRider::Subscriber do
       end
 
       context "when process is interupted twice" do
+        let(:message_subject) { :twice_interupter }
         before do
           allow(subject.demultiplexer).to receive(:stop) # do nothing just ignore the interuption
           allow(subject).to receive(:exit) { throw :got_forcefully_stopped }
@@ -144,7 +153,6 @@ describe DispatchRider::Subscriber do
           end
           stub_const("TwiceInterupter", konst)
           subject.register_handler(:twice_interupter)
-          message = DispatchRider::Message.new(subject: :twice_interupter, body: {})
           subject.queue_service_registrar.fetch(:simple).push(message)
         end
 

data/spec/spec_helper.rb

@@ -1,10 +1,15 @@
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+Bundler.require
+
 require 'aws'
 require 'rake'
 require 'tempfile'
 
+require 'dispatch-rider'
+Dir['./spec/support/**/*.rb'].each { |fn| require(fn) }
+
 RSpec.configure do |config|
   config.raise_errors_for_deprecations!
   config.mock_with :rspec
@@ -14,10 +19,14 @@ RSpec.configure do |config|
   config.mock_with :rspec do |mocks|
     mocks.yield_receiver_to_any_instance_implementation_blocks = false
   end
+
+  config.include IntegrationSupport
+
+  config.before do
+    DispatchRider.config.logger = NullLogger.new
+  end
 end
 
 # Airbrake dummy module
 module Airbrake
 end
-
-require 'dispatch-rider'

data/spec/support/crashing_handler.rb

@@ -0,0 +1,5 @@
+class CrashingHandler < DispatchRider::Handlers::Base
+  def process(_params)
+    raise "I crashed!"
+  end
+end

data/spec/support/integration_support.rb

@@ -0,0 +1,33 @@
+module IntegrationSupport
+  def setup_publisher
+    publisher = DispatchRider::Publisher.new
+    publisher.register_notification_service :file_system
+    publisher.register_destination :dst, :file_system, :dst_channel, path: 'tmp/test_queue'
+    publisher
+  end
+
+  def purge_test_queue
+    Dir['tmp/test_queue/*'].each { |fn| File.unlink(fn) }
+  end
+
+  def setup_subscriber
+    subscriber = DispatchRider.config.subscriber.new
+    subscriber.register_queue(:file_system, path: 'tmp/test_queue')
+    subscriber.register_handler(:sample_handler)
+    subscriber.register_handler(:crashing_handler)
+    subscriber
+  end
+
+  def work_off_jobs(subscriber, fail_on_error: true)
+    subscriber.setup_demultiplexer(:file_system, ->(_message, error) { raise error if fail_on_error })
+
+    class << subscriber.demultiplexer
+      def keep_going?
+        !queue.empty?
+      end
+    end
+
+    subscriber.demultiplexer.start
+  end
+
+end

data/spec/support/null_logger.rb

@@ -0,0 +1,9 @@
+class NullLogger
+  def ignore(*_args)
+  end
+
+  alias info ignore
+  alias error ignore
+  alias debug ignore
+  alias warn ignore
+end

data/spec/support/sample_handler.rb

@@ -0,0 +1,5 @@
+class SampleHandler < DispatchRider::Handlers::Base
+  def process(_params)
+    # do nothing
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dispatch-rider
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Suman Mukherjee
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-20 00:00:00.000000000 Z
+date: 2015-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -165,6 +165,7 @@ files:
 - lib/generators/dispatch_rider/job/templates/publisher/publisher_spec.rb.erb
 - spec/fixtures/handlers/another_test_handler.rb
 - spec/fixtures/handlers/test_handler.rb
+- spec/integration/logging_spec.rb
 - spec/lib/dispatch-rider/airbrake_error_handler_spec.rb
 - spec/lib/dispatch-rider/callbacks/access_spec.rb
 - spec/lib/dispatch-rider/callbacks/storage_spec.rb
@@ -209,6 +210,10 @@ files:
 - spec/lib/dispatch-rider/subscriber_spec.rb
 - spec/lib/dispatch-rider_spec.rb
 - spec/spec_helper.rb
+- spec/support/crashing_handler.rb
+- spec/support/integration_support.rb
+- spec/support/null_logger.rb
+- spec/support/sample_handler.rb
 homepage: https://github.com/payrollhero/dispatch-rider
 licenses:
 - MIT
@@ -237,6 +242,7 @@ summary: Messaging system that is customizable based on which queueing system we
 test_files:
 - spec/fixtures/handlers/another_test_handler.rb
 - spec/fixtures/handlers/test_handler.rb
+- spec/integration/logging_spec.rb
 - spec/lib/dispatch-rider/airbrake_error_handler_spec.rb
 - spec/lib/dispatch-rider/callbacks/access_spec.rb
 - spec/lib/dispatch-rider/callbacks/storage_spec.rb
@@ -281,3 +287,7 @@ test_files:
 - spec/lib/dispatch-rider/subscriber_spec.rb
 - spec/lib/dispatch-rider_spec.rb
 - spec/spec_helper.rb
+- spec/support/crashing_handler.rb
+- spec/support/integration_support.rb
+- spec/support/null_logger.rb
+- spec/support/sample_handler.rb