dispatch-rider 1.5.3 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00791dcc6642fcdce813e92abe1e6112af845b47
-  data.tar.gz: 4db374187ef33441a01b17999c6c4dac5a87b535
+  metadata.gz: 865cf06c42c11dd3d75306800df557467d80a114
+  data.tar.gz: c86261b609d01cc2db299bf8cd5a74be20de1a95
 SHA512:
-  metadata.gz: 89bd9117b23f3ae1a26c0e5b2bd0b5dafaf2509413ac5a06ed12f01b0acdbd29f61b9694a0a3805d92acc4b9c7dfa402b425a263f3fb91055d328d1f4778eb23
-  data.tar.gz: 49054767ae3bc31d56144826a5d41d5b1410045b35020f264815e8e096cd7c9e7a6f2c55cffe446bd2743b8b082e0b37683cdd8dbe3acd3e4d527ea9bf929fb6
+  metadata.gz: ac475def2ee3a4fa688d4dd693da7da42e29f6826f880e9142532e5026533af96eec09d3d8bd79e735fbb09e7ce63d6a028a3a3c1bd8639e150d76e200ea5243
+  data.tar.gz: 28def6f13e59297cc5269443c94f22121fa87bae86d0fc6f02e436fe285ffb3d6605b245da91a56728b860c579238d7337d2a7beedc21aa60c7ffe536fc4e620

data/Gemfile

@@ -14,8 +14,7 @@ gem 'github_changelog_generator'
 gem 'yard'
 
 # Testing
-gem "rspec", "~> 3.0"
-gem 'rspec-its'
+gem "rspec", "~> 3.3"
 
 # CI
 gem "travis-lint"
@@ -23,3 +22,4 @@ gem "travis-lint"
 # Dev/Debugging
 gem "byebug", platform: :ruby_20, require: !ENV['CI']
 gem "aws-sdk", "~> 1"
+gem "pry"

data/lib/dispatch-rider.rb

@@ -37,3 +37,4 @@ require "dispatch-rider/demultiplexer"
 require "dispatch-rider/runner"
 require "dispatch-rider/publisher"
 require "dispatch-rider/subscriber"
+require "dispatch-rider/logging"

data/lib/dispatch-rider/configuration.rb

@@ -1,6 +1,6 @@
 module DispatchRider
   class Configuration
-    attr_accessor :handler_path, :error_handler, :queue_info, :queue_kind, :subscriber, :logger, :debug
+    attr_accessor :handler_path, :error_handler, :queue_info, :queue_kind, :subscriber, :logger, :log_formatter, :debug
     attr_reader :callbacks
 
     def initialize
@@ -10,21 +10,28 @@ module DispatchRider
       @queue_info = { path: "tmp/dispatch-rider-queue" }
       @callbacks = Callbacks::Storage.new
       @subscriber = DispatchRider::Subscriber
+      @log_formatter = DispatchRider::Logging::TextFormatter.new
       @logger = Logger.new(STDERR)
       @debug = false
+
+      @callbacks.around(:handle_message) do |job, message|
+        Logging::LifecycleLogger.wrap_handling(message) do
+          job.call
+        end
+      end
     end
 
     delegate :before, :after, :around, :to => :callbacks
-    
+
     def default_retry_timeout=(val)
       DispatchRider::Handlers::Base.set_default_retry(val)
     end
 
     def handlers
       @handlers ||= begin
-                      load_handler_files
-                      DispatchRider::Handlers::Base.subclasses.map{ |klass| klass.name.underscore.to_sym }
-                    end
+        load_handler_files
+        DispatchRider::Handlers::Base.subclasses.map { |klass| klass.name.underscore.to_sym }
+      end
     end
 
     private

data/lib/dispatch-rider/demultiplexer.rb

@@ -29,9 +29,7 @@ module DispatchRider
 
     def stop(reason: nil)
       @continue = false
-      if @current_message
-        logger.info "Got stop #{reason ? '(' + reason + ') ' : ' ' }while executing: #{message_info_fragment(@current_message)}"
-      end
+      Logging::LifecycleLogger.log_got_stop reason, @current_message if @current_message
     end
 
     private
@@ -64,39 +62,17 @@ module DispatchRider
       end
     end
 
-    def message_info_fragment(message)
-      "(#{message.guid}): #{message.subject} : #{message_info_arguments(message).inspect}"
-    end
-
-    def message_info_arguments(message)
-      message.body.dup.tap { |m|
-        m.delete('guid')
-      }
-    end
-
     def handle_next_queue_item
       queue.pop do |message|
-        begin
-          logger.info "Starting execution of: #{message_info_fragment(message)}"
-          dispatch_message(message).tap {
-            logger.info "Succeded execution of: #{message_info_fragment(message)}"
-          }
-        ensure
-          logger.info "Completed execution of: #{message_info_fragment(message)}"
-        end
+        dispatch_message(message)
       end
     end
 
-    def exception_info_fragment(message, exception)
-      "(#{message.object_id}): #{message.subject} with #{exception.class}: #{exception.message}"
-    end
-
     def handle_message_error(message, exception)
       begin
-        logger.error "Failed execution of: #{exception_info_fragment(message, exception)}"
         error_handler.call(message, exception)
       rescue => error_handler_exception # the error handler crashed
-        logger.error "Failed error handling of: #{exception_info_fragment(message, error_handler_exception)}"
+        Logging::LifecycleLogger.log_error_handler_fail message, error_handler_exception
         raise error_handler_exception
       end
     end

data/lib/dispatch-rider/logging.rb

@@ -0,0 +1,10 @@
+module DispatchRider
+  module Logging
+
+  end
+end
+
+require_relative 'logging/lifecycle_logger'
+require_relative 'logging/base_formatter'
+require_relative 'logging/text_formatter'
+require_relative 'logging/json_formatter'

data/lib/dispatch-rider/logging/base_formatter.rb

@@ -0,0 +1,30 @@
+module DispatchRider
+  module Logging
+    class BaseFormatter
+      def format_error_handler_fail(*)
+        raise NotImplementedError
+      end
+
+      def format_got_stop(*)
+        raise NotImplementedError
+      end
+
+      def format_handling(*)
+        raise NotImplementedError
+      end
+
+      private
+
+      def message_info_arguments(message)
+        message.body.dup.tap do |m|
+          m.delete('guid')
+          m.delete('object_id')
+        end
+      end
+
+      def format_duration(duration)
+        format '%.2f', duration
+      end
+    end
+  end
+end

data/lib/dispatch-rider/logging/json_formatter.rb

@@ -0,0 +1,75 @@
+require 'json'
+
+# JSON Log Formatter
+module DispatchRider
+  module Logging
+    class JsonFormatter < BaseFormatter
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [Exception] exception
+      # @return [String] JSON representation of the log item
+      def format_error_handler_fail(message, exception)
+        as_json do
+          {
+            phase: :failed,
+          }.merge exception_info_fragment(message, exception)
+        end
+      end
+
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [String] reason
+      # @return [String] JSON representation of the log item
+      def format_got_stop(message, reason)
+        as_json do
+          {
+            phase: :stop,
+            reason: reason,
+          }.merge message_info_fragment(message)
+        end
+      end
+
+      # @param [Symbol] kind of log action. one of: :start, :success, :fail, :complete
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [Exception] exception
+      # @param [Float] Job execution duration
+      # @return [String] JSON representation of the log item
+      def format_handling(kind, message, exception: nil, duration: nil)
+        as_json do
+          case kind
+          when :start
+            message_info_fragment(message)
+          when :success
+            message_info_fragment(message)
+          when :fail
+            exception_info_fragment(message, exception)
+          when :complete
+            { duration: format_duration(duration) }.merge message_info_fragment(message)
+          end.merge(phase: kind)
+        end
+      end
+
+      private
+
+      def as_json
+        JSON.generate yield
+      end
+
+      def message_info_fragment(message)
+        {
+          guid: message.guid.to_s,
+          subject: message.subject,
+          body: message_info_arguments(message),
+        }
+      end
+
+      def exception_info_fragment(message, exception)
+        exception_details = {
+          expection: {
+            class: exception.class.to_s,
+            message: exception.message,
+          }
+        }
+        message_info_fragment(message).merge exception_details
+      end
+    end
+  end
+end

data/lib/dispatch-rider/logging/lifecycle_logger.rb

@@ -0,0 +1,53 @@
+module DispatchRider
+  module Logging
+    class LifecycleLogger
+      class << self
+        def log_error_handler_fail(message, exception)
+          logger.error formatter.format_error_handler_fail(message, exception)
+        end
+
+        def log_got_stop(reason, message)
+          logger.info formatter.format_got_stop(message, reason)
+        end
+
+        def wrap_handling(message)
+          log_start(message)
+          start_time = Time.now
+          yield
+          log_success(message)
+        rescue => exception
+          log_fail(message, exception)
+          raise exception
+        ensure
+          log_complete(message, Time.now - start_time)
+        end
+
+        private
+
+        def log_complete(message, duration)
+          logger.info formatter.format_handling :complete, message, duration: duration
+        end
+
+        def log_fail(message, exception)
+          logger.error formatter.format_handling :fail, message, exception: exception
+        end
+
+        def log_success(message)
+          logger.info formatter.format_handling :success, message
+        end
+
+        def log_start(message)
+          logger.info formatter.format_handling :start, message
+        end
+
+        def formatter
+          DispatchRider.config.log_formatter
+        end
+
+        def logger
+          DispatchRider.config.logger
+        end
+      end
+    end
+  end
+end

data/lib/dispatch-rider/logging/text_formatter.rb

@@ -0,0 +1,48 @@
+# Text Log Formatter
+module DispatchRider
+  module Logging
+    class TextFormatter < BaseFormatter
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [Exception] exception
+      # @return [String] log item
+      def format_error_handler_fail(message, exception)
+        "Failed error handling of: #{exception_info_fragment(message, exception)}"
+      end
+
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [String] reason
+      # @return [String] log item
+      def format_got_stop(message, reason)
+        "Got stop #{reason ? '(' + reason + ') ' : ' ' }while executing: #{message_info_fragment(message)}"
+      end
+
+      # @param [Symbol] kind of log action. one of: :start, :success, :fail, :complete
+      # @param [DispatchRider::QueueServices::ReceivedMessage] message
+      # @param [Exception] exception
+      # @param [Float] Job execution duration
+      # @return [String] log item
+      def format_handling(kind, message, exception: nil, duration: nil)
+        case kind
+        when :start
+          "Starting execution of: #{message_info_fragment(message)}"
+        when :success
+          "Succeeded execution of: #{message_info_fragment(message)}"
+        when :fail
+          "Failed execution of: #{exception_info_fragment(message, exception)}"
+        when :complete
+          "Completed execution of: #{message_info_fragment(message)} in #{format_duration(duration)} seconds"
+        end
+      end
+
+      private
+
+      def message_info_fragment(message)
+        "(#{message.guid}): #{message.subject} : #{message_info_arguments(message).inspect}"
+      end
+
+      def exception_info_fragment(message, exception)
+        "(#{message.guid}): #{message.subject} with #{exception.class}: #{exception.message}"
+      end
+    end
+  end
+end

data/lib/dispatch-rider/queue_services/file_system/queue.rb

@@ -15,13 +15,13 @@ module DispatchRider
         end
 
         def pop
-          file_path = file_paths.first
+          file_path = next_item(10)
           return nil unless file_path
           file_path_inflight = file_path.gsub(/\.ready$/, '.inflight')
           FileUtils.mv(file_path, file_path_inflight)
           File.new(file_path_inflight)
         end
-        
+
         def put_back(item)
           add(item)
           remove(item)
@@ -38,6 +38,17 @@ module DispatchRider
 
         private
 
+        # Long polling next item fetcher
+        # allows to sleep between checks for a new file and not run the main loop as much
+        def next_item(timeout = 10.seconds)
+          Timeout.timeout(timeout) do
+            sleep 1 until file_paths.first
+            file_paths.first
+          end
+        rescue Timeout::Error
+          nil
+        end
+
         def file_paths
           Dir["#{@path}/*.ready"]
         end

data/lib/dispatch-rider/version.rb

@@ -1,4 +1,4 @@
 # This file specifies the current version of the gem.
 module DispatchRider
-  VERSION = "1.5.3"
+  VERSION = "1.6.0"
 end

data/spec/lib/dispatch-rider/airbrake_error_handler_spec.rb

@@ -3,11 +3,18 @@ require "spec_helper"
 describe DispatchRider::AirbrakeErrorHandler do
 
   describe ".call" do
-    let(:message){ DispatchRider::Message.new(subject: "TestMessage", body: "foo" )}
-    let(:exception){ Exception.new("Something went terribly wrong") }
+    let(:message) { DispatchRider::Message.new(subject: "TestMessage", body: "foo") }
+    let(:exception) { Exception.new("Something went terribly wrong") }
 
     example do
-      Airbrake.should_receive(:notify).with(exception, controller: "DispatchRider", action: "TestMessage", parameters: { subject: "TestMessage", body: "foo" }, cgi_data: anything)
+      args = [
+        exception,
+        controller: "DispatchRider",
+        action: "TestMessage",
+        parameters: { subject: "TestMessage", body: "foo" },
+        cgi_data: anything,
+      ]
+      expect(Airbrake).to receive(:notify).with(*args)
 
       described_class.call(message, exception)
     end

data/spec/lib/dispatch-rider/callbacks/access_spec.rb

@@ -2,7 +2,6 @@ require 'spec_helper'
 
 describe DispatchRider::Callbacks::Access do
   describe "#invoke" do
-
     let(:callback_a1) { proc { |x| x.call } }
     let(:callback_a2) { proc { |x| x.call } }
     let(:callback_a3) { proc { |x| x.call } }
@@ -12,7 +11,7 @@ describe DispatchRider::Callbacks::Access do
     let(:callbacks_b) { [callback_b1] }
 
     let(:storage) { DispatchRider::Callbacks::Storage.new }
-    let(:action) { proc { } }
+    let(:action) { proc {} }
 
     subject { described_class.new(storage) }
 
@@ -26,37 +25,36 @@ describe DispatchRider::Callbacks::Access do
     end
 
     example "a bunch of handlers" do
-      callback_a1.should_receive(:call).once.and_call_original
-      callback_a2.should_receive(:call).once.and_call_original
-      callback_a3.should_receive(:call).once.and_call_original
-      callback_b1.should_not_receive(:call)
+      expect(callback_a1).to receive(:call).once.and_call_original
+      expect(callback_a2).to receive(:call).once.and_call_original
+      expect(callback_a3).to receive(:call).once.and_call_original
+      expect(callback_b1).not_to receive(:call)
 
-      action.should_receive(:call).once.and_call_original
+      expect(action).to receive(:call).once.and_call_original
 
       subject.invoke(:event1, &action)
     end
 
     example "single handler" do
-      callback_a1.should_not_receive(:call)
-      callback_a2.should_not_receive(:call)
-      callback_a3.should_not_receive(:call)
-      callback_b1.should_receive(:call).once.and_call_original
+      expect(callback_a1).not_to receive(:call)
+      expect(callback_a2).not_to receive(:call)
+      expect(callback_a3).not_to receive(:call)
+      expect(callback_b1).to receive(:call).once.and_call_original
 
-      action.should_receive(:call).once.and_call_original
+      expect(action).to receive(:call).once.and_call_original
 
       subject.invoke(:event2, &action)
     end
 
     example "no handlers" do
-      callback_a1.should_not_receive(:call)
-      callback_a2.should_not_receive(:call)
-      callback_a3.should_not_receive(:call)
-      callback_b1.should_not_receive(:call)
+      expect(callback_a1).not_to receive(:call)
+      expect(callback_a2).not_to receive(:call)
+      expect(callback_a3).not_to receive(:call)
+      expect(callback_b1).not_to receive(:call)
 
-      action.should_receive(:call).once.and_call_original
+      expect(action).to receive(:call).once.and_call_original
 
       subject.invoke(:event3, &action)
     end
   end
-
 end