dispatch-rider 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/lib/dispatch-rider.rb +1 -0
- data/lib/dispatch-rider/configuration.rb +12 -5
- data/lib/dispatch-rider/demultiplexer.rb +3 -27
- data/lib/dispatch-rider/logging.rb +10 -0
- data/lib/dispatch-rider/logging/base_formatter.rb +30 -0
- data/lib/dispatch-rider/logging/json_formatter.rb +75 -0
- data/lib/dispatch-rider/logging/lifecycle_logger.rb +53 -0
- data/lib/dispatch-rider/logging/text_formatter.rb +48 -0
- data/lib/dispatch-rider/queue_services/file_system/queue.rb +13 -2
- data/lib/dispatch-rider/version.rb +1 -1
- data/spec/lib/dispatch-rider/airbrake_error_handler_spec.rb +10 -3
- data/spec/lib/dispatch-rider/callbacks/access_spec.rb +16 -18
- data/spec/lib/dispatch-rider/callbacks/storage_spec.rb +4 -9
- data/spec/lib/dispatch-rider/configuration_spec.rb +3 -3
- data/spec/lib/dispatch-rider/default_error_handler_spec.rb +2 -2
- data/spec/lib/dispatch-rider/demultiplexer_spec.rb +14 -14
- data/spec/lib/dispatch-rider/dispatcher_spec.rb +10 -8
- data/spec/lib/dispatch-rider/handlers/base_spec.rb +27 -22
- data/spec/lib/dispatch-rider/handlers/inheritance_tracking_spec.rb +6 -6
- data/spec/lib/dispatch-rider/logging/json_formatter_spec.rb +72 -0
- data/spec/lib/dispatch-rider/logging/lifecycle_logger_spec.rb +73 -0
- data/spec/lib/dispatch-rider/logging/text_formatter_spec.rb +61 -0
- data/spec/lib/dispatch-rider/message_spec.rb +11 -11
- data/spec/lib/dispatch-rider/notification_services/aws_sns_spec.rb +14 -13
- data/spec/lib/dispatch-rider/notification_services/base_spec.rb +18 -13
- data/spec/lib/dispatch-rider/notification_services/file_system/channel_spec.rb +2 -3
- data/spec/lib/dispatch-rider/notification_services/file_system/notifier_spec.rb +1 -3
- data/spec/lib/dispatch-rider/notification_services/file_system_spec.rb +3 -4
- data/spec/lib/dispatch-rider/publisher/configuration/destination_spec.rb +30 -21
- data/spec/lib/dispatch-rider/publisher/configuration/notification_service_spec.rb +22 -16
- data/spec/lib/dispatch-rider/publisher/configuration_reader_spec.rb +11 -10
- data/spec/lib/dispatch-rider/publisher/configuration_spec.rb +12 -12
- data/spec/lib/dispatch-rider/publisher/configuration_support_spec.rb +11 -11
- data/spec/lib/dispatch-rider/publisher_spec.rb +22 -15
- data/spec/lib/dispatch-rider/queue_services/aws_sqs_spec.rb +44 -36
- data/spec/lib/dispatch-rider/queue_services/base_spec.rb +41 -28
- data/spec/lib/dispatch-rider/queue_services/file_system_spec.rb +15 -14
- data/spec/lib/dispatch-rider/queue_services/received_message_spec.rb +3 -3
- data/spec/lib/dispatch-rider/queue_services/simple_spec.rb +9 -9
- data/spec/lib/dispatch-rider/registrars/base_spec.rb +5 -5
- data/spec/lib/dispatch-rider/registrars/file_system_channel_spec.rb +3 -3
- data/spec/lib/dispatch-rider/registrars/handler_spec.rb +1 -1
- data/spec/lib/dispatch-rider/registrars/notification_service_spec.rb +1 -1
- data/spec/lib/dispatch-rider/registrars/publishing_destination_spec.rb +2 -2
- data/spec/lib/dispatch-rider/registrars/queue_service_spec.rb +1 -1
- data/spec/lib/dispatch-rider/registrars/sns_channel_spec.rb +5 -5
- data/spec/lib/dispatch-rider/runner_spec.rb +1 -1
- data/spec/lib/dispatch-rider/subscriber_spec.rb +45 -29
- data/spec/lib/dispatch-rider_spec.rb +3 -3
- data/spec/spec_helper.rb +3 -1
- metadata +13 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 865cf06c42c11dd3d75306800df557467d80a114
|
4
|
+
data.tar.gz: c86261b609d01cc2db299bf8cd5a74be20de1a95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac475def2ee3a4fa688d4dd693da7da42e29f6826f880e9142532e5026533af96eec09d3d8bd79e735fbb09e7ce63d6a028a3a3c1bd8639e150d76e200ea5243
|
7
|
+
data.tar.gz: 28def6f13e59297cc5269443c94f22121fa87bae86d0fc6f02e436fe285ffb3d6605b245da91a56728b860c579238d7337d2a7beedc21aa60c7ffe536fc4e620
|
data/Gemfile
CHANGED
@@ -14,8 +14,7 @@ gem 'github_changelog_generator'
|
|
14
14
|
gem 'yard'
|
15
15
|
|
16
16
|
# Testing
|
17
|
-
gem "rspec", "~> 3.
|
18
|
-
gem 'rspec-its'
|
17
|
+
gem "rspec", "~> 3.3"
|
19
18
|
|
20
19
|
# CI
|
21
20
|
gem "travis-lint"
|
@@ -23,3 +22,4 @@ gem "travis-lint"
|
|
23
22
|
# Dev/Debugging
|
24
23
|
gem "byebug", platform: :ruby_20, require: !ENV['CI']
|
25
24
|
gem "aws-sdk", "~> 1"
|
25
|
+
gem "pry"
|
data/lib/dispatch-rider.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module DispatchRider
|
2
2
|
class Configuration
|
3
|
-
attr_accessor :handler_path, :error_handler, :queue_info, :queue_kind, :subscriber, :logger, :debug
|
3
|
+
attr_accessor :handler_path, :error_handler, :queue_info, :queue_kind, :subscriber, :logger, :log_formatter, :debug
|
4
4
|
attr_reader :callbacks
|
5
5
|
|
6
6
|
def initialize
|
@@ -10,21 +10,28 @@ module DispatchRider
|
|
10
10
|
@queue_info = { path: "tmp/dispatch-rider-queue" }
|
11
11
|
@callbacks = Callbacks::Storage.new
|
12
12
|
@subscriber = DispatchRider::Subscriber
|
13
|
+
@log_formatter = DispatchRider::Logging::TextFormatter.new
|
13
14
|
@logger = Logger.new(STDERR)
|
14
15
|
@debug = false
|
16
|
+
|
17
|
+
@callbacks.around(:handle_message) do |job, message|
|
18
|
+
Logging::LifecycleLogger.wrap_handling(message) do
|
19
|
+
job.call
|
20
|
+
end
|
21
|
+
end
|
15
22
|
end
|
16
23
|
|
17
24
|
delegate :before, :after, :around, :to => :callbacks
|
18
|
-
|
25
|
+
|
19
26
|
def default_retry_timeout=(val)
|
20
27
|
DispatchRider::Handlers::Base.set_default_retry(val)
|
21
28
|
end
|
22
29
|
|
23
30
|
def handlers
|
24
31
|
@handlers ||= begin
|
25
|
-
|
26
|
-
|
27
|
-
|
32
|
+
load_handler_files
|
33
|
+
DispatchRider::Handlers::Base.subclasses.map { |klass| klass.name.underscore.to_sym }
|
34
|
+
end
|
28
35
|
end
|
29
36
|
|
30
37
|
private
|
@@ -29,9 +29,7 @@ module DispatchRider
|
|
29
29
|
|
30
30
|
def stop(reason: nil)
|
31
31
|
@continue = false
|
32
|
-
if @current_message
|
33
|
-
logger.info "Got stop #{reason ? '(' + reason + ') ' : ' ' }while executing: #{message_info_fragment(@current_message)}"
|
34
|
-
end
|
32
|
+
Logging::LifecycleLogger.log_got_stop reason, @current_message if @current_message
|
35
33
|
end
|
36
34
|
|
37
35
|
private
|
@@ -64,39 +62,17 @@ module DispatchRider
|
|
64
62
|
end
|
65
63
|
end
|
66
64
|
|
67
|
-
def message_info_fragment(message)
|
68
|
-
"(#{message.guid}): #{message.subject} : #{message_info_arguments(message).inspect}"
|
69
|
-
end
|
70
|
-
|
71
|
-
def message_info_arguments(message)
|
72
|
-
message.body.dup.tap { |m|
|
73
|
-
m.delete('guid')
|
74
|
-
}
|
75
|
-
end
|
76
|
-
|
77
65
|
def handle_next_queue_item
|
78
66
|
queue.pop do |message|
|
79
|
-
|
80
|
-
logger.info "Starting execution of: #{message_info_fragment(message)}"
|
81
|
-
dispatch_message(message).tap {
|
82
|
-
logger.info "Succeded execution of: #{message_info_fragment(message)}"
|
83
|
-
}
|
84
|
-
ensure
|
85
|
-
logger.info "Completed execution of: #{message_info_fragment(message)}"
|
86
|
-
end
|
67
|
+
dispatch_message(message)
|
87
68
|
end
|
88
69
|
end
|
89
70
|
|
90
|
-
def exception_info_fragment(message, exception)
|
91
|
-
"(#{message.object_id}): #{message.subject} with #{exception.class}: #{exception.message}"
|
92
|
-
end
|
93
|
-
|
94
71
|
def handle_message_error(message, exception)
|
95
72
|
begin
|
96
|
-
logger.error "Failed execution of: #{exception_info_fragment(message, exception)}"
|
97
73
|
error_handler.call(message, exception)
|
98
74
|
rescue => error_handler_exception # the error handler crashed
|
99
|
-
|
75
|
+
Logging::LifecycleLogger.log_error_handler_fail message, error_handler_exception
|
100
76
|
raise error_handler_exception
|
101
77
|
end
|
102
78
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module DispatchRider
|
2
|
+
module Logging
|
3
|
+
class BaseFormatter
|
4
|
+
def format_error_handler_fail(*)
|
5
|
+
raise NotImplementedError
|
6
|
+
end
|
7
|
+
|
8
|
+
def format_got_stop(*)
|
9
|
+
raise NotImplementedError
|
10
|
+
end
|
11
|
+
|
12
|
+
def format_handling(*)
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def message_info_arguments(message)
|
19
|
+
message.body.dup.tap do |m|
|
20
|
+
m.delete('guid')
|
21
|
+
m.delete('object_id')
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def format_duration(duration)
|
26
|
+
format '%.2f', duration
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
# JSON Log Formatter
|
4
|
+
module DispatchRider
|
5
|
+
module Logging
|
6
|
+
class JsonFormatter < BaseFormatter
|
7
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
8
|
+
# @param [Exception] exception
|
9
|
+
# @return [String] JSON representation of the log item
|
10
|
+
def format_error_handler_fail(message, exception)
|
11
|
+
as_json do
|
12
|
+
{
|
13
|
+
phase: :failed,
|
14
|
+
}.merge exception_info_fragment(message, exception)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
19
|
+
# @param [String] reason
|
20
|
+
# @return [String] JSON representation of the log item
|
21
|
+
def format_got_stop(message, reason)
|
22
|
+
as_json do
|
23
|
+
{
|
24
|
+
phase: :stop,
|
25
|
+
reason: reason,
|
26
|
+
}.merge message_info_fragment(message)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# @param [Symbol] kind of log action. one of: :start, :success, :fail, :complete
|
31
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
32
|
+
# @param [Exception] exception
|
33
|
+
# @param [Float] Job execution duration
|
34
|
+
# @return [String] JSON representation of the log item
|
35
|
+
def format_handling(kind, message, exception: nil, duration: nil)
|
36
|
+
as_json do
|
37
|
+
case kind
|
38
|
+
when :start
|
39
|
+
message_info_fragment(message)
|
40
|
+
when :success
|
41
|
+
message_info_fragment(message)
|
42
|
+
when :fail
|
43
|
+
exception_info_fragment(message, exception)
|
44
|
+
when :complete
|
45
|
+
{ duration: format_duration(duration) }.merge message_info_fragment(message)
|
46
|
+
end.merge(phase: kind)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def as_json
|
53
|
+
JSON.generate yield
|
54
|
+
end
|
55
|
+
|
56
|
+
def message_info_fragment(message)
|
57
|
+
{
|
58
|
+
guid: message.guid.to_s,
|
59
|
+
subject: message.subject,
|
60
|
+
body: message_info_arguments(message),
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
def exception_info_fragment(message, exception)
|
65
|
+
exception_details = {
|
66
|
+
expection: {
|
67
|
+
class: exception.class.to_s,
|
68
|
+
message: exception.message,
|
69
|
+
}
|
70
|
+
}
|
71
|
+
message_info_fragment(message).merge exception_details
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module DispatchRider
|
2
|
+
module Logging
|
3
|
+
class LifecycleLogger
|
4
|
+
class << self
|
5
|
+
def log_error_handler_fail(message, exception)
|
6
|
+
logger.error formatter.format_error_handler_fail(message, exception)
|
7
|
+
end
|
8
|
+
|
9
|
+
def log_got_stop(reason, message)
|
10
|
+
logger.info formatter.format_got_stop(message, reason)
|
11
|
+
end
|
12
|
+
|
13
|
+
def wrap_handling(message)
|
14
|
+
log_start(message)
|
15
|
+
start_time = Time.now
|
16
|
+
yield
|
17
|
+
log_success(message)
|
18
|
+
rescue => exception
|
19
|
+
log_fail(message, exception)
|
20
|
+
raise exception
|
21
|
+
ensure
|
22
|
+
log_complete(message, Time.now - start_time)
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def log_complete(message, duration)
|
28
|
+
logger.info formatter.format_handling :complete, message, duration: duration
|
29
|
+
end
|
30
|
+
|
31
|
+
def log_fail(message, exception)
|
32
|
+
logger.error formatter.format_handling :fail, message, exception: exception
|
33
|
+
end
|
34
|
+
|
35
|
+
def log_success(message)
|
36
|
+
logger.info formatter.format_handling :success, message
|
37
|
+
end
|
38
|
+
|
39
|
+
def log_start(message)
|
40
|
+
logger.info formatter.format_handling :start, message
|
41
|
+
end
|
42
|
+
|
43
|
+
def formatter
|
44
|
+
DispatchRider.config.log_formatter
|
45
|
+
end
|
46
|
+
|
47
|
+
def logger
|
48
|
+
DispatchRider.config.logger
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# Text Log Formatter
|
2
|
+
module DispatchRider
|
3
|
+
module Logging
|
4
|
+
class TextFormatter < BaseFormatter
|
5
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
6
|
+
# @param [Exception] exception
|
7
|
+
# @return [String] log item
|
8
|
+
def format_error_handler_fail(message, exception)
|
9
|
+
"Failed error handling of: #{exception_info_fragment(message, exception)}"
|
10
|
+
end
|
11
|
+
|
12
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
13
|
+
# @param [String] reason
|
14
|
+
# @return [String] log item
|
15
|
+
def format_got_stop(message, reason)
|
16
|
+
"Got stop #{reason ? '(' + reason + ') ' : ' ' }while executing: #{message_info_fragment(message)}"
|
17
|
+
end
|
18
|
+
|
19
|
+
# @param [Symbol] kind of log action. one of: :start, :success, :fail, :complete
|
20
|
+
# @param [DispatchRider::QueueServices::ReceivedMessage] message
|
21
|
+
# @param [Exception] exception
|
22
|
+
# @param [Float] Job execution duration
|
23
|
+
# @return [String] log item
|
24
|
+
def format_handling(kind, message, exception: nil, duration: nil)
|
25
|
+
case kind
|
26
|
+
when :start
|
27
|
+
"Starting execution of: #{message_info_fragment(message)}"
|
28
|
+
when :success
|
29
|
+
"Succeeded execution of: #{message_info_fragment(message)}"
|
30
|
+
when :fail
|
31
|
+
"Failed execution of: #{exception_info_fragment(message, exception)}"
|
32
|
+
when :complete
|
33
|
+
"Completed execution of: #{message_info_fragment(message)} in #{format_duration(duration)} seconds"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def message_info_fragment(message)
|
40
|
+
"(#{message.guid}): #{message.subject} : #{message_info_arguments(message).inspect}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def exception_info_fragment(message, exception)
|
44
|
+
"(#{message.guid}): #{message.subject} with #{exception.class}: #{exception.message}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -15,13 +15,13 @@ module DispatchRider
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def pop
|
18
|
-
file_path =
|
18
|
+
file_path = next_item(10)
|
19
19
|
return nil unless file_path
|
20
20
|
file_path_inflight = file_path.gsub(/\.ready$/, '.inflight')
|
21
21
|
FileUtils.mv(file_path, file_path_inflight)
|
22
22
|
File.new(file_path_inflight)
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def put_back(item)
|
26
26
|
add(item)
|
27
27
|
remove(item)
|
@@ -38,6 +38,17 @@ module DispatchRider
|
|
38
38
|
|
39
39
|
private
|
40
40
|
|
41
|
+
# Long polling next item fetcher
|
42
|
+
# allows to sleep between checks for a new file and not run the main loop as much
|
43
|
+
def next_item(timeout = 10.seconds)
|
44
|
+
Timeout.timeout(timeout) do
|
45
|
+
sleep 1 until file_paths.first
|
46
|
+
file_paths.first
|
47
|
+
end
|
48
|
+
rescue Timeout::Error
|
49
|
+
nil
|
50
|
+
end
|
51
|
+
|
41
52
|
def file_paths
|
42
53
|
Dir["#{@path}/*.ready"]
|
43
54
|
end
|
@@ -3,11 +3,18 @@ require "spec_helper"
|
|
3
3
|
describe DispatchRider::AirbrakeErrorHandler do
|
4
4
|
|
5
5
|
describe ".call" do
|
6
|
-
let(:message){ DispatchRider::Message.new(subject: "TestMessage", body: "foo"
|
7
|
-
let(:exception){ Exception.new("Something went terribly wrong") }
|
6
|
+
let(:message) { DispatchRider::Message.new(subject: "TestMessage", body: "foo") }
|
7
|
+
let(:exception) { Exception.new("Something went terribly wrong") }
|
8
8
|
|
9
9
|
example do
|
10
|
-
|
10
|
+
args = [
|
11
|
+
exception,
|
12
|
+
controller: "DispatchRider",
|
13
|
+
action: "TestMessage",
|
14
|
+
parameters: { subject: "TestMessage", body: "foo" },
|
15
|
+
cgi_data: anything,
|
16
|
+
]
|
17
|
+
expect(Airbrake).to receive(:notify).with(*args)
|
11
18
|
|
12
19
|
described_class.call(message, exception)
|
13
20
|
end
|
@@ -2,7 +2,6 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe DispatchRider::Callbacks::Access do
|
4
4
|
describe "#invoke" do
|
5
|
-
|
6
5
|
let(:callback_a1) { proc { |x| x.call } }
|
7
6
|
let(:callback_a2) { proc { |x| x.call } }
|
8
7
|
let(:callback_a3) { proc { |x| x.call } }
|
@@ -12,7 +11,7 @@ describe DispatchRider::Callbacks::Access do
|
|
12
11
|
let(:callbacks_b) { [callback_b1] }
|
13
12
|
|
14
13
|
let(:storage) { DispatchRider::Callbacks::Storage.new }
|
15
|
-
let(:action) { proc {
|
14
|
+
let(:action) { proc {} }
|
16
15
|
|
17
16
|
subject { described_class.new(storage) }
|
18
17
|
|
@@ -26,37 +25,36 @@ describe DispatchRider::Callbacks::Access do
|
|
26
25
|
end
|
27
26
|
|
28
27
|
example "a bunch of handlers" do
|
29
|
-
callback_a1.
|
30
|
-
callback_a2.
|
31
|
-
callback_a3.
|
32
|
-
callback_b1.
|
28
|
+
expect(callback_a1).to receive(:call).once.and_call_original
|
29
|
+
expect(callback_a2).to receive(:call).once.and_call_original
|
30
|
+
expect(callback_a3).to receive(:call).once.and_call_original
|
31
|
+
expect(callback_b1).not_to receive(:call)
|
33
32
|
|
34
|
-
action.
|
33
|
+
expect(action).to receive(:call).once.and_call_original
|
35
34
|
|
36
35
|
subject.invoke(:event1, &action)
|
37
36
|
end
|
38
37
|
|
39
38
|
example "single handler" do
|
40
|
-
callback_a1.
|
41
|
-
callback_a2.
|
42
|
-
callback_a3.
|
43
|
-
callback_b1.
|
39
|
+
expect(callback_a1).not_to receive(:call)
|
40
|
+
expect(callback_a2).not_to receive(:call)
|
41
|
+
expect(callback_a3).not_to receive(:call)
|
42
|
+
expect(callback_b1).to receive(:call).once.and_call_original
|
44
43
|
|
45
|
-
action.
|
44
|
+
expect(action).to receive(:call).once.and_call_original
|
46
45
|
|
47
46
|
subject.invoke(:event2, &action)
|
48
47
|
end
|
49
48
|
|
50
49
|
example "no handlers" do
|
51
|
-
callback_a1.
|
52
|
-
callback_a2.
|
53
|
-
callback_a3.
|
54
|
-
callback_b1.
|
50
|
+
expect(callback_a1).not_to receive(:call)
|
51
|
+
expect(callback_a2).not_to receive(:call)
|
52
|
+
expect(callback_a3).not_to receive(:call)
|
53
|
+
expect(callback_b1).not_to receive(:call)
|
55
54
|
|
56
|
-
action.
|
55
|
+
expect(action).to receive(:call).once.and_call_original
|
57
56
|
|
58
57
|
subject.invoke(:event3, &action)
|
59
58
|
end
|
60
59
|
end
|
61
|
-
|
62
60
|
end
|