pheme 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +143 -0
- data/lib/pheme/configuration.rb +2 -1
- data/lib/pheme/logger.rb +4 -0
- data/lib/pheme/message_type/aws_event.rb +1 -1
- data/lib/pheme/queue_poller.rb +86 -27
- data/lib/pheme/version.rb +1 -1
- data/pheme.gemspec +10 -7
- data/spec/message_handler_spec.rb +1 -1
- data/spec/message_type/aws_event_spec.rb +13 -18
- data/spec/message_type/sns_message_spec.rb +2 -2
- data/spec/queue_poller_spec.rb +121 -79
- data/spec/spec_helper.rb +5 -0
- data/spec/support/example_message_handler.rb +2 -2
- data/spec/topic_publisher_spec.rb +6 -6
- metadata +61 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d38042f356beb16dc19336d1f8a7103264cd7fc9
|
4
|
+
data.tar.gz: aea9f789ac089d39c8d3645c31202c6e8d368894
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0977af6e8deee4b1a19a69b86f2709055926bd18d925024b41979b292147066019a6a919652fb0063351c1b42278565eecea5986ee9f7f79d98e6f1fd42a292c'
|
7
|
+
data.tar.gz: 0c0e8351a7cda60992dcf6839b58edc0d87de7f7c5f056187773647a3e2a350423949d2bdbaf3923e4262a18e6bde0ba533297ed221f10c4b0a0cc5d9ecc2dde
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
AllCops:
|
2
|
+
DisplayCopNames: true
|
3
|
+
|
4
|
+
Exclude:
|
5
|
+
- 'vendor/**/*'
|
6
|
+
- 'log/**/*'
|
7
|
+
- 'public/**/*'
|
8
|
+
- 'data/**/*'
|
9
|
+
- 'tmp/**/*'
|
10
|
+
- 'bin/**/*'
|
11
|
+
- 'db/**/*'
|
12
|
+
- 'lib/tasks/circle_0.rake' # Ignore circleci code:
|
13
|
+
- 'node_modules/**/*'
|
14
|
+
TargetRubyVersion: 2.4
|
15
|
+
|
16
|
+
Lint/AmbiguousOperator:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Naming/FileName:
|
20
|
+
Enabled: false
|
21
|
+
|
22
|
+
Lint/IneffectiveAccessModifier:
|
23
|
+
Enabled: false
|
24
|
+
|
25
|
+
Lint/UselessAccessModifier:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
Layout/SpaceBeforeBlockBraces:
|
29
|
+
Enabled: false
|
30
|
+
|
31
|
+
FrozenStringLiteralComment:
|
32
|
+
Enabled: false
|
33
|
+
|
34
|
+
Metrics/AbcSize:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
Metrics/BlockLength:
|
38
|
+
Enabled: false
|
39
|
+
|
40
|
+
Metrics/ClassLength:
|
41
|
+
Enabled: false
|
42
|
+
|
43
|
+
Metrics/MethodLength:
|
44
|
+
Enabled: false
|
45
|
+
|
46
|
+
Metrics/LineLength:
|
47
|
+
Enabled: false
|
48
|
+
Max: 120
|
49
|
+
|
50
|
+
Metrics/CyclomaticComplexity:
|
51
|
+
Enabled: false
|
52
|
+
|
53
|
+
Metrics/PerceivedComplexity:
|
54
|
+
Enabled: false
|
55
|
+
|
56
|
+
Metrics/ParameterLists:
|
57
|
+
Enabled: false
|
58
|
+
|
59
|
+
Style/For:
|
60
|
+
EnforcedStyle: each
|
61
|
+
|
62
|
+
Style/HashSyntax:
|
63
|
+
EnforcedStyle: ruby19
|
64
|
+
|
65
|
+
Layout/ExtraSpacing:
|
66
|
+
Enabled: false
|
67
|
+
|
68
|
+
Layout/SpaceAroundEqualsInParameterDefault:
|
69
|
+
Enabled: false
|
70
|
+
|
71
|
+
Style/Documentation:
|
72
|
+
Enabled: false
|
73
|
+
|
74
|
+
Layout/TrailingBlankLines:
|
75
|
+
EnforcedStyle: final_newline
|
76
|
+
|
77
|
+
Style/VariableName:
|
78
|
+
EnforcedStyle: snake_case
|
79
|
+
|
80
|
+
Style/MethodDefParentheses:
|
81
|
+
EnforcedStyle: require_parentheses
|
82
|
+
|
83
|
+
Style/Semicolon:
|
84
|
+
AllowAsExpressionSeparator: false
|
85
|
+
|
86
|
+
Style/TrailingCommaInLiteral:
|
87
|
+
EnforcedStyleForMultiline: comma
|
88
|
+
|
89
|
+
Style/Alias:
|
90
|
+
EnforcedStyle: prefer_alias_method
|
91
|
+
|
92
|
+
Style/AndOr:
|
93
|
+
EnforcedStyle: conditionals
|
94
|
+
|
95
|
+
Style/StringLiterals:
|
96
|
+
Enabled: false
|
97
|
+
|
98
|
+
Style/CollectionMethods:
|
99
|
+
PreferredMethods:
|
100
|
+
collect: 'map'
|
101
|
+
collect!: 'map!'
|
102
|
+
inject: 'reduce'
|
103
|
+
detect: 'find'
|
104
|
+
find_all: 'select'
|
105
|
+
|
106
|
+
Layout/SpaceBeforeFirstArg:
|
107
|
+
Enabled: false
|
108
|
+
|
109
|
+
Style/ClassVars:
|
110
|
+
Enabled: false
|
111
|
+
|
112
|
+
Layout/MultilineMethodCallIndentation:
|
113
|
+
EnforcedStyle: indented
|
114
|
+
|
115
|
+
Layout/DotPosition:
|
116
|
+
EnforcedStyle: trailing
|
117
|
+
|
118
|
+
Style/NumericPredicate:
|
119
|
+
EnforcedStyle: comparison
|
120
|
+
|
121
|
+
Layout/IndentHash:
|
122
|
+
EnforcedStyle: consistent
|
123
|
+
|
124
|
+
Style/TrailingCommaInArguments:
|
125
|
+
EnforcedStyleForMultiline: comma
|
126
|
+
|
127
|
+
Style/BracesAroundHashParameters:
|
128
|
+
Enabled: false
|
129
|
+
|
130
|
+
Style/Lambda:
|
131
|
+
Enabled: false
|
132
|
+
|
133
|
+
Style/SingleLineBlockParams:
|
134
|
+
Enabled: false
|
135
|
+
|
136
|
+
Style/GuardClause:
|
137
|
+
Enabled: false
|
138
|
+
|
139
|
+
Style/DoubleNegation:
|
140
|
+
Enabled: false
|
141
|
+
|
142
|
+
Style/ClassAndModuleChildren:
|
143
|
+
Enabled: false
|
data/lib/pheme/configuration.rb
CHANGED
@@ -16,11 +16,12 @@ module Pheme
|
|
16
16
|
end
|
17
17
|
|
18
18
|
class Configuration
|
19
|
-
ATTRIBUTES = [
|
19
|
+
ATTRIBUTES = %i[sns_client sqs_client logger rollbar].freeze
|
20
20
|
attr_accessor *ATTRIBUTES
|
21
21
|
|
22
22
|
def initialize
|
23
23
|
@logger ||= Logger.new(STDOUT)
|
24
|
+
@logger = ActiveSupport::TaggedLogging.new(@logger) unless @logger.respond_to?(:tagged)
|
24
25
|
end
|
25
26
|
|
26
27
|
def validate!
|
data/lib/pheme/logger.rb
CHANGED
@@ -19,7 +19,7 @@ module Pheme
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def parse_aws_event(message_contents)
|
22
|
-
RecursiveOpenStruct.new({wrapper: message_contents}, recurse_over_arrays: true).wrapper
|
22
|
+
RecursiveOpenStruct.new({ wrapper: message_contents }, recurse_over_arrays: true).wrapper
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
data/lib/pheme/queue_poller.rb
CHANGED
@@ -7,10 +7,12 @@ module Pheme
|
|
7
7
|
@queue_url = queue_url
|
8
8
|
@queue_poller = Aws::SQS::QueuePoller.new(queue_url)
|
9
9
|
@connection_pool_block = connection_pool_block
|
10
|
+
@messages_processed = 0
|
11
|
+
@messages_received = 0
|
10
12
|
@format = format
|
11
13
|
@max_messages = max_messages
|
12
14
|
@poller_configuration = {
|
13
|
-
wait_time_seconds: 10, # amount of time a long polling receive call can wait for a
|
15
|
+
wait_time_seconds: 10, # amount of time a long polling receive call can wait for a message before receiving a empty response (which will trigger another polling request)
|
14
16
|
idle_timeout: 20, # disconnects poller after 20 seconds of idle time
|
15
17
|
skip_delete: true, # manually delete messages
|
16
18
|
}.merge(poller_configuration || {})
|
@@ -23,38 +25,49 @@ module Pheme
|
|
23
25
|
end
|
24
26
|
|
25
27
|
def poll
|
26
|
-
|
28
|
+
time_start = log_polling_start
|
27
29
|
with_optional_connection_pool_block do
|
28
|
-
queue_poller.poll(poller_configuration) do |
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
30
|
+
queue_poller.poll(poller_configuration) do |queue_message|
|
31
|
+
@messages_received += 1
|
32
|
+
Pheme.logger.tagged(queue_message.message_id) do
|
33
|
+
begin
|
34
|
+
content = parse_body(queue_message)
|
35
|
+
handle(content)
|
36
|
+
queue_poller.delete_message(queue_message)
|
37
|
+
log_delete(queue_message)
|
38
|
+
@messages_processed += 1
|
39
|
+
rescue SignalException
|
40
|
+
throw :stop_polling
|
41
|
+
rescue StandardError => e
|
42
|
+
Pheme.logger.error(e)
|
43
|
+
Pheme.rollbar(e, "#{self.class} failed to process message", content)
|
44
|
+
end
|
39
45
|
end
|
40
46
|
end
|
41
47
|
end
|
42
|
-
|
48
|
+
log_polling_end(time_start)
|
43
49
|
end
|
44
50
|
|
45
|
-
|
46
|
-
|
47
|
-
|
51
|
+
# returns queue_message.body as hash,
|
52
|
+
# stores and parses get_content to body[:content]
|
53
|
+
def parse_body(queue_message)
|
54
|
+
message_body = JSON.parse(queue_message.body)
|
55
|
+
raw_content = get_content(message_body)
|
56
|
+
|
48
57
|
case format
|
49
58
|
when :csv
|
50
|
-
parse_csv(
|
59
|
+
parsed_content = parse_csv(raw_content)
|
51
60
|
when :json
|
52
|
-
parse_json(
|
61
|
+
parsed_content = parse_json(raw_content)
|
53
62
|
else
|
54
63
|
method_name = "parse_#{format}".to_sym
|
55
|
-
raise ArgumentError
|
56
|
-
|
64
|
+
raise ArgumentError, "Unknown format #{format}" unless respond_to?(method_name)
|
65
|
+
parsed_content = __send__(method_name, raw_content)
|
57
66
|
end
|
67
|
+
|
68
|
+
body = format == :csv ? raw_content : parsed_content
|
69
|
+
log_message_received(queue_message, body)
|
70
|
+
parsed_content
|
58
71
|
end
|
59
72
|
|
60
73
|
def get_content(body)
|
@@ -68,21 +81,67 @@ module Pheme
|
|
68
81
|
|
69
82
|
def parse_json(message_contents)
|
70
83
|
parsed_body = JSON.parse(message_contents)
|
71
|
-
RecursiveOpenStruct.new({wrapper: parsed_body}, recurse_over_arrays: true).wrapper
|
84
|
+
RecursiveOpenStruct.new({ wrapper: parsed_body }, recurse_over_arrays: true).wrapper
|
72
85
|
end
|
73
86
|
|
74
|
-
def handle(
|
87
|
+
def handle(_message)
|
75
88
|
raise NotImplementedError
|
76
89
|
end
|
77
90
|
|
78
|
-
|
91
|
+
private
|
79
92
|
|
80
|
-
def with_optional_connection_pool_block
|
93
|
+
def with_optional_connection_pool_block
|
81
94
|
if connection_pool_block
|
82
|
-
ActiveRecord::Base.connection_pool.with_connection {
|
95
|
+
ActiveRecord::Base.connection_pool.with_connection { yield }
|
83
96
|
else
|
84
|
-
|
97
|
+
yield
|
85
98
|
end
|
86
99
|
end
|
100
|
+
|
101
|
+
def log_polling_start
|
102
|
+
time_start = Time.now
|
103
|
+
Pheme.logger.info({
|
104
|
+
message: "Start long-polling #{queue_url}",
|
105
|
+
type: self.class.name,
|
106
|
+
queue_url: queue_url,
|
107
|
+
format: format,
|
108
|
+
max_messages: max_messages,
|
109
|
+
connection_pool_block: connection_pool_block,
|
110
|
+
poller_configuration: poller_configuration,
|
111
|
+
}.to_json)
|
112
|
+
time_start
|
113
|
+
end
|
114
|
+
|
115
|
+
def log_polling_end(time_start)
|
116
|
+
time_end = Time.now
|
117
|
+
elapsed = time_end - time_start
|
118
|
+
Pheme.logger.info({
|
119
|
+
message: "Finished long-polling #{queue_url}, duration: #{elapsed.round(2)} seconds.",
|
120
|
+
queue_url: queue_url,
|
121
|
+
format: format,
|
122
|
+
messages_received: @messages_received,
|
123
|
+
messages_processed: @messages_processed,
|
124
|
+
duration: elapsed.round(2),
|
125
|
+
start_time: time_start.utc.iso8601,
|
126
|
+
end_time: time_end.utc.iso8601,
|
127
|
+
}.to_json)
|
128
|
+
end
|
129
|
+
|
130
|
+
def log_delete(queue_message)
|
131
|
+
Pheme.logger.info({
|
132
|
+
message: "Deleted message #{queue_message.message_id}",
|
133
|
+
message_id: queue_message.message_id,
|
134
|
+
queue_url: queue_url,
|
135
|
+
}.to_json)
|
136
|
+
end
|
137
|
+
|
138
|
+
def log_message_received(queue_message, body)
|
139
|
+
Pheme.logger.info({
|
140
|
+
message: "Received message #{queue_message.message_id}",
|
141
|
+
message_id: queue_message.message_id,
|
142
|
+
queue_url: queue_url,
|
143
|
+
body: body,
|
144
|
+
}.to_json)
|
145
|
+
end
|
87
146
|
end
|
88
147
|
end
|
data/lib/pheme/version.rb
CHANGED
data/pheme.gemspec
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
@@ -9,22 +9,25 @@ Gem::Specification.new do |gem|
|
|
9
9
|
gem.version = Pheme::VERSION
|
10
10
|
gem.authors = ["Peter Graham"]
|
11
11
|
gem.email = ["peter@wealthsimple.com"]
|
12
|
-
gem.description =
|
13
|
-
gem.summary =
|
12
|
+
gem.description = 'Ruby AWS SNS publisher + SQS poller & message handler'
|
13
|
+
gem.summary = 'Ruby SNS publisher + SQS poller & message handler'
|
14
14
|
gem.homepage = "https://github.com/wealthsimple/pheme"
|
15
15
|
|
16
|
-
gem.files = `git ls-files`.split(
|
16
|
+
gem.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
|
17
17
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
18
18
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
19
19
|
gem.require_paths = ["lib"]
|
20
20
|
gem.licenses = ["MIT"]
|
21
21
|
gem.required_ruby_version = ">= 2.1.0"
|
22
22
|
|
23
|
-
gem.add_dependency "aws-sdk", ">= 2", "< 4"
|
24
23
|
gem.add_dependency "activesupport", ">= 4"
|
24
|
+
gem.add_dependency "aws-sdk", ">= 2", "< 4"
|
25
25
|
gem.add_dependency "recursive-open-struct", "~> 1"
|
26
26
|
gem.add_dependency "smarter_csv", "~> 1"
|
27
27
|
|
28
|
-
gem.add_development_dependency "rspec"
|
29
|
-
gem.add_development_dependency "
|
28
|
+
gem.add_development_dependency "rspec"
|
29
|
+
gem.add_development_dependency "rspec-collection_matchers"
|
30
|
+
gem.add_development_dependency "rspec-its"
|
31
|
+
gem.add_development_dependency "rspec_junit_formatter"
|
32
|
+
gem.add_development_dependency "rubocop"
|
30
33
|
end
|
@@ -1,26 +1,21 @@
|
|
1
1
|
describe Pheme::MessageType::AwsEvent do
|
2
|
-
|
3
|
-
|
4
|
-
let(:
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
2
|
+
let(:poller) { ExampleAwsEventQueuePoller.new }
|
3
|
+
let(:message_id) { SecureRandom.uuid }
|
4
|
+
let(:queue_url) { 'http://queue_url' }
|
5
|
+
let(:queue_message) do
|
6
|
+
OpenStruct.new(
|
7
|
+
message_id: message_id,
|
8
|
+
body: { 'Records' => records }.to_json,
|
9
|
+
queue_url: queue_url,
|
10
|
+
)
|
10
11
|
end
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
allow(Aws::SQS::QueuePoller).to receive(:new) { poller }
|
15
|
-
end
|
13
|
+
describe "#parse_body" do
|
14
|
+
subject { poller.parse_body(queue_message) }
|
16
15
|
|
17
|
-
describe "#parse_message" do
|
18
16
|
context "with JSON message" do
|
19
|
-
let!(:
|
20
|
-
|
21
|
-
it 'should parse the message correctly' do
|
22
|
-
expect(subject.parse_message(message).first.eventVersion).to eq("2.0")
|
23
|
-
end
|
17
|
+
let!(:records) { [{ 'eventVersion' => '2.0' }] }
|
18
|
+
its('first.eventVersion') { is_expected.to eq('2.0') }
|
24
19
|
end
|
25
20
|
end
|
26
21
|
end
|
@@ -14,7 +14,7 @@ describe Pheme::MessageType::SnsMessage do
|
|
14
14
|
allow(poller).to receive(:before_request)
|
15
15
|
poller
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
before(:each) do
|
19
19
|
use_default_configuration!
|
20
20
|
allow(Aws::SQS::QueuePoller).to receive(:new) { poller }
|
@@ -25,7 +25,7 @@ describe Pheme::MessageType::SnsMessage do
|
|
25
25
|
let!(:message) { OpenStruct.new({ body: '{"Message":"{\"test\":\"test\"}"}' }) }
|
26
26
|
|
27
27
|
it 'should parse the message correctly' do
|
28
|
-
expect(subject.
|
28
|
+
expect(subject.parse_body(message).test).to eq("test")
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|
data/spec/queue_poller_spec.rb
CHANGED
@@ -1,16 +1,5 @@
|
|
1
1
|
describe Pheme::QueuePoller do
|
2
2
|
let(:queue_url) { "https://sqs.us-east-1.amazonaws.com/whatever" }
|
3
|
-
let(:poller) do
|
4
|
-
poller = double
|
5
|
-
allow(poller).to receive(:poll).with(kind_of(Hash))
|
6
|
-
allow(poller).to receive(:parse_message)
|
7
|
-
allow(poller).to receive(:before_request)
|
8
|
-
poller
|
9
|
-
end
|
10
|
-
before(:each) do
|
11
|
-
use_default_configuration!
|
12
|
-
allow(Aws::SQS::QueuePoller).to receive(:new) { poller }
|
13
|
-
end
|
14
3
|
|
15
4
|
describe ".new" do
|
16
5
|
context "when initialized with valid params" do
|
@@ -32,54 +21,63 @@ describe Pheme::QueuePoller do
|
|
32
21
|
end
|
33
22
|
end
|
34
23
|
|
35
|
-
|
36
|
-
|
37
|
-
|
24
|
+
let(:poller) { ExampleQueuePoller.new(queue_url: queue_url, format: format) }
|
25
|
+
let(:message_id) { SecureRandom.uuid }
|
26
|
+
let(:message) { nil }
|
27
|
+
let!(:queue_message) do
|
28
|
+
OpenStruct.new(
|
29
|
+
body: { Message: message }.to_json,
|
30
|
+
message_id: message_id,
|
31
|
+
)
|
32
|
+
end
|
38
33
|
|
39
|
-
|
40
|
-
|
41
|
-
})}
|
34
|
+
describe "#parse_body" do
|
35
|
+
subject { poller.parse_body(queue_message) }
|
42
36
|
|
43
|
-
|
44
|
-
|
45
|
-
|
37
|
+
context "message is JSON string" do
|
38
|
+
let(:format) { :json }
|
39
|
+
let!(:message) { { test: 'test' }.to_json }
|
40
|
+
its([:test]) { is_expected.to eq('test') }
|
46
41
|
end
|
47
42
|
|
48
|
-
context "
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
43
|
+
context "message is CSV string" do
|
44
|
+
let(:format) { :csv }
|
45
|
+
let(:expected_message) do
|
46
|
+
[
|
47
|
+
{ test1: 'value1', test2: 'value2' },
|
48
|
+
{ test1: 'value3', test2: 'value4' },
|
49
|
+
]
|
50
|
+
end
|
51
|
+
let(:message) do
|
52
|
+
[
|
53
|
+
%w[test1 test2].join(','),
|
54
|
+
%w[value1 value2].join(','),
|
55
|
+
%w[value3 value4].join(','),
|
56
|
+
].join("\n")
|
58
57
|
end
|
58
|
+
|
59
|
+
it { is_expected.to have(2).items }
|
60
|
+
it { is_expected.to eq(RecursiveOpenStruct.new({ wrapper: expected_message }, recurse_over_arrays: true).wrapper) }
|
59
61
|
end
|
60
62
|
|
61
63
|
context "with unknown message format" do
|
62
|
-
|
63
|
-
|
64
|
-
let!(:message) { OpenStruct.new({
|
65
|
-
body:'{"Message":"test,test2\nvalue,value2\nvalue3,value4"}'
|
66
|
-
})}
|
64
|
+
let(:format) { :invalid_format }
|
65
|
+
let(:message) { 'unkonwn' }
|
67
66
|
|
68
|
-
it
|
69
|
-
expect{ subject
|
67
|
+
it "should raise error" do
|
68
|
+
expect{ subject }.to raise_error(ArgumentError)
|
70
69
|
end
|
71
70
|
end
|
72
71
|
|
73
72
|
context "with array JSON message" do
|
74
|
-
|
75
|
-
let
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
expect(subject.first).to
|
82
|
-
expect(subject.first.first).to be_a RecursiveOpenStruct
|
73
|
+
let(:format) { :json }
|
74
|
+
let(:message) { [[{ test: 'test' }]].to_json }
|
75
|
+
|
76
|
+
it { is_expected.to be_a(Array) }
|
77
|
+
its(:first) { is_expected.to be_a(Array) }
|
78
|
+
its('first.first') { is_expected.to be_a(RecursiveOpenStruct) }
|
79
|
+
it "parses the nested object" do
|
80
|
+
expect(subject.first.first.test).to eq('test')
|
83
81
|
end
|
84
82
|
end
|
85
83
|
end
|
@@ -88,22 +86,30 @@ describe Pheme::QueuePoller do
|
|
88
86
|
before(:each) do
|
89
87
|
module ActiveRecord
|
90
88
|
class Base
|
91
|
-
def self.connection_pool
|
92
|
-
end
|
89
|
+
def self.connection_pool; end
|
93
90
|
end
|
94
91
|
end
|
95
92
|
end
|
96
93
|
|
97
94
|
context "with connection pool block" do
|
98
95
|
let(:mock_connection_pool) { double }
|
96
|
+
subject { ExampleQueuePoller.new(queue_url: queue_url, connection_pool_block: true) }
|
97
|
+
let(:message) { { status: 'complete' } }
|
98
|
+
let(:notification) { { 'MessageId' => SecureRandom.uuid, 'Message' => message.to_json, 'Type' => 'Notification' } }
|
99
|
+
let!(:queue_message) do
|
100
|
+
OpenStruct.new(
|
101
|
+
body: notification.to_json,
|
102
|
+
message_id: message_id,
|
103
|
+
)
|
104
|
+
end
|
99
105
|
|
100
106
|
before(:each) do
|
101
107
|
allow(ActiveRecord::Base).to receive(:connection_pool) { mock_connection_pool }
|
102
108
|
allow(mock_connection_pool).to receive(:with_connection).and_yield
|
109
|
+
allow(subject.queue_poller).to receive(:poll).and_yield(queue_message)
|
110
|
+
allow(subject.queue_poller).to receive(:delete_message).with(queue_message)
|
103
111
|
end
|
104
112
|
|
105
|
-
subject { ExampleQueuePoller.new(queue_url: queue_url, connection_pool_block: true) }
|
106
|
-
|
107
113
|
it "uses the connection pool block" do
|
108
114
|
expect(mock_connection_pool).to receive(:with_connection)
|
109
115
|
subject.poll
|
@@ -112,6 +118,19 @@ describe Pheme::QueuePoller do
|
|
112
118
|
|
113
119
|
context "without connection pool block" do
|
114
120
|
subject { ExampleQueuePoller.new(queue_url: queue_url) }
|
121
|
+
let(:message) { { status: 'complete' } }
|
122
|
+
let(:notification) { { 'MessageId' => SecureRandom.uuid, 'Message' => message.to_json, 'Type' => 'Notification' } }
|
123
|
+
let!(:queue_message) do
|
124
|
+
OpenStruct.new(
|
125
|
+
body: notification.to_json,
|
126
|
+
message_id: message_id,
|
127
|
+
)
|
128
|
+
end
|
129
|
+
|
130
|
+
before(:each) do
|
131
|
+
allow(subject.queue_poller).to receive(:poll).and_yield(queue_message)
|
132
|
+
allow(subject.queue_poller).to receive(:delete_message).with(queue_message)
|
133
|
+
end
|
115
134
|
|
116
135
|
it "does not call ActiveRecord" do
|
117
136
|
expect(ActiveRecord::Base).not_to receive(:connection_pool)
|
@@ -120,64 +139,87 @@ describe Pheme::QueuePoller do
|
|
120
139
|
end
|
121
140
|
|
122
141
|
context "when a valid message is yielded" do
|
123
|
-
|
142
|
+
subject { ExampleQueuePoller.new(queue_url: queue_url) }
|
143
|
+
let(:message) { { id: "id-123", status: "complete" } }
|
144
|
+
let(:notification) do
|
124
145
|
{
|
125
|
-
|
126
|
-
|
146
|
+
'MessageId' => SecureRandom.uuid,
|
147
|
+
'Message' => message.to_json,
|
148
|
+
'Type' => 'Notification',
|
127
149
|
}
|
128
150
|
end
|
129
|
-
let(:
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
message
|
151
|
+
let!(:queue_message) do
|
152
|
+
OpenStruct.new(
|
153
|
+
body: notification.to_json,
|
154
|
+
message_id: message_id,
|
155
|
+
)
|
135
156
|
end
|
157
|
+
|
136
158
|
before(:each) do
|
137
|
-
allow(
|
159
|
+
allow(subject.queue_poller).to receive(:poll).and_yield(queue_message)
|
160
|
+
allow(subject.queue_poller).to receive(:delete_message).with(queue_message)
|
138
161
|
end
|
139
162
|
|
140
|
-
subject { ExampleQueuePoller.new(queue_url: queue_url) }
|
141
|
-
|
142
163
|
it "handles the message" do
|
143
|
-
expect(ExampleMessageHandler).to receive(:new).with(message: RecursiveOpenStruct.new(
|
164
|
+
expect(ExampleMessageHandler).to receive(:new).with(message: RecursiveOpenStruct.new(message))
|
144
165
|
subject.poll
|
145
166
|
end
|
146
167
|
|
147
168
|
it "deletes the message from the queue" do
|
148
|
-
expect(
|
169
|
+
expect(subject.queue_poller).to receive(:delete_message).with(queue_message)
|
149
170
|
subject.poll
|
150
171
|
end
|
151
172
|
end
|
152
173
|
|
153
174
|
context "when an invalid message is yielded" do
|
154
|
-
|
175
|
+
subject { ExampleQueuePoller.new(queue_url: queue_url) }
|
176
|
+
let(:message) { { id: "id-123", status: "unknown-abc" } }
|
177
|
+
let(:notification) do
|
155
178
|
{
|
156
|
-
|
157
|
-
|
179
|
+
'MessageId' => SecureRandom.uuid,
|
180
|
+
'Message' => message.to_json,
|
181
|
+
'Type' => 'Notification',
|
158
182
|
}
|
159
183
|
end
|
160
|
-
let(:
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
message
|
184
|
+
let!(:queue_message) do
|
185
|
+
OpenStruct.new(
|
186
|
+
body: notification.to_json,
|
187
|
+
message_id: message_id,
|
188
|
+
)
|
166
189
|
end
|
190
|
+
|
167
191
|
before(:each) do
|
168
|
-
allow(
|
169
|
-
allow(
|
192
|
+
allow(subject.queue_poller).to receive(:poll).and_yield(queue_message)
|
193
|
+
allow(subject.queue_poller).to receive(:delete).with(queue_message)
|
194
|
+
allow(Pheme.logger).to receive(:error)
|
170
195
|
end
|
171
196
|
|
172
|
-
subject { ExampleQueuePoller.new(queue_url: queue_url) }
|
173
|
-
|
174
197
|
it "logs the error" do
|
175
198
|
subject.poll
|
176
|
-
expect(Pheme).to have_received(:
|
199
|
+
expect(Pheme.logger).to have_received(:error) do |error|
|
200
|
+
expect(error).to be_a(ArgumentError)
|
201
|
+
expect(error.message).to eq('Unknown message status: unknown-abc')
|
202
|
+
end
|
177
203
|
end
|
178
204
|
|
179
205
|
it "does not delete the message from the queue" do
|
180
|
-
expect(
|
206
|
+
expect(subject.queue_poller).not_to receive(:delete_message)
|
207
|
+
subject.poll
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
context "AWS-event message" do
|
212
|
+
subject { ExampleAwsEventQueuePoller.new(queue_url: queue_url) }
|
213
|
+
let(:queue_message) { OpenStruct.new(body: { 'Records' => records }.to_json) }
|
214
|
+
let(:records) do
|
215
|
+
[{ 'eventVersion' => '2.0', 'eventSource': 'aws:s3' }]
|
216
|
+
end
|
217
|
+
before(:each) do
|
218
|
+
allow(subject.queue_poller).to receive(:poll).and_yield(queue_message)
|
219
|
+
allow(subject.queue_poller).to receive(:delete).with(queue_message)
|
220
|
+
end
|
221
|
+
|
222
|
+
it "logs the message" do
|
181
223
|
subject.poll
|
182
224
|
end
|
183
225
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
require 'rspec'
|
2
|
+
require 'rspec/its'
|
3
|
+
require 'rspec/collection_matchers'
|
4
|
+
require 'pp'
|
2
5
|
|
3
6
|
require './lib/pheme'
|
4
7
|
|
@@ -8,6 +11,8 @@ RSpec.configure do |config|
|
|
8
11
|
config.filter_run :focus
|
9
12
|
config.run_all_when_everything_filtered = true
|
10
13
|
|
14
|
+
ENV['AWS_REGION'] = 'us-east-1'
|
15
|
+
|
11
16
|
config.before(:each) do
|
12
17
|
Pheme.reset_configuration!
|
13
18
|
end
|
@@ -21,11 +21,11 @@ describe Pheme::TopicPublisher do
|
|
21
21
|
it "publishes the correct events" do
|
22
22
|
expect(Pheme.configuration.sns_client).to receive(:publish).with({
|
23
23
|
topic_arn: "arn:aws:sns:whatever",
|
24
|
-
message: {id: "id-0", status: "complete"}.to_json,
|
24
|
+
message: { id: "id-0", status: "complete" }.to_json,
|
25
25
|
})
|
26
26
|
expect(Pheme.configuration.sns_client).to receive(:publish).with({
|
27
27
|
topic_arn: "arn:aws:sns:whatever",
|
28
|
-
message: {id: "id-1", status: "complete"}.to_json,
|
28
|
+
message: { id: "id-1", status: "complete" }.to_json,
|
29
29
|
})
|
30
30
|
subject.publish_events
|
31
31
|
end
|
@@ -34,13 +34,13 @@ describe Pheme::TopicPublisher do
|
|
34
34
|
let(:topic_arn) { "arn:aws:sns:anything" }
|
35
35
|
let(:message) { "don't touch my string" }
|
36
36
|
|
37
|
-
subject {Pheme::TopicPublisher.new(topic_arn: topic_arn)}
|
37
|
+
subject { Pheme::TopicPublisher.new(topic_arn: topic_arn) }
|
38
38
|
|
39
39
|
it "publishes unchanged message" do
|
40
40
|
expect(Pheme.configuration.sns_client).to receive(:publish).with({
|
41
|
-
|
42
|
-
|
43
|
-
|
41
|
+
topic_arn: topic_arn,
|
42
|
+
message: message,
|
43
|
+
})
|
44
44
|
subject.publish(message)
|
45
45
|
end
|
46
46
|
end
|
metadata
CHANGED
@@ -1,23 +1,20 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pheme
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Graham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-11-
|
11
|
+
date: 2017-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '2'
|
20
|
-
- - "<"
|
21
18
|
- !ruby/object:Gem::Version
|
22
19
|
version: '4'
|
23
20
|
type: :runtime
|
@@ -25,16 +22,16 @@ dependencies:
|
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
26
23
|
requirements:
|
27
24
|
- - ">="
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: '2'
|
30
|
-
- - "<"
|
31
25
|
- !ruby/object:Gem::Version
|
32
26
|
version: '4'
|
33
27
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
28
|
+
name: aws-sdk
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
36
30
|
requirements:
|
37
31
|
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2'
|
34
|
+
- - "<"
|
38
35
|
- !ruby/object:Gem::Version
|
39
36
|
version: '4'
|
40
37
|
type: :runtime
|
@@ -42,6 +39,9 @@ dependencies:
|
|
42
39
|
version_requirements: !ruby/object:Gem::Requirement
|
43
40
|
requirements:
|
44
41
|
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '2'
|
44
|
+
- - "<"
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '4'
|
47
47
|
- !ruby/object:Gem::Dependency
|
@@ -76,30 +76,72 @@ dependencies:
|
|
76
76
|
name: rspec
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
|
-
- - "
|
79
|
+
- - ">="
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version: '
|
81
|
+
version: '0'
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- - "
|
86
|
+
- - ">="
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: '
|
88
|
+
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: rspec-collection_matchers
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: rspec-its
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - ">="
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '0'
|
89
117
|
- !ruby/object:Gem::Dependency
|
90
118
|
name: rspec_junit_formatter
|
91
119
|
requirement: !ruby/object:Gem::Requirement
|
92
120
|
requirements:
|
93
|
-
- - "
|
121
|
+
- - ">="
|
94
122
|
- !ruby/object:Gem::Version
|
95
|
-
version: '0
|
123
|
+
version: '0'
|
96
124
|
type: :development
|
97
125
|
prerelease: false
|
98
126
|
version_requirements: !ruby/object:Gem::Requirement
|
99
127
|
requirements:
|
100
|
-
- - "
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
name: rubocop
|
133
|
+
requirement: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
type: :development
|
139
|
+
prerelease: false
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - ">="
|
101
143
|
- !ruby/object:Gem::Version
|
102
|
-
version: '0
|
144
|
+
version: '0'
|
103
145
|
description: Ruby AWS SNS publisher + SQS poller & message handler
|
104
146
|
email:
|
105
147
|
- peter@wealthsimple.com
|
@@ -109,6 +151,7 @@ extra_rdoc_files: []
|
|
109
151
|
files:
|
110
152
|
- ".gitignore"
|
111
153
|
- ".rspec"
|
154
|
+
- ".rubocop.yml"
|
112
155
|
- Gemfile
|
113
156
|
- LICENSE
|
114
157
|
- README.md
|