sqs_buffer 0.3.2 → 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/lib/sqs_buffer/client.rb +25 -12
- data/lib/sqs_buffer/version.rb +1 -1
- data/sqs_buffer.gemspec +5 -4
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e3d90a0a63db12cacc1eb5269edd25ff91eb1f1
|
4
|
+
data.tar.gz: 5318180628aef11ee2070b2f05eba6d3df903baf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0ce665df45a315522d9765c7d0e590737fd8405fe120ad2c165768663b514d602e66827ef537094a06b4c1f5e46c35d40bdacfae60f1d0013930cf943fdbbf59
|
7
|
+
data.tar.gz: d0b3baa4c2806fecec224d4db64a3fde2c3c2c193cfef59475bc2f90d4b547c526581e07588e3fb06a5f2a1edff8d3370b470c0275006076bbbd1cfcd65801f1
|
data/.gitignore
CHANGED
data/lib/sqs_buffer/client.rb
CHANGED
@@ -6,17 +6,17 @@ module SqsBuffer
|
|
6
6
|
class Client
|
7
7
|
def initialize(opts)
|
8
8
|
@queue_url = opts.fetch(:queue_url) { |k| missing_key!(k) }
|
9
|
-
client
|
9
|
+
client = opts.fetch(:client) { |k| missing_key!(k) }
|
10
10
|
|
11
11
|
@poller = Aws::SQS::QueuePoller.new(@queue_url, client: client)
|
12
12
|
@skip_delete = opts.fetch(:skip_delete, true)
|
13
13
|
@max_number_of_messages = opts.fetch(:max_number_of_messages, 10).to_i
|
14
14
|
@logger = opts.fetch(:logger, Logger.new(STDOUT))
|
15
|
-
@before_request_block
|
16
|
-
@process_block
|
17
|
-
@message_queue
|
18
|
-
@last_process_time
|
19
|
-
@running
|
15
|
+
@before_request_block = Concurrent::MutexAtomicReference.new
|
16
|
+
@process_block = Concurrent::MutexAtomicReference.new
|
17
|
+
@message_queue = Concurrent::Array.new
|
18
|
+
@last_process_time = Concurrent::AtomicFixnum.new(Time.now.to_i)
|
19
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
20
20
|
|
21
21
|
@max_wait_time = Concurrent::AtomicFixnum.new(
|
22
22
|
opts.fetch(:max_wait_time, 300).to_i
|
@@ -32,15 +32,20 @@ module SqsBuffer
|
|
32
32
|
|
33
33
|
@worker_thread = Thread.new do
|
34
34
|
begin
|
35
|
+
sleep_seconds ||= 1
|
35
36
|
opts = {
|
36
37
|
skip_delete: @skip_delete,
|
37
38
|
max_number_of_messages: @max_number_of_messages
|
38
39
|
}
|
39
40
|
@poller.poll(opts) do |messages|
|
40
41
|
store_messages(messages)
|
42
|
+
sleep_seconds = 1
|
41
43
|
end
|
42
44
|
rescue => e
|
43
|
-
|
45
|
+
sleep_seconds = sleep_seconds * 2
|
46
|
+
@logger.error "An unhandled exception(#{e.message}) occurred in worker thread. Sleeping #{sleep_seconds} seconds before retry. | Backtrace: #{e.backtrace}"
|
47
|
+
sleep([sleep_seconds, 30].min)
|
48
|
+
retry
|
44
49
|
end
|
45
50
|
end # End worker thread
|
46
51
|
|
@@ -93,7 +98,11 @@ module SqsBuffer
|
|
93
98
|
end
|
94
99
|
|
95
100
|
def process_all_messages
|
96
|
-
@process_block.value
|
101
|
+
if @process_block.value
|
102
|
+
call_process_block_safely
|
103
|
+
else
|
104
|
+
@logger.info "No process block was given. Discarding all messages."
|
105
|
+
end
|
97
106
|
delete_all_messages
|
98
107
|
touch_process_time
|
99
108
|
rescue StandardError => e
|
@@ -110,6 +119,12 @@ module SqsBuffer
|
|
110
119
|
|
111
120
|
private
|
112
121
|
|
122
|
+
def call_process_block_safely
|
123
|
+
@process_block.value.call(buffer)
|
124
|
+
rescue
|
125
|
+
@logger.error "An exception(#{e.message}) occurred while processing the message queue | Backtrace: #{e.backtrace}"
|
126
|
+
end
|
127
|
+
|
113
128
|
def need_to_process?
|
114
129
|
if !buffer_empty? && (buffer_full? || last_process_time_stale?)
|
115
130
|
true
|
@@ -137,7 +152,7 @@ module SqsBuffer
|
|
137
152
|
if need_to_process?
|
138
153
|
process_all_messages
|
139
154
|
end
|
140
|
-
rescue => e
|
155
|
+
rescue StandardError => e
|
141
156
|
@logger.error "Exception: #{e.message} in before_request block. | Backtrace: #{e.backtrace}"
|
142
157
|
end
|
143
158
|
end # End Poller loop
|
@@ -147,14 +162,12 @@ module SqsBuffer
|
|
147
162
|
messages.each do |msg|
|
148
163
|
store_message(msg)
|
149
164
|
end
|
150
|
-
rescue => e
|
165
|
+
rescue StandardError => e
|
151
166
|
@logger.error "Exception: #{e.message} while storing messages: #{messages} | Backtrace: #{e.backtrace}"
|
152
167
|
end
|
153
168
|
|
154
169
|
def store_message(msg)
|
155
170
|
@message_queue << msg
|
156
|
-
rescue StandardError => e
|
157
|
-
@logger.error "Exception: #{e.message} while storing message: #{msg} | Backtrace: #{e.backtrace}"
|
158
171
|
end
|
159
172
|
|
160
173
|
def touch_process_time
|
data/lib/sqs_buffer/version.rb
CHANGED
data/sqs_buffer.gemspec
CHANGED
@@ -26,9 +26,10 @@ more than 10 at at time.}
|
|
26
26
|
spec.add_dependency "aws-sdk", "~> 2.2"
|
27
27
|
spec.add_dependency "concurrent-ruby", "~> 1.0"
|
28
28
|
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
29
|
+
spec.add_development_dependency 'bundler'
|
30
|
+
spec.add_development_dependency 'rake'
|
31
|
+
spec.add_development_dependency 'rspec'
|
32
|
+
spec.add_development_dependency 'dotenv'
|
33
|
+
spec.add_development_dependency 'pry'
|
33
34
|
end
|
34
35
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqs_buffer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Thomas
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
name: dotenv
|
90
|
+
prerelease: false
|
91
|
+
type: :development
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
requirement: !ruby/object:Gem::Requirement
|
85
99
|
requirements:
|