sqs_buffer 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/lib/sqs_buffer/client.rb +25 -12
- data/lib/sqs_buffer/version.rb +1 -1
- data/sqs_buffer.gemspec +5 -4
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e3d90a0a63db12cacc1eb5269edd25ff91eb1f1
|
4
|
+
data.tar.gz: 5318180628aef11ee2070b2f05eba6d3df903baf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0ce665df45a315522d9765c7d0e590737fd8405fe120ad2c165768663b514d602e66827ef537094a06b4c1f5e46c35d40bdacfae60f1d0013930cf943fdbbf59
|
7
|
+
data.tar.gz: d0b3baa4c2806fecec224d4db64a3fde2c3c2c193cfef59475bc2f90d4b547c526581e07588e3fb06a5f2a1edff8d3370b470c0275006076bbbd1cfcd65801f1
|
data/.gitignore
CHANGED
data/lib/sqs_buffer/client.rb
CHANGED
@@ -6,17 +6,17 @@ module SqsBuffer
|
|
6
6
|
class Client
|
7
7
|
def initialize(opts)
|
8
8
|
@queue_url = opts.fetch(:queue_url) { |k| missing_key!(k) }
|
9
|
-
client
|
9
|
+
client = opts.fetch(:client) { |k| missing_key!(k) }
|
10
10
|
|
11
11
|
@poller = Aws::SQS::QueuePoller.new(@queue_url, client: client)
|
12
12
|
@skip_delete = opts.fetch(:skip_delete, true)
|
13
13
|
@max_number_of_messages = opts.fetch(:max_number_of_messages, 10).to_i
|
14
14
|
@logger = opts.fetch(:logger, Logger.new(STDOUT))
|
15
|
-
@before_request_block
|
16
|
-
@process_block
|
17
|
-
@message_queue
|
18
|
-
@last_process_time
|
19
|
-
@running
|
15
|
+
@before_request_block = Concurrent::MutexAtomicReference.new
|
16
|
+
@process_block = Concurrent::MutexAtomicReference.new
|
17
|
+
@message_queue = Concurrent::Array.new
|
18
|
+
@last_process_time = Concurrent::AtomicFixnum.new(Time.now.to_i)
|
19
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
20
20
|
|
21
21
|
@max_wait_time = Concurrent::AtomicFixnum.new(
|
22
22
|
opts.fetch(:max_wait_time, 300).to_i
|
@@ -32,15 +32,20 @@ module SqsBuffer
|
|
32
32
|
|
33
33
|
@worker_thread = Thread.new do
|
34
34
|
begin
|
35
|
+
sleep_seconds ||= 1
|
35
36
|
opts = {
|
36
37
|
skip_delete: @skip_delete,
|
37
38
|
max_number_of_messages: @max_number_of_messages
|
38
39
|
}
|
39
40
|
@poller.poll(opts) do |messages|
|
40
41
|
store_messages(messages)
|
42
|
+
sleep_seconds = 1
|
41
43
|
end
|
42
44
|
rescue => e
|
43
|
-
|
45
|
+
sleep_seconds = sleep_seconds * 2
|
46
|
+
@logger.error "An unhandled exception(#{e.message}) occurred in worker thread. Sleeping #{sleep_seconds} seconds before retry. | Backtrace: #{e.backtrace}"
|
47
|
+
sleep([sleep_seconds, 30].min)
|
48
|
+
retry
|
44
49
|
end
|
45
50
|
end # End worker thread
|
46
51
|
|
@@ -93,7 +98,11 @@ module SqsBuffer
|
|
93
98
|
end
|
94
99
|
|
95
100
|
def process_all_messages
|
96
|
-
@process_block.value
|
101
|
+
if @process_block.value
|
102
|
+
call_process_block_safely
|
103
|
+
else
|
104
|
+
@logger.info "No process block was given. Discarding all messages."
|
105
|
+
end
|
97
106
|
delete_all_messages
|
98
107
|
touch_process_time
|
99
108
|
rescue StandardError => e
|
@@ -110,6 +119,12 @@ module SqsBuffer
|
|
110
119
|
|
111
120
|
private
|
112
121
|
|
122
|
+
def call_process_block_safely
|
123
|
+
@process_block.value.call(buffer)
|
124
|
+
rescue
|
125
|
+
@logger.error "An exception(#{e.message}) occurred while processing the message queue | Backtrace: #{e.backtrace}"
|
126
|
+
end
|
127
|
+
|
113
128
|
def need_to_process?
|
114
129
|
if !buffer_empty? && (buffer_full? || last_process_time_stale?)
|
115
130
|
true
|
@@ -137,7 +152,7 @@ module SqsBuffer
|
|
137
152
|
if need_to_process?
|
138
153
|
process_all_messages
|
139
154
|
end
|
140
|
-
rescue => e
|
155
|
+
rescue StandardError => e
|
141
156
|
@logger.error "Exception: #{e.message} in before_request block. | Backtrace: #{e.backtrace}"
|
142
157
|
end
|
143
158
|
end # End Poller loop
|
@@ -147,14 +162,12 @@ module SqsBuffer
|
|
147
162
|
messages.each do |msg|
|
148
163
|
store_message(msg)
|
149
164
|
end
|
150
|
-
rescue => e
|
165
|
+
rescue StandardError => e
|
151
166
|
@logger.error "Exception: #{e.message} while storing messages: #{messages} | Backtrace: #{e.backtrace}"
|
152
167
|
end
|
153
168
|
|
154
169
|
def store_message(msg)
|
155
170
|
@message_queue << msg
|
156
|
-
rescue StandardError => e
|
157
|
-
@logger.error "Exception: #{e.message} while storing message: #{msg} | Backtrace: #{e.backtrace}"
|
158
171
|
end
|
159
172
|
|
160
173
|
def touch_process_time
|
data/lib/sqs_buffer/version.rb
CHANGED
data/sqs_buffer.gemspec
CHANGED
@@ -26,9 +26,10 @@ more than 10 at at time.}
|
|
26
26
|
spec.add_dependency "aws-sdk", "~> 2.2"
|
27
27
|
spec.add_dependency "concurrent-ruby", "~> 1.0"
|
28
28
|
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
29
|
+
spec.add_development_dependency 'bundler'
|
30
|
+
spec.add_development_dependency 'rake'
|
31
|
+
spec.add_development_dependency 'rspec'
|
32
|
+
spec.add_development_dependency 'dotenv'
|
33
|
+
spec.add_development_dependency 'pry'
|
33
34
|
end
|
34
35
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqs_buffer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Thomas
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
name: dotenv
|
90
|
+
prerelease: false
|
91
|
+
type: :development
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
requirement: !ruby/object:Gem::Requirement
|
85
99
|
requirements:
|