sqs_buffer 0.3.1 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +4 -0
- data/.ruby-version +1 -1
- data/benchmark/spike.rb +24 -0
- data/lib/sqs_buffer/client.rb +38 -18
- data/lib/sqs_buffer/version.rb +1 -1
- data/sqs_buffer.gemspec +6 -5
- metadata +49 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 37994eb983ac117a3ff59c7a2ed4a8f590f28a5c16f300fe6babdc858c6c70fe
|
4
|
+
data.tar.gz: 2efc4a0a7980668b110361afc5dec585fbbcc5ad7f2f3a11ade0710fb1892e84
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60ead383ef647c7e4beee85da8c837a9bb5ed72ca54119a143e34e8b8d6094b805f4144b48b450447059fdf2df70ec95fe36433907120cddcda593b4bb926c2b
|
7
|
+
data.tar.gz: 1d4d56be0a2f52485186b7c47447ca80e522f3e3e2a3ec5a780ef003cfa1f341ae06fdfdbb72e00793ab166325f28e6d15d7e25ba1dfb284c3ec90164d5d51c8
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
jruby-
|
1
|
+
jruby-9.2.9.0
|
data/benchmark/spike.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'pry'
|
2
|
+
require 'benchmark'
|
3
|
+
require 'concurrent'
|
4
|
+
|
5
|
+
@message_queue = Concurrent::Array.new
|
6
|
+
10000.times { @message_queue << Class.new }
|
7
|
+
|
8
|
+
1.times do
|
9
|
+
Benchmark.bmbm do |x|
|
10
|
+
x.report("reader") { 100000.times { @message_queue } }
|
11
|
+
end
|
12
|
+
|
13
|
+
Benchmark.bmbm do |x|
|
14
|
+
x.report("dup") { 100000.times { @message_queue.dup } }
|
15
|
+
end
|
16
|
+
|
17
|
+
Benchmark.bmbm do |x|
|
18
|
+
# This is super slow.....
|
19
|
+
x.report("internal dup") { 100.times { @message_queue.map(&:dup) } }
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
|
data/lib/sqs_buffer/client.rb
CHANGED
@@ -6,17 +6,17 @@ module SqsBuffer
|
|
6
6
|
class Client
|
7
7
|
def initialize(opts)
|
8
8
|
@queue_url = opts.fetch(:queue_url) { |k| missing_key!(k) }
|
9
|
-
client
|
9
|
+
client = opts.fetch(:client) { |k| missing_key!(k) }
|
10
10
|
|
11
11
|
@poller = Aws::SQS::QueuePoller.new(@queue_url, client: client)
|
12
12
|
@skip_delete = opts.fetch(:skip_delete, true)
|
13
13
|
@max_number_of_messages = opts.fetch(:max_number_of_messages, 10).to_i
|
14
14
|
@logger = opts.fetch(:logger, Logger.new(STDOUT))
|
15
|
-
@before_request_block
|
16
|
-
@process_block
|
17
|
-
@message_queue
|
18
|
-
@last_process_time
|
19
|
-
@running
|
15
|
+
@before_request_block = Concurrent::MutexAtomicReference.new
|
16
|
+
@process_block = Concurrent::MutexAtomicReference.new
|
17
|
+
@message_queue = Concurrent::Array.new
|
18
|
+
@last_process_time = Concurrent::AtomicFixnum.new(Time.now.to_i)
|
19
|
+
@running = Concurrent::AtomicBoolean.new(false)
|
20
20
|
|
21
21
|
@max_wait_time = Concurrent::AtomicFixnum.new(
|
22
22
|
opts.fetch(:max_wait_time, 300).to_i
|
@@ -31,12 +31,21 @@ module SqsBuffer
|
|
31
31
|
@running.make_true
|
32
32
|
|
33
33
|
@worker_thread = Thread.new do
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
34
|
+
begin
|
35
|
+
sleep_seconds ||= 1
|
36
|
+
opts = {
|
37
|
+
skip_delete: @skip_delete,
|
38
|
+
max_number_of_messages: @max_number_of_messages
|
39
|
+
}
|
40
|
+
@poller.poll(opts) do |messages|
|
41
|
+
store_messages(messages)
|
42
|
+
sleep_seconds = 1
|
43
|
+
end
|
44
|
+
rescue => e
|
45
|
+
sleep_seconds = sleep_seconds * 2
|
46
|
+
@logger.error "An unhandled exception(#{e.message}) occurred in worker thread. Sleeping #{sleep_seconds} seconds before retry. | Backtrace: #{e.backtrace}"
|
47
|
+
sleep([sleep_seconds, 30].min)
|
48
|
+
retry
|
40
49
|
end
|
41
50
|
end # End worker thread
|
42
51
|
|
@@ -65,7 +74,10 @@ module SqsBuffer
|
|
65
74
|
|
66
75
|
def buffer
|
67
76
|
# Return a copy of the array events to guard against potential mutation
|
68
|
-
|
77
|
+
# This fails on jruby 9000 for some reason....
|
78
|
+
# Marshal.load( Marshal.dump(@message_queue) )
|
79
|
+
# Should we dup all items? look at benchmark :(
|
80
|
+
@message_queue.dup
|
69
81
|
end
|
70
82
|
|
71
83
|
def shutting_down?
|
@@ -89,7 +101,11 @@ module SqsBuffer
|
|
89
101
|
end
|
90
102
|
|
91
103
|
def process_all_messages
|
92
|
-
@process_block.value
|
104
|
+
if @process_block.value
|
105
|
+
call_process_block_safely
|
106
|
+
else
|
107
|
+
@logger.info "No process block was given. Discarding all messages."
|
108
|
+
end
|
93
109
|
delete_all_messages
|
94
110
|
touch_process_time
|
95
111
|
rescue StandardError => e
|
@@ -106,6 +122,12 @@ module SqsBuffer
|
|
106
122
|
|
107
123
|
private
|
108
124
|
|
125
|
+
def call_process_block_safely
|
126
|
+
@process_block.value.call(buffer)
|
127
|
+
rescue StandardError => e
|
128
|
+
@logger.error "An exception(#{e.message}) occurred while processing the message queue | Backtrace: #{e.backtrace}"
|
129
|
+
end
|
130
|
+
|
109
131
|
def need_to_process?
|
110
132
|
if !buffer_empty? && (buffer_full? || last_process_time_stale?)
|
111
133
|
true
|
@@ -133,7 +155,7 @@ module SqsBuffer
|
|
133
155
|
if need_to_process?
|
134
156
|
process_all_messages
|
135
157
|
end
|
136
|
-
rescue => e
|
158
|
+
rescue StandardError => e
|
137
159
|
@logger.error "Exception: #{e.message} in before_request block. | Backtrace: #{e.backtrace}"
|
138
160
|
end
|
139
161
|
end # End Poller loop
|
@@ -143,14 +165,12 @@ module SqsBuffer
|
|
143
165
|
messages.each do |msg|
|
144
166
|
store_message(msg)
|
145
167
|
end
|
146
|
-
rescue => e
|
168
|
+
rescue StandardError => e
|
147
169
|
@logger.error "Exception: #{e.message} while storing messages: #{messages} | Backtrace: #{e.backtrace}"
|
148
170
|
end
|
149
171
|
|
150
172
|
def store_message(msg)
|
151
173
|
@message_queue << msg
|
152
|
-
rescue StandardError => e
|
153
|
-
@logger.error "Exception: #{e.message} while storing message: #{msg} | Backtrace: #{e.backtrace}"
|
154
174
|
end
|
155
175
|
|
156
176
|
def touch_process_time
|
data/lib/sqs_buffer/version.rb
CHANGED
data/sqs_buffer.gemspec
CHANGED
@@ -23,12 +23,13 @@ more than 10 at at time.}
|
|
23
23
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
24
|
spec.require_paths = ["lib"]
|
25
25
|
|
26
|
-
spec.add_dependency "aws-sdk", "~>
|
26
|
+
spec.add_dependency "aws-sdk", "~> 3"
|
27
27
|
spec.add_dependency "concurrent-ruby", "~> 1.0"
|
28
28
|
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
29
|
+
spec.add_development_dependency 'bundler'
|
30
|
+
spec.add_development_dependency 'rake'
|
31
|
+
spec.add_development_dependency 'rspec'
|
32
|
+
spec.add_development_dependency 'dotenv'
|
33
|
+
spec.add_development_dependency 'pry'
|
33
34
|
end
|
34
35
|
|
metadata
CHANGED
@@ -1,97 +1,111 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqs_buffer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Thomas
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
|
-
- - ~>
|
17
|
+
- - "~>"
|
17
18
|
- !ruby/object:Gem::Version
|
18
|
-
version: '
|
19
|
-
name: aws-sdk
|
20
|
-
prerelease: false
|
19
|
+
version: '3'
|
21
20
|
type: :runtime
|
21
|
+
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '3'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
+
name: concurrent-ruby
|
28
29
|
requirement: !ruby/object:Gem::Requirement
|
29
30
|
requirements:
|
30
|
-
- - ~>
|
31
|
+
- - "~>"
|
31
32
|
- !ruby/object:Gem::Version
|
32
33
|
version: '1.0'
|
33
|
-
name: concurrent-ruby
|
34
|
-
prerelease: false
|
35
34
|
type: :runtime
|
35
|
+
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
42
43
|
requirement: !ruby/object:Gem::Requirement
|
43
44
|
requirements:
|
44
|
-
- -
|
45
|
+
- - ">="
|
45
46
|
- !ruby/object:Gem::Version
|
46
47
|
version: '0'
|
47
|
-
name: bundler
|
48
|
-
prerelease: false
|
49
48
|
type: :development
|
49
|
+
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
56
57
|
requirement: !ruby/object:Gem::Requirement
|
57
58
|
requirements:
|
58
|
-
- -
|
59
|
+
- - ">="
|
59
60
|
- !ruby/object:Gem::Version
|
60
61
|
version: '0'
|
61
|
-
name: rake
|
62
|
-
prerelease: false
|
63
62
|
type: :development
|
63
|
+
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
70
71
|
requirement: !ruby/object:Gem::Requirement
|
71
72
|
requirements:
|
72
|
-
- -
|
73
|
+
- - ">="
|
73
74
|
- !ruby/object:Gem::Version
|
74
75
|
version: '0'
|
75
|
-
name: rspec
|
76
|
-
prerelease: false
|
77
76
|
type: :development
|
77
|
+
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
+
name: dotenv
|
84
85
|
requirement: !ruby/object:Gem::Requirement
|
85
86
|
requirements:
|
86
|
-
- -
|
87
|
+
- - ">="
|
87
88
|
- !ruby/object:Gem::Version
|
88
89
|
version: '0'
|
89
|
-
|
90
|
+
type: :development
|
90
91
|
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: pry
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
91
104
|
type: :development
|
105
|
+
prerelease: false
|
92
106
|
version_requirements: !ruby/object:Gem::Requirement
|
93
107
|
requirements:
|
94
|
-
- -
|
108
|
+
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
110
|
version: '0'
|
97
111
|
description: |-
|
@@ -104,14 +118,15 @@ executables: []
|
|
104
118
|
extensions: []
|
105
119
|
extra_rdoc_files: []
|
106
120
|
files:
|
107
|
-
- .gitignore
|
108
|
-
- .rspec
|
109
|
-
- .ruby-version
|
110
|
-
- .travis.yml
|
121
|
+
- ".gitignore"
|
122
|
+
- ".rspec"
|
123
|
+
- ".ruby-version"
|
124
|
+
- ".travis.yml"
|
111
125
|
- Gemfile
|
112
126
|
- LICENSE.txt
|
113
127
|
- README.md
|
114
128
|
- Rakefile
|
129
|
+
- benchmark/spike.rb
|
115
130
|
- bin/console
|
116
131
|
- bin/setup
|
117
132
|
- lib/sqs_buffer.rb
|
@@ -128,17 +143,16 @@ require_paths:
|
|
128
143
|
- lib
|
129
144
|
required_ruby_version: !ruby/object:Gem::Requirement
|
130
145
|
requirements:
|
131
|
-
- -
|
146
|
+
- - ">="
|
132
147
|
- !ruby/object:Gem::Version
|
133
148
|
version: '0'
|
134
149
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
135
150
|
requirements:
|
136
|
-
- -
|
151
|
+
- - ">="
|
137
152
|
- !ruby/object:Gem::Version
|
138
153
|
version: '0'
|
139
154
|
requirements: []
|
140
|
-
|
141
|
-
rubygems_version: 2.4.5
|
155
|
+
rubygems_version: 3.0.8
|
142
156
|
signing_key:
|
143
157
|
specification_version: 4
|
144
158
|
summary: Buffer SQS messages to process more than 10 events
|