sqs_buffer 0.3.1 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: facb347e330b3c794eb6224a914ffd48e80e5ca0
4
- data.tar.gz: c4bb7fe4e7cef0beba0554de06c01aa1b1bcefb1
2
+ SHA256:
3
+ metadata.gz: 37994eb983ac117a3ff59c7a2ed4a8f590f28a5c16f300fe6babdc858c6c70fe
4
+ data.tar.gz: 2efc4a0a7980668b110361afc5dec585fbbcc5ad7f2f3a11ade0710fb1892e84
5
5
  SHA512:
6
- metadata.gz: d98f8cf578b7b93db709cb89015650477ebeea7f91487408fc8a7093e86429815868d9e228cbece3eb275db3b2f2379e2df3aa7c72fbf88dfaa52ccf6cad9201
7
- data.tar.gz: 3af6d9bef57e50dfefd62c00ba5fd04cef32e0d779c465b8fa9ff04418c5a2dbbaa4f21c5143b62d19a87fbb10f7baaed94ce9c5e5dce8274c9f908e195de874
6
+ metadata.gz: 60ead383ef647c7e4beee85da8c837a9bb5ed72ca54119a143e34e8b8d6094b805f4144b48b450447059fdf2df70ec95fe36433907120cddcda593b4bb926c2b
7
+ data.tar.gz: 1d4d56be0a2f52485186b7c47447ca80e522f3e3e2a3ec5a780ef003cfa1f341ae06fdfdbb72e00793ab166325f28e6d15d7e25ba1dfb284c3ec90164d5d51c8
data/.gitignore CHANGED
@@ -7,3 +7,7 @@
7
7
  /pkg/
8
8
  /spec/reports/
9
9
  /tmp/
10
+
11
+ # Ignore application configuration
12
+ .env
13
+ /.idea/
@@ -1 +1 @@
1
- jruby-1.7.19
1
+ jruby-9.2.9.0
@@ -0,0 +1,24 @@
1
+ require 'pry'
2
+ require 'benchmark'
3
+ require 'concurrent'
4
+
5
+ @message_queue = Concurrent::Array.new
6
+ 10000.times { @message_queue << Class.new }
7
+
8
+ 1.times do
9
+ Benchmark.bmbm do |x|
10
+ x.report("reader") { 100000.times { @message_queue } }
11
+ end
12
+
13
+ Benchmark.bmbm do |x|
14
+ x.report("dup") { 100000.times { @message_queue.dup } }
15
+ end
16
+
17
+ Benchmark.bmbm do |x|
18
+ # This is super slow.....
19
+ x.report("internal dup") { 100.times { @message_queue.map(&:dup) } }
20
+ end
21
+
22
+ end
23
+
24
+
@@ -6,17 +6,17 @@ module SqsBuffer
6
6
  class Client
7
7
  def initialize(opts)
8
8
  @queue_url = opts.fetch(:queue_url) { |k| missing_key!(k) }
9
- client = opts.fetch(:client) { |k| missing_key!(k) }
9
+ client = opts.fetch(:client) { |k| missing_key!(k) }
10
10
 
11
11
  @poller = Aws::SQS::QueuePoller.new(@queue_url, client: client)
12
12
  @skip_delete = opts.fetch(:skip_delete, true)
13
13
  @max_number_of_messages = opts.fetch(:max_number_of_messages, 10).to_i
14
14
  @logger = opts.fetch(:logger, Logger.new(STDOUT))
15
- @before_request_block = Concurrent::MutexAtomicReference.new
16
- @process_block = Concurrent::MutexAtomicReference.new
17
- @message_queue = Concurrent::Array.new
18
- @last_process_time = Concurrent::AtomicFixnum.new(Time.now.to_i)
19
- @running = Concurrent::AtomicBoolean.new(false)
15
+ @before_request_block = Concurrent::MutexAtomicReference.new
16
+ @process_block = Concurrent::MutexAtomicReference.new
17
+ @message_queue = Concurrent::Array.new
18
+ @last_process_time = Concurrent::AtomicFixnum.new(Time.now.to_i)
19
+ @running = Concurrent::AtomicBoolean.new(false)
20
20
 
21
21
  @max_wait_time = Concurrent::AtomicFixnum.new(
22
22
  opts.fetch(:max_wait_time, 300).to_i
@@ -31,12 +31,21 @@ module SqsBuffer
31
31
  @running.make_true
32
32
 
33
33
  @worker_thread = Thread.new do
34
- opts = {
35
- skip_delete: @skip_delete,
36
- max_number_of_messages: @max_number_of_messages
37
- }
38
- @poller.poll(opts) do |messages|
39
- store_messages(messages)
34
+ begin
35
+ sleep_seconds ||= 1
36
+ opts = {
37
+ skip_delete: @skip_delete,
38
+ max_number_of_messages: @max_number_of_messages
39
+ }
40
+ @poller.poll(opts) do |messages|
41
+ store_messages(messages)
42
+ sleep_seconds = 1
43
+ end
44
+ rescue => e
45
+ sleep_seconds = sleep_seconds * 2
46
+ @logger.error "An unhandled exception(#{e.message}) occurred in worker thread. Sleeping #{sleep_seconds} seconds before retry. | Backtrace: #{e.backtrace}"
47
+ sleep([sleep_seconds, 30].min)
48
+ retry
40
49
  end
41
50
  end # End worker thread
42
51
 
@@ -65,7 +74,10 @@ module SqsBuffer
65
74
 
66
75
  def buffer
67
76
  # Return a copy of the array events to guard against potential mutation
68
- Marshal.load( Marshal.dump(@message_queue) )
77
+ # This fails on jruby 9000 for some reason....
78
+ # Marshal.load( Marshal.dump(@message_queue) )
79
+ # Should we dup all items? look at benchmark :(
80
+ @message_queue.dup
69
81
  end
70
82
 
71
83
  def shutting_down?
@@ -89,7 +101,11 @@ module SqsBuffer
89
101
  end
90
102
 
91
103
  def process_all_messages
92
- @process_block.value.call(buffer)
104
+ if @process_block.value
105
+ call_process_block_safely
106
+ else
107
+ @logger.info "No process block was given. Discarding all messages."
108
+ end
93
109
  delete_all_messages
94
110
  touch_process_time
95
111
  rescue StandardError => e
@@ -106,6 +122,12 @@ module SqsBuffer
106
122
 
107
123
  private
108
124
 
125
+ def call_process_block_safely
126
+ @process_block.value.call(buffer)
127
+ rescue StandardError => e
128
+ @logger.error "An exception(#{e.message}) occurred while processing the message queue | Backtrace: #{e.backtrace}"
129
+ end
130
+
109
131
  def need_to_process?
110
132
  if !buffer_empty? && (buffer_full? || last_process_time_stale?)
111
133
  true
@@ -133,7 +155,7 @@ module SqsBuffer
133
155
  if need_to_process?
134
156
  process_all_messages
135
157
  end
136
- rescue => e
158
+ rescue StandardError => e
137
159
  @logger.error "Exception: #{e.message} in before_request block. | Backtrace: #{e.backtrace}"
138
160
  end
139
161
  end # End Poller loop
@@ -143,14 +165,12 @@ module SqsBuffer
143
165
  messages.each do |msg|
144
166
  store_message(msg)
145
167
  end
146
- rescue => e
168
+ rescue StandardError => e
147
169
  @logger.error "Exception: #{e.message} while storing messages: #{messages} | Backtrace: #{e.backtrace}"
148
170
  end
149
171
 
150
172
  def store_message(msg)
151
173
  @message_queue << msg
152
- rescue StandardError => e
153
- @logger.error "Exception: #{e.message} while storing message: #{msg} | Backtrace: #{e.backtrace}"
154
174
  end
155
175
 
156
176
  def touch_process_time
@@ -1,3 +1,3 @@
1
1
  module SqsBuffer
2
- VERSION = "0.3.1"
2
+ VERSION = "0.3.7"
3
3
  end
@@ -23,12 +23,13 @@ more than 10 at at time.}
23
23
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
24
24
  spec.require_paths = ["lib"]
25
25
 
26
- spec.add_dependency "aws-sdk", "~> 2.2"
26
+ spec.add_dependency "aws-sdk", "~> 3"
27
27
  spec.add_dependency "concurrent-ruby", "~> 1.0"
28
28
 
29
- spec.add_development_dependency "bundler"
30
- spec.add_development_dependency "rake"
31
- spec.add_development_dependency "rspec"
32
- spec.add_development_dependency "pry"
29
+ spec.add_development_dependency 'bundler'
30
+ spec.add_development_dependency 'rake'
31
+ spec.add_development_dependency 'rspec'
32
+ spec.add_development_dependency 'dotenv'
33
+ spec.add_development_dependency 'pry'
33
34
  end
34
35
 
metadata CHANGED
@@ -1,97 +1,111 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sqs_buffer
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.3.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Thomas
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-02-23 00:00:00.000000000 Z
11
+ date: 2020-07-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: aws-sdk
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ~>
17
+ - - "~>"
17
18
  - !ruby/object:Gem::Version
18
- version: '2.2'
19
- name: aws-sdk
20
- prerelease: false
19
+ version: '3'
21
20
  type: :runtime
21
+ prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '2.2'
26
+ version: '3'
27
27
  - !ruby/object:Gem::Dependency
28
+ name: concurrent-ruby
28
29
  requirement: !ruby/object:Gem::Requirement
29
30
  requirements:
30
- - - ~>
31
+ - - "~>"
31
32
  - !ruby/object:Gem::Version
32
33
  version: '1.0'
33
- name: concurrent-ruby
34
- prerelease: false
35
34
  type: :runtime
35
+ prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.0'
41
41
  - !ruby/object:Gem::Dependency
42
+ name: bundler
42
43
  requirement: !ruby/object:Gem::Requirement
43
44
  requirements:
44
- - - '>='
45
+ - - ">="
45
46
  - !ruby/object:Gem::Version
46
47
  version: '0'
47
- name: bundler
48
- prerelease: false
49
48
  type: :development
49
+ prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
+ name: rake
56
57
  requirement: !ruby/object:Gem::Requirement
57
58
  requirements:
58
- - - '>='
59
+ - - ">="
59
60
  - !ruby/object:Gem::Version
60
61
  version: '0'
61
- name: rake
62
- prerelease: false
63
62
  type: :development
63
+ prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
+ name: rspec
70
71
  requirement: !ruby/object:Gem::Requirement
71
72
  requirements:
72
- - - '>='
73
+ - - ">="
73
74
  - !ruby/object:Gem::Version
74
75
  version: '0'
75
- name: rspec
76
- prerelease: false
77
76
  type: :development
77
+ prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
+ name: dotenv
84
85
  requirement: !ruby/object:Gem::Requirement
85
86
  requirements:
86
- - - '>='
87
+ - - ">="
87
88
  - !ruby/object:Gem::Version
88
89
  version: '0'
89
- name: pry
90
+ type: :development
90
91
  prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: pry
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
91
104
  type: :development
105
+ prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
- - - '>='
108
+ - - ">="
95
109
  - !ruby/object:Gem::Version
96
110
  version: '0'
97
111
  description: |-
@@ -104,14 +118,15 @@ executables: []
104
118
  extensions: []
105
119
  extra_rdoc_files: []
106
120
  files:
107
- - .gitignore
108
- - .rspec
109
- - .ruby-version
110
- - .travis.yml
121
+ - ".gitignore"
122
+ - ".rspec"
123
+ - ".ruby-version"
124
+ - ".travis.yml"
111
125
  - Gemfile
112
126
  - LICENSE.txt
113
127
  - README.md
114
128
  - Rakefile
129
+ - benchmark/spike.rb
115
130
  - bin/console
116
131
  - bin/setup
117
132
  - lib/sqs_buffer.rb
@@ -128,17 +143,16 @@ require_paths:
128
143
  - lib
129
144
  required_ruby_version: !ruby/object:Gem::Requirement
130
145
  requirements:
131
- - - '>='
146
+ - - ">="
132
147
  - !ruby/object:Gem::Version
133
148
  version: '0'
134
149
  required_rubygems_version: !ruby/object:Gem::Requirement
135
150
  requirements:
136
- - - '>='
151
+ - - ">="
137
152
  - !ruby/object:Gem::Version
138
153
  version: '0'
139
154
  requirements: []
140
- rubyforge_project:
141
- rubygems_version: 2.4.5
155
+ rubygems_version: 3.0.8
142
156
  signing_key:
143
157
  specification_version: 4
144
158
  summary: Buffer SQS messages to process more than 10 events