sqs_buffer 0.3.1 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: facb347e330b3c794eb6224a914ffd48e80e5ca0
-  data.tar.gz: c4bb7fe4e7cef0beba0554de06c01aa1b1bcefb1
+SHA256:
+  metadata.gz: 37994eb983ac117a3ff59c7a2ed4a8f590f28a5c16f300fe6babdc858c6c70fe
+  data.tar.gz: 2efc4a0a7980668b110361afc5dec585fbbcc5ad7f2f3a11ade0710fb1892e84
 SHA512:
-  metadata.gz: d98f8cf578b7b93db709cb89015650477ebeea7f91487408fc8a7093e86429815868d9e228cbece3eb275db3b2f2379e2df3aa7c72fbf88dfaa52ccf6cad9201
-  data.tar.gz: 3af6d9bef57e50dfefd62c00ba5fd04cef32e0d779c465b8fa9ff04418c5a2dbbaa4f21c5143b62d19a87fbb10f7baaed94ce9c5e5dce8274c9f908e195de874
+  metadata.gz: 60ead383ef647c7e4beee85da8c837a9bb5ed72ca54119a143e34e8b8d6094b805f4144b48b450447059fdf2df70ec95fe36433907120cddcda593b4bb926c2b
+  data.tar.gz: 1d4d56be0a2f52485186b7c47447ca80e522f3e3e2a3ec5a780ef003cfa1f341ae06fdfdbb72e00793ab166325f28e6d15d7e25ba1dfb284c3ec90164d5d51c8

data/.gitignore

@@ -7,3 +7,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+
+# Ignore application configuration
+.env
+/.idea/

data/.ruby-version

@@ -1 +1 @@
-jruby-1.7.19
+jruby-9.2.9.0

data/benchmark/spike.rb

@@ -0,0 +1,24 @@
+require 'pry'
+require 'benchmark'
+require 'concurrent'
+
+@message_queue = Concurrent::Array.new
+10000.times { @message_queue << Class.new }
+
+1.times do
+  Benchmark.bmbm do |x|
+    x.report("reader")  { 100000.times { @message_queue }  }
+  end
+
+  Benchmark.bmbm do |x|
+    x.report("dup")  { 100000.times { @message_queue.dup }  }
+  end
+
+  Benchmark.bmbm do |x|
+    # This is super slow.....
+    x.report("internal dup")  { 100.times { @message_queue.map(&:dup) }  }
+  end
+
+end
+
+

data/lib/sqs_buffer/client.rb

@@ -6,17 +6,17 @@ module SqsBuffer
   class Client
     def initialize(opts)
       @queue_url = opts.fetch(:queue_url) { |k| missing_key!(k) }
-      client = opts.fetch(:client) { |k| missing_key!(k) }
+      client     = opts.fetch(:client) { |k| missing_key!(k) }
 
       @poller = Aws::SQS::QueuePoller.new(@queue_url, client: client)
       @skip_delete            = opts.fetch(:skip_delete, true)
       @max_number_of_messages = opts.fetch(:max_number_of_messages, 10).to_i
       @logger                 = opts.fetch(:logger, Logger.new(STDOUT))
-      @before_request_block = Concurrent::MutexAtomicReference.new
-      @process_block        = Concurrent::MutexAtomicReference.new
-      @message_queue        = Concurrent::Array.new
-      @last_process_time    = Concurrent::AtomicFixnum.new(Time.now.to_i)
-      @running              = Concurrent::AtomicBoolean.new(false)
+      @before_request_block   = Concurrent::MutexAtomicReference.new
+      @process_block          = Concurrent::MutexAtomicReference.new
+      @message_queue          = Concurrent::Array.new
+      @last_process_time      = Concurrent::AtomicFixnum.new(Time.now.to_i)
+      @running                = Concurrent::AtomicBoolean.new(false)
 
       @max_wait_time = Concurrent::AtomicFixnum.new(
         opts.fetch(:max_wait_time, 300).to_i
@@ -31,12 +31,21 @@ module SqsBuffer
       @running.make_true
 
       @worker_thread = Thread.new do
-        opts = {
-          skip_delete: @skip_delete,
-          max_number_of_messages: @max_number_of_messages
-        }
-        @poller.poll(opts) do |messages|
-          store_messages(messages)
+        begin
+          sleep_seconds ||= 1
+          opts = {
+            skip_delete: @skip_delete,
+            max_number_of_messages: @max_number_of_messages
+          }
+          @poller.poll(opts) do |messages|
+            store_messages(messages)
+            sleep_seconds = 1
+          end
+        rescue => e
+          sleep_seconds = sleep_seconds * 2
+          @logger.error "An unhandled exception(#{e.message}) occurred in worker thread. Sleeping #{sleep_seconds} seconds before retry. | Backtrace: #{e.backtrace}"
+          sleep([sleep_seconds, 30].min)
+          retry
         end
       end # End worker thread
 
@@ -65,7 +74,10 @@ module SqsBuffer
 
     def buffer
       # Return a copy of the array events to guard against potential mutation
-      Marshal.load( Marshal.dump(@message_queue) )
+      # This fails on jruby 9000 for some reason....
+      # Marshal.load( Marshal.dump(@message_queue) )
+      # Should we dup all items? look at benchmark :(
+      @message_queue.dup
     end
 
     def shutting_down?
@@ -89,7 +101,11 @@ module SqsBuffer
     end
 
     def process_all_messages
-      @process_block.value.call(buffer)
+      if @process_block.value
+        call_process_block_safely
+      else
+        @logger.info "No process block was given. Discarding all messages."
+      end
       delete_all_messages
       touch_process_time
     rescue StandardError => e
@@ -106,6 +122,12 @@ module SqsBuffer
 
     private
 
+    def call_process_block_safely
+      @process_block.value.call(buffer)
+    rescue StandardError => e
+      @logger.error "An exception(#{e.message}) occurred while processing the message queue | Backtrace: #{e.backtrace}"
+    end
+
     def need_to_process?
       if !buffer_empty? && (buffer_full? || last_process_time_stale?)
         true
@@ -133,7 +155,7 @@ module SqsBuffer
           if need_to_process?
             process_all_messages
           end
-        rescue => e
+        rescue StandardError => e
           @logger.error "Exception: #{e.message} in before_request block. | Backtrace: #{e.backtrace}"
         end
       end # End Poller loop
@@ -143,14 +165,12 @@ module SqsBuffer
       messages.each do |msg|
         store_message(msg)
       end
-    rescue => e
+    rescue StandardError => e
       @logger.error "Exception: #{e.message} while storing messages: #{messages} | Backtrace: #{e.backtrace}"
     end
 
     def store_message(msg)
       @message_queue << msg
-    rescue StandardError => e
-      @logger.error "Exception: #{e.message} while storing message: #{msg} | Backtrace: #{e.backtrace}"
     end
 
     def touch_process_time

data/lib/sqs_buffer/version.rb

@@ -1,3 +1,3 @@
 module SqsBuffer
-  VERSION = "0.3.1"
+  VERSION = "0.3.7"
 end

data/sqs_buffer.gemspec

@@ -23,12 +23,13 @@ more than 10 at at time.}
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "aws-sdk", "~> 2.2"
+  spec.add_dependency "aws-sdk", "~> 3"
   spec.add_dependency "concurrent-ruby", "~> 1.0"
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'dotenv'
+  spec.add_development_dependency 'pry'
 end
 

metadata

@@ -1,97 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: sqs_buffer
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.7
 platform: ruby
 authors:
 - John Thomas
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2016-02-23 00:00:00.000000000 Z
+date: 2020-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
-  name: aws-sdk
-  prerelease: false
+        version: '3'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3'
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: pry
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: |-
@@ -104,14 +118,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .ruby-version
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmark/spike.rb
 - bin/console
 - bin/setup
 - lib/sqs_buffer.rb
@@ -128,17 +143,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 3.0.8
 signing_key:
 specification_version: 4
 summary: Buffer SQS messages to process more than 10 events