sqewer 6.2.1 → 6.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8e1b1d8c965c0c3ffbfcc3905e1d25762d15f727
-  data.tar.gz: 6bbac73b80768676784320373a12bcf76f55289d
+SHA256:
+  metadata.gz: a69e4c565c8d9532e871d74b077188f617fe513519fdc4f72f03ad25ba8c67d2
+  data.tar.gz: 92b73c32b4f9535fc53868f9ba61a3e579bfbba1244c191fc6c34c5a64bd7e2c
 SHA512:
-  metadata.gz: 5b3eb41d4dc80e7ef572743c2d0f67cc3cfbce1df828faea8a84c8949c7f2d087922441d35aa5cfa7eab28d885baffba3167d9bc039c9357e337736e5d7e4f3d
-  data.tar.gz: cb752122d384939a115a025d23dc82634b5da9b8409b5a8c4b4fa5f37b14a73322718c5fd3aaf88e4db2da65a6616d911749ffd7f61d5892fe3c75fe9611bc9b
+  metadata.gz: 339e77231c3b905a5b3477c2bb933d5fadcbb3f6291f30f12b5e4682bc0d858ee9bb1b8da44e1ba59b46b4647cb3bb877b200f180fd78b91e2c775f0c7103755
+  data.tar.gz: 16eb06ee13263bd2af1ce18af5c8c4c5da20bf95bca329644a7ccce9fbbe77c54ea18769b3d2819e13f7f7c0374be293ec7beaa6f324ded588a1f2313591154b

data/.gitignore

@@ -1,54 +1,56 @@
-.rspec
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
 .env
 
-# rcov generated
-coverage
-coverage.data
+# Ignore Byebug command history file.
+.byebug_history
 
-# rdoc generated
-rdoc
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
 
-# yard generated
-doc
-.yardoc
-
-# bundler
-.bundle
-Gemfile.lock
-gemfiles/*.lock
-
-# jeweler generated
-pkg
-
-# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+## Specific to RubyMotion (use of CocoaPods):
 #
-# * Create a file at ~/.gitignore
-# * Include files you want ignored
-# * Run: git config --global core.excludesfile ~/.gitignore
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
 #
-# After doing this, these files will be ignored in all your git projects,
-# saving you from having to 'pollute' every project you touch with them
-#
-# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
-#
-# For MacOS:
-#
-#.DS_Store
+# vendor/Pods/
 
-# For TextMate
-#*.tmproj
-#tmtags
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
 
-# For emacs:
-#*~
-#\#*
-#.\#*
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
 
-# For vim:
-#*.swp
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
 
-# For redcar:
-#.redcar
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
 
-# For rubinius:
-#*.rbc
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*

data/.travis.yml

@@ -6,6 +6,8 @@ rvm:
   - 2.3.7
   - 2.4.5
   - 2.5.1
+  - 2.6.5
+  - 2.7.0
 cache: bundler
 sudo: false
 env:
@@ -19,3 +21,7 @@ matrix:
     gemfile: gemfiles/Gemfile.rails-5.1.x
   - rvm: 2.5.1
     gemfile: gemfiles/Gemfile.rails-4.2.x
+  - rvm: 2.6.5
+    gemfile: gemfiles/Gemfile.rails-4.2.x
+  - rvm: 2.7.0
+    gemfile: gemfiles/Gemfile.rails-4.2.x

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+### 6.5.0
+- Adds `$stdout.sync = true` to CLI to flush the logs to STDOUT
+
+### 6.4.1
+- Retrieve attributes when receiving messages from SQS
+
+### 6.4.0
+- Raise an exception in submit! if the job serializes to a message that is
+  above the native SQS limit for message size.
+- Ensure SendMessageBatch is only performed for batches totaling 256KB of message size or less.
+- Insert Sqewer::Error between StandardError and our custom errors for easier rescuing
+
+### 6.3.0
+- Add support for Ruby 2.7
+
+### 6.2.2
+- Test the Appsignal integration using actual Appsignal libraries
+- In the Appsignal integration, replace a call of `set_queue_start=` with `set_queue_start`
+
 ### 6.2.1
 - Appsignal queue start time should be set as an Integer of milliseconds, not as a Time object
 

data/gemfiles/Gemfile.rails-4.2.x

@@ -4,5 +4,5 @@ source "http://rubygems.org"
 gemspec path: __dir__ + '/..'
 
 gem 'sqlite3', '~> 1.3.6'
-gem 'activejob', "~> 4.2"
-gem 'activerecord', "~> 4.2"
+gem 'activejob', "~> 4.2.0"
+gem 'activerecord', "~> 4.2.0"

data/gemfiles/Gemfile.rails-5.0.x

@@ -4,5 +4,5 @@ source "http://rubygems.org"
 gemspec path: __dir__ + '/..'
 
 gem 'sqlite3', '~> 1.3.6'
-gem 'activejob', "~> 5", "< 5.1"
-gem 'activerecord', "~> 5", "< 5.1"
+gem 'activejob', "~> 5.0.0"
+gem 'activerecord', "~> 5.0.0"

data/gemfiles/Gemfile.rails-5.1.x

@@ -3,6 +3,6 @@ source "http://rubygems.org"
 # Gemspec as base dependency set
 gemspec path: __dir__ + '/..'
 
-gem 'sqlite3', '~> 1.3.6'
-gem 'activejob', "~> 5.1", "< 5.2"
-gem 'activerecord', "~> 5.1", "< 5.2"
+gem 'sqlite3', "~> 1.3", ">= 1.3.6"
+gem 'activejob', "~> 5.1.0"
+gem 'activerecord', "~> 5.1.0"

data/lib/sqewer.rb

@@ -1,5 +1,8 @@
 # The enclosing module for the library
 module Sqewer
+  class Error < StandardError
+  end
+
   # Eager-load everything except extensions. Sort to ensure
   # the files load in the same order on all platforms.
   Dir.glob(__dir__ + '/**/*.rb').sort.each do |path|
@@ -23,7 +26,7 @@ module Sqewer
   def self.submit!(*jobs, **options)
     Sqewer::Submitter.default.submit!(*jobs, **options)
   end
-  
+
   # If we are within Rails, load the railtie
   require_relative 'sqewer/extensions/railtie' if defined?(Rails)
 

data/lib/sqewer/cli.rb

@@ -1,3 +1,4 @@
+$stdout.sync = true
 # Wraps a Worker object in a process-wide commanline handler. Once the `start` method is
 # called, signal handlers will be installed for the following signals:
 #

data/lib/sqewer/connection.rb

@@ -11,7 +11,7 @@ class Sqewer::Connection
   MAX_RANDOM_FAILURES_PER_CALL = 10
   MAX_RANDOM_RECEIVE_FAILURES = 100 # sure to hit the max_elapsed_time of 900 seconds
 
-  NotOurFaultAwsError = Class.new(StandardError)
+  NotOurFaultAwsError = Class.new(Sqewer::Error)
 
   # A wrapper for most important properties of the received message
   class Message < Struct.new(:receipt_handle, :body, :attributes)
@@ -55,8 +55,12 @@ class Sqewer::Connection
   # @return [Array<Message>] an array of Message objects 
   def receive_messages
     Retriable.retriable on: Seahorse::Client::NetworkingError, tries: MAX_RANDOM_RECEIVE_FAILURES do
-      response = client.receive_message(queue_url: @queue_url,
-        wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: BATCH_RECEIVE_SIZE)
+      response = client.receive_message(
+        queue_url: @queue_url,
+        attribute_names: ['All'],
+        wait_time_seconds: DEFAULT_TIMEOUT_SECONDS,
+        max_number_of_messages: BATCH_RECEIVE_SIZE
+      )
       response.messages.map {|message| Message.new(message.receipt_handle, message.body, message.attributes) }
     end
   end
@@ -91,6 +95,58 @@ class Sqewer::Connection
       m[:delay_seconds] = kwargs_for_send[:delay_seconds] if kwargs_for_send[:delay_seconds]
       messages << m
     end
+
+    # each_batch here also needs to ensure that the sum of payload lengths does not exceed 256kb
+    def each_batch
+      regrouped = pack_into_batches(messages, weight_limit: 256 * 1024, batch_length_limit: 10) do |message|
+        message.fetch(:message_body).bytesize
+      end
+      regrouped.each { |b| yield(b) }
+    end
+
+    # Optimizes a large list of items into batches of 10 items
+    # or less and with the sum of item lengths being below 256KB
+    # The block given to the method should return the weight of the given item
+    def pack_into_batches(items, weight_limit:, batch_length_limit:)
+      batches = []
+      current_batch = []
+      current_batch_weight = 0
+
+      # Sort the items by their weight (length of the body).
+      sorted_items = items.sort_by { |item| yield(item) }
+
+      # and then take 1 item from the list and append it to the batch if it fits.
+      # If it doesn't fit, no item after it will fit into this batch either (!)
+      # which is how we can optimize
+      sorted_items.each_with_index do |item|
+        weight_of_this_item = yield(item)
+
+        # First protect from invalid input
+        if weight_of_this_item > weight_limit
+          raise "#{item.inspect} was larger than the permissible limit"
+        # The first limit is on the item count per batch -
+        # if we are limited on that the batch needs to be closed
+        elsif current_batch.length == batch_length_limit
+          batches << current_batch
+          current_batch = []
+          current_batch_weight = 0
+        # If placing this item in the batch would make the batch overweight
+        # we need to close the batch, because all the items which come after
+        # this one will be same size or larger. This is the key part of the optimization.
+        elsif (current_batch_weight + weight_of_this_item) > weight_limit
+          batches << current_batch
+          current_batch = []
+          current_batch_weight = 0
+        end
+
+        # and then append the item to the current batch
+        current_batch_weight += weight_of_this_item
+        current_batch << item
+      end
+      batches << current_batch unless current_batch.empty?
+
+      batches
+    end
   end
 
   # Saves the receipt handles to batch-delete from the SQS queue

data/lib/sqewer/extensions/appsignal_wrapper.rb

@@ -4,7 +4,6 @@ module Sqewer
     # to Appsignal and to monitor performance. Will only activate
     # if the Appsignal gem is loaded within the current process and active.
     class AppsignalWrapper
-
       def self.new
         if defined?(Appsignal)
           super
@@ -36,7 +35,7 @@ module Sqewer
         transaction.set_action('%s#%s' % [serializer.class, 'unserialize'])
         transaction.request.params = {:sqs_message_body => msg_payload.to_s}
         if msg_attributes.key?('SentTimestamp')
-          transaction.set_queue_start = msg_attributes['SentTimestamp'].to_i
+          transaction.set_queue_start(msg_attributes['SentTimestamp'].to_i)
         end
 
         job_unserialized = yield

data/lib/sqewer/serializer.rb

@@ -13,7 +13,7 @@ class Sqewer::Serializer
     @instance ||= new
   end
 
-  AnonymousJobClass = Class.new(StandardError)
+  AnonymousJobClass = Class.new(Sqewer::Error)
 
   # Instantiate a Job object from a message body string. If the
   # returned result is `nil`, the job will be skipped.
@@ -33,18 +33,18 @@ class Sqewer::Serializer
     # use a default that will put us ahead of that execution deadline from the start.
     t = Time.now.to_i
     execute_after = job_ticket_hash.fetch(:_execute_after) { t - 5 }
-    
+
     job_params = job_ticket_hash.delete(:_job_params)
     job = if job_params.nil? || job_params.empty?
       job_class.new # no args
     else
       job_class.new(**job_params) # The rest of the message are keyword arguments for the job
     end
-    
-    # If the job is not up for execution now, wrap it with something that will 
+
+    # If the job is not up for execution now, wrap it with something that will
     # re-submit it for later execution when the run() method is called
     return ::Sqewer::Resubmit.new(job, execute_after) if execute_after > t
-    
+
     job
   end
 
@@ -65,7 +65,7 @@ class Sqewer::Serializer
     job_params = job.respond_to?(:to_h) ? job.to_h : nil
     job_ticket_hash = {_job_class: job_class_name, _job_params: job_params}
     job_ticket_hash[:_execute_after] = execute_after_timestamp.to_i if execute_after_timestamp
-    
+
     JSON.dump(job_ticket_hash)
   end
 end

data/lib/sqewer/simple_job.rb

@@ -5,8 +5,8 @@
 # * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
 # * inspect() will provide a sensible default string representation for logging
 module Sqewer::SimpleJob
-  UnknownJobAttribute = Class.new(StandardError)
-  MissingAttribute = Class.new(StandardError)
+  UnknownJobAttribute = Class.new(Sqewer::Error)
+  MissingAttribute = Class.new(Sqewer::Error)
 
   EQ_END = /(\w+)(\=)$/
 

data/lib/sqewer/submitter.rb

@@ -3,7 +3,10 @@
 # and the serializer (something that responds to `#serialize`) to
 # convert the job into the string that will be put in the queue.
 class Sqewer::Submitter < Struct.new(:connection, :serializer)
-  NotSqewerJob = Class.new(StandardError)
+  MAX_PERMITTED_MESSAGE_SIZE_BYTES = 256 * 1024
+
+  NotSqewerJob = Class.new(Sqewer::Error)
+  MessageTooLarge = Class.new(Sqewer::Error)
 
   # Returns a default Submitter, configured with the default connection
   # and the default serializer.
@@ -12,7 +15,7 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
   end
 
   def submit!(job, **kwargs_for_send)
-    raise NotSqewerJob.new("Submitted object is not a valid job: #{job.inspect}") unless job.respond_to?(:run)
+    validate_job_responds_to_run!(job)
     message_body = if delay_by_seconds = kwargs_for_send[:delay_seconds]
       clamped_delay = clamp_delay(delay_by_seconds)
       kwargs_for_send[:delay_seconds] = clamped_delay
@@ -21,11 +24,26 @@ class Sqewer::Submitter < Struct.new(:connection, :serializer)
     else
       serializer.serialize(job)
     end
+    validate_message_for_size!(message_body, job)
+
     connection.send_message(message_body, **kwargs_for_send)
   end
-  
+
   private
-  
+
+  def validate_job_responds_to_run!(job)
+    return if job.respond_to?(:run)
+    error_message = "Submitted object is not a valid job (does not respond to #run): #{job.inspect}"
+    raise NotSqewerJob.new(error_message)
+  end
+
+  def validate_message_for_size!(message_body, job)
+    actual_bytesize = message_body.bytesize
+    return if actual_bytesize <= MAX_PERMITTED_MESSAGE_SIZE_BYTES
+    error_message = "Job #{job.inspect} serialized to a message which was too large (#{actual_bytesize} bytes)"
+    raise MessageTooLarge.new(error_message)
+  end
+
   def clamp_delay(delay)
     [1, 899, delay].sort[1]
   end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '6.2.1'
+  VERSION = '6.5.0'
 end

data/lib/sqewer/worker.rb

@@ -48,7 +48,7 @@ class Sqewer::Worker
   #
   # @param connection[Sqewer::Connection] the object that handles polling and submitting
   # @param serializer[#serialize, #unserialize] the serializer/unserializer for the jobs
-  # @param execution_context_class[Class] the class for the execution context (will be instantiated by 
+  # @param execution_context_class[Class] the class for the execution context (will be instantiated by
   # the worker for each job execution)
   # @param submitter_class[Class] the class used for submitting jobs (will be instantiated by the worker for each job execution)
   # @param middleware_stack[Sqewer::MiddlewareStack] the middleware stack that is going to be used
@@ -212,17 +212,21 @@ class Sqewer::Worker
     submitter = submitter_class.new(box, serializer)
     context = execution_context_class.new(submitter, {'logger' => logger})
 
-    t = Time.now
+    t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     middleware_stack.around_execution(job, context) do
       job.method(:run).arity.zero? ? job.run : job.run(context)
+      # delete_message will enqueue the message for deletion,
+      # but when flush! is called _first_ the messages pending will be
+      # delivered to the queue, THEN all the deletes are going to be performed.
+      # So it is safe to call delete here first - the delete won't get to the queue
+      # if flush! fails to spool pending messages.
+      box.delete_message(message.receipt_handle)
+      n_flushed = box.flush!
+      logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed > 0
     end
-    box.delete_message(message.receipt_handle)
 
-    delta = Time.now - t
+    delta = Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
     logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
-  ensure
-    n_flushed = box.flush!
-    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
   end
 
   def take_and_execute
@@ -235,35 +239,4 @@ class Sqewer::Worker
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..64], e.class, e.message] }
     e.backtrace.each { |s| @logger.debug{"\t#{s}"} }
   end
-
-  def perform(message)
-    # Create a messagebox that buffers all the calls to Connection, so that
-    # we can send out those commands in one go (without interfering with senders
-    # on other threads, as it seems the Aws::SQS::Client is not entirely
-    # thread-safe - or at least not it's HTTP client part).
-    box = Sqewer::ConnectionMessagebox.new(connection)
-
-    job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body, message.attributes) do
-      serializer.unserialize(message.body)
-    end
-    return unless job
-
-    submitter = submitter_class.new(box, serializer)
-    context = execution_context_class.new(submitter, {'logger' => logger})
-
-    t = Time.now
-    middleware_stack.around_execution(job, context) do
-      job.method(:run).arity.zero? ? job.run : job.run(context)
-    end
-
-    # Perform two flushes, one for any possible jobs the job has spawned,
-    # and one for the job delete afterwards
-    box.delete_message(message.receipt_handle)
-
-    delta = Time.now - t
-    logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
-  ensure
-    n_flushed = box.flush!
-    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
-  end
 end

data/sqewer.gemspec

@@ -6,8 +6,8 @@ require "sqewer/version"
 Gem::Specification.new do |spec|
   spec.name          = "sqewer"
   spec.version       = Sqewer::VERSION
-  spec.authors       = ["Julik Tarkhanov"]
-  spec.email         = ["me@julik.nl"]
+  spec.authors       = ["Julik Tarkhanov", "Andrei Horak"]
+  spec.email         = ["me@julik.nl", "linkyndy@gmail.com"]
 
   spec.summary       = %q{Process jobs from SQS}
   spec.description   = %q{A full-featured library for all them SQS worker needs}
@@ -34,11 +34,11 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "very_tiny_state_machine"
   spec.add_runtime_dependency "ks"
   spec.add_runtime_dependency "retriable"
-  
+
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  
+
   # The Rails deps can be relaxed, they are specified more exactly in the gemfiles/
   # for testing against a specific Rails version
   spec.add_development_dependency "activerecord", ">= 4.2"
@@ -48,4 +48,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "sqlite3"
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "appsignal", '~> 2'
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 6.2.1
+  version: 6.5.0
 platform: ruby
 authors:
 - Julik Tarkhanov
+- Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-11 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -100,14 +101,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -206,9 +207,24 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: appsignal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: A full-featured library for all them SQS worker needs
 email:
 - me@julik.nl
+- linkyndy@gmail.com
 executables:
 - sqewer
 - sqewer_rails
@@ -269,8 +285,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Process jobs from SQS