hermes_messenger_of_the_gods 2.1.1 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9fb8caf1a8299a378495d5f33f4be6e226725ceb3d9a2bfca2ee301ddfa74b2
-  data.tar.gz: 20ae2dedfc73f959bbb8705c398a293bb105d4f3362e1b33c62a0bf00705c7e4
+  metadata.gz: 604daff161d76c0ebf41ab253989906c75c519aba9460282fc0644a3b3ed1cad
+  data.tar.gz: 201cda28475512177767a542e0c13f15780431dbc8cfd45f324425bccecc090c
 SHA512:
-  metadata.gz: 82d1ef250df05d7630816d9c02fb1cc35024d101d82263b213762336110daaef10cc33d6b84ced72eb3a712981e75ffc1ad5aa0327081a8c46ebfc1cd60ffb86
-  data.tar.gz: 0b618c1608e636d4afb210fe3cafa8bd0786a8e0c74da0f14b5ad10067c0b6293f2b495d283c9c016abcc2fb63b3dfbeb56d69d5b8fac28405e108eb30f3d66d
+  metadata.gz: 607884c3e2d370128952cd2a832686dbaeade734ae3580143a677715d388faf47dc620049252e07230c7f920e64db09a4b30b2c8ee08c85a4e5a2a667bd7595d
+  data.tar.gz: a627e353104644dce6b85e0fb8dee0e6fe821742c2b29c3662140241c968cf0335991fe146143fd777460bd22b086f03dc1852249cdf64836f2d5b0b8816b071

data/.github/workflows/test.yml

@@ -31,6 +31,7 @@ jobs:
           COVERALLS_NOISY=true bundle exec rspec --color --format documentation;"
 
       - name: Push Coveralls
+        continue-on-error: true
         env:
           TOKEN: ${{ secrets.COVERALLS_TOKEN }}
         working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}

data/.gitignore

@@ -10,3 +10,4 @@
 # rspec failure tracking
 .rspec_status
 .bundle
+*.gem

data/lib/hermes_messenger_of_the_gods/concerns/message.rb

@@ -61,7 +61,7 @@ module HermesMessengerOfTheGods
           endpoints.collect do |ep_name, endpoint|
             next if targeted_endpoints && !targeted_endpoints.include?(ep_name)
 
-            Thread.new do
+            begin
               endpoint.dispatch!(self, endpoint_args) unless HermesMessengerOfTheGods.config.stub_dispatch
               register_success(ep_name, endpoint.result)
             rescue StandardError => e
@@ -70,7 +70,7 @@ module HermesMessengerOfTheGods
             ensure
               endpoint.teardown
             end
-          end.compact.map(&:join)
+          end
 
           unless dispatch_errors.empty?
             klass = if successes.empty?

data/lib/hermes_messenger_of_the_gods/concerns/worker.rb

@@ -32,6 +32,9 @@ module HermesMessengerOfTheGods
         def work_off
           WorkerStatusServer.start!(worker: self) if health_check_enabled?
 
+          trap(:TERM) { endpoint.shutdown! }
+          trap(:INT) { endpoint.shutdown! }
+
           instrument(:starting)
           self.consecutive_failures = 0
           endpoint.work_off do |job, original_message|

data/lib/hermes_messenger_of_the_gods/endpoints/sqs.rb

@@ -10,21 +10,63 @@ module HermesMessengerOfTheGods
       VISIBILITY_EXTEND_DURATION  = 120
       VISIBILITY_EXTEND_FREQUENCY = 60
 
+      def initialize(*args)
+        super
+        @message_mux = Monitor.new
+      end
+
       def poller
         @poller ||= Aws::SQS::QueuePoller.new(endpoint)
       end
 
+      def inflight_messages
+        @message_mux.synchronize { @inflight_messages ||= [] }
+      end
+
+      def inflight_messages=(val)
+        @message_mux.synchronize { @inflight_messages = val }
+      end
+
+      def shutting_down?
+        @shutdown || false
+      end
+
+      # Basic Shutdown behavior:
+      #   Allow in-progress message to finish working.
+      #   Reset visbility timeout to all un-executed messages (from current message to end of array) to 0 so they move
+      #   to other works
+      #
+      #   Break from polling
+      def shutdown!
+        say Logger::INFO, 'Shutdown command received'
+        @shutdown = true
+      end
+
       def poll_options
         (options[:poll_options] || {}).merge(skip_delete: true)
       end
 
       def work_off(&blk)
-        poller.poll(poll_options) do |messages, _stats|
-          messages = Array.wrap(messages)
+        poller.before_request { |_stats| throw :stop_polling if shutting_down? }
 
-          working_messages(messages) do
-            completed = messages.select { |msg| work_message(msg, &blk) }
-            poller.delete_messages(completed) unless completed.empty?
+        poller.poll(poll_options) do |messages, _stats|
+          self.inflight_messages = messages = Array.wrap(messages)
+
+          working_messages do
+            completion_results = messages.group_by do |msg|
+              # We return false if we are shutting down so the messages are not deleted.
+              # It is also possible that this process has already releases these SQS messages
+              # back in to the queue so they may be picked up by another process.
+              #
+              # Work message returns true if the messager should be considered successful
+              shutting_down? ? :shutdown : work_message(msg, &blk)
+            end
+
+            poller.delete_messages(completion_results[true]) unless completion_results.fetch(true, []).empty?
+            # Messages skipped due to shutdowns get their visibility set back to 0 so they restart
+            # normal failed jobs will be left in queue until their visibility timeout expires to indicate a backoff
+            set_message_visibility(completion_results[:shutdown], 0) unless completion_results.fetch(:shutdown,
+                                                                                                     []).empty?
           end
         end
       end
@@ -54,9 +96,9 @@ module HermesMessengerOfTheGods
       def has_pending_work?
         data = queue_data.attributes
 
-        approximate_pending_messages = data["ApproximateNumberOfMessages"].to_i -
-          data["ApproximateNumberOfMessagesNotVisible"].to_i -
-          data["ApproximateNumberOfMessagesDelayed"].to_i
+        approximate_pending_messages = data['ApproximateNumberOfMessages'].to_i -
+                                       data['ApproximateNumberOfMessagesNotVisible'].to_i -
+                                       data['ApproximateNumberOfMessagesDelayed'].to_i
 
         # Just in case the math is off
         approximate_pending_messages > 0
@@ -77,38 +119,40 @@ module HermesMessengerOfTheGods
         message_body
       end
 
-      def working_messages(messages)
-        thread = start_visibility_update_thread(messages)
-
+      def working_messages
+        thread = start_visibility_update_thread
         yield
       ensure
         thread&.terminate
       end
 
-      def start_visibility_update_thread(messages)
+      def start_visibility_update_thread
         start_work_time = Time.now
         Thread.new do
           loop do
             new_time = (Time.now - start_work_time) + VISIBILITY_EXTEND_DURATION
-            queue.change_message_visibility_batch(
-              entries: messages.collect do |message|
-                {
-                  id: SecureRandom.uuid,
-                  receipt_handle: message.receipt_handle,
-                  visibility_timeout: new_time
-                }
-              end
-            )
-
+            set_message_visibility(inflight_messages, new_time)
             sleep VISIBILITY_EXTEND_FREQUENCY
           rescue StandardError => e
-            STDERR.puts 'Error received trying to extend visibility'
-            STDERR.puts e.message
+            warn 'Error received trying to extend visibility'
+            warn e.message
 
             raise
           end
         end
       end
+
+      def set_message_visibility(messages, new_time)
+        queue.change_message_visibility_batch(
+          entries: messages.collect do |message|
+            {
+              id: SecureRandom.uuid,
+              receipt_handle: message.receipt_handle,
+              visibility_timeout: new_time
+            }
+          end
+        )
+      end
     end
   end
 end

data/lib/hermes_messenger_of_the_gods/testing/array_endpoint.rb

@@ -15,6 +15,8 @@ module HermesMessengerOfTheGods
         end
       end
 
+      attr_accessor :on_receive
+
       def received
         self.class.received[endpoint]
       end
@@ -29,7 +31,9 @@ module HermesMessengerOfTheGods
 
       def do_transmit(msg, opts, raw_message = {})
         pub_opts = fetch_option(:publish_options, raw_message) || {}
-        received << { message: msg, options: opts.merge(pub_opts) }
+        new_msg = { message: msg, options: opts.merge(pub_opts) }
+        received << new_msg
+        on_receive.call(new_msg) if on_receive
         "Recorded msg ##{self.class.received[endpoint].length} for #{endpoint}"
       end
     end

data/lib/hermes_messenger_of_the_gods/version.rb

@@ -1,3 +1,3 @@
 module HermesMessengerOfTheGods
-  VERSION = '2.1.1'
+  VERSION = '2.3.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermes_messenger_of_the_gods
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.3.1
 platform: ruby
 authors:
 - Brian Malinconico
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-21 00:00:00.000000000 Z
+date: 2022-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -226,8 +226,8 @@ files:
 - vendor/cache/activesupport-6.1.4.1.gem
 - vendor/cache/addressable-2.8.0.gem
 - vendor/cache/aws-eventstream-1.2.0.gem
-- vendor/cache/aws-partitions-1.503.0.gem
-- vendor/cache/aws-sdk-core-3.121.0.gem
+- vendor/cache/aws-partitions-1.509.0.gem
+- vendor/cache/aws-sdk-core-3.121.1.gem
 - vendor/cache/aws-sdk-sns-1.45.0.gem
 - vendor/cache/aws-sdk-sqs-1.44.0.gem
 - vendor/cache/aws-sigv4-1.4.0.gem
@@ -238,9 +238,11 @@ files:
 - vendor/cache/crack-0.4.5.gem
 - vendor/cache/diff-lcs-1.4.4.gem
 - vendor/cache/docile-1.4.0.gem
-- vendor/cache/google-protobuf-3.17.3-x86_64-linux.gem
-- vendor/cache/googleapis-common-protos-types-1.1.0.gem
-- vendor/cache/grpc-1.38.0-x86_64-linux.gem
+- vendor/cache/google-protobuf-3.18.0-universal-darwin.gem
+- vendor/cache/google-protobuf-3.18.0-x86_64-linux.gem
+- vendor/cache/googleapis-common-protos-types-1.2.0.gem
+- vendor/cache/grpc-1.40.0-universal-darwin.gem
+- vendor/cache/grpc-1.40.0-x86_64-linux.gem
 - vendor/cache/hashdiff-1.0.1.gem
 - vendor/cache/i18n-1.8.10.gem
 - vendor/cache/jmespath-1.4.0.gem
@@ -287,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.27
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Create and receive messages like a god!