hermes_messenger_of_the_gods 2.1.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb74386f4dbe4a881d0a8bbf122dc30ffc391d2ea00ae111799fef01e08cb452
-  data.tar.gz: f4b0dea8e4daedb15b2ee0725afbb2b9d7fefb2509a73be5815e1faa12e40dcb
+  metadata.gz: f14fead382b4de9c274888ee9c417f5b7e500846e118b9e655af25b182164184
+  data.tar.gz: 47edc08f7f760c868ae6c2e2443d4f7138ce23c4b260ae9a288e2b1c5d73f4a4
 SHA512:
-  metadata.gz: bfc97f115f564ce0d9731e093b14eb331dec9fef32567f80d0ee0986ce7d62eb5c330fe5c644900e006adf09d3dfc84ed066685b5c4b7be0b4020099030ca706
-  data.tar.gz: 0a28e656fedf9e14b572d543b2bf8abd945800b07fa723ab84db0224bb436afc6868aced15dccebf2e5558d35a357d26d2afcb370bf6f15d22f884569a3d18ce
+  metadata.gz: daf9a9781dae2a7243c5b98f06d4a929526c2430d29e52a328d5a94c045a7e016c59844dbf72eed9d773da91f396e0c51efc5d1af6d307b69f2d8940af36084e
+  data.tar.gz: 2e6e02d9a36ce06a0df77e7b732fafbb023cad0e3b0a33e107522b7d798ed9363e655e9c96da92c5a9af2f4961cfe460f33e6771a4bea671fa7998c9d1967801

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test
+
+on: push
+
+jobs:
+  ruby-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        rubyimage: ["getterminus/ruby-docker-images:3.0-je-20210920", "getterminus/ruby-docker-images:2.7-je-20210920",]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.rubyimage }}
+    steps:
+      - name: Install SSH Key
+        uses: webfactory/ssh-agent@v0.5.2
+        with:
+          ssh-private-key: ${{ secrets.GH_ACTIONS_SSH_PRIVATE_KEY }}
+
+      - name: Pull
+        uses: actions/checkout@v2
+        with:
+          path: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+
+      - name: Run Test
+        working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+        run: >
+          bash -c "
+          mkdir -p /coverage;
+          bundle --local;
+          COVERALLS_NOISY=true bundle exec rspec --color --format documentation;"
+
+      - name: Push Coveralls
+        continue-on-error: true
+        env:
+          TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+        working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
+        run: >
+          bash -c "
+          wget -cq https://coveralls.io/coveralls-linux.tar.gz -O - | tar -xz;
+          ./coveralls -r $TOKEN -f /coverage/lcov.info;"

data/.gitignore

@@ -10,3 +10,4 @@
 # rspec failure tracking
 .rspec_status
 .bundle
+*.gem

data/.ruby-version

@@ -1 +1 @@
-2.6.1
+3.0.2

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 2.1.1
+
+Ruby 3.0 support

data/Gemfile

@@ -6,7 +6,8 @@ gemspec
 gem 'grpc'
 
 group :test do
-  gem "simplecov"
-  gem "webmock"
+  gem 'coveralls_reborn', '~> 0.22.0', require: false
   gem "codeclimate-test-reporter", "~> 1.0.0"
+  gem 'simplecov-lcov', '~> 0.8.0'
+  gem "webmock"
 end

data/README.md

@@ -1,7 +1,7 @@
 # HermesMessengerOfTheGods
 [![CircleCI](https://circleci.com/gh/GetTerminus/hermes_messenger_of_the_gods.svg?style=svg&circle-token=9f757798d1f2d06b93ee2f560c0689cc71f1f96a)](https://circleci.com/gh/GetTerminus/hermes_messenger_of_the_gods)
 [![Code Climate](https://codeclimate.com/repos/58d16440cf31c32d1000041c/badges/bf27bdcafb886574909d/gpa.svg)](https://codeclimate.com/repos/58d16440cf31c32d1000041c/feed)
-[![Test Coverage](https://codeclimate.com/repos/58d16440cf31c32d1000041c/badges/bf27bdcafb886574909d/coverage.svg)](https://codeclimate.com/repos/58d16440cf31c32d1000041c/coverage)
+[![Coverage Status](https://coveralls.io/repos/github/GetTerminus/hermes_messenger_of_the_gods/badge.svg?t=zVNSvv)](https://coveralls.io/github/GetTerminus/hermes_messenger_of_the_gods)
 [![Issue Count](https://codeclimate.com/repos/58d16440cf31c32d1000041c/badges/bf27bdcafb886574909d/issue_count.svg)](https://codeclimate.com/repos/58d16440cf31c32d1000041c/feed)
 
 ## Installation

data/hermes_messenger_of_the_gods.gemspec

@@ -34,12 +34,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-sns', '~> 1.0'
   spec.add_dependency 'aws-sdk-sqs', '~> 1.0'
   spec.add_dependency 'thor'
+  spec.add_dependency 'rexml'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'memory_profiler'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'simplecov', '~> 1.14'
   spec.add_development_dependency 'timecop'
 end

data/lib/hermes_messenger_of_the_gods/concerns/worker.rb

@@ -32,6 +32,9 @@ module HermesMessengerOfTheGods
         def work_off
           WorkerStatusServer.start!(worker: self) if health_check_enabled?
 
+          trap(:TERM) { endpoint.shutdown! }
+          trap(:INT) { endpoint.shutdown! }
+
           instrument(:starting)
           self.consecutive_failures = 0
           endpoint.work_off do |job, original_message|

data/lib/hermes_messenger_of_the_gods/endpoints/sqs.rb

@@ -10,21 +10,63 @@ module HermesMessengerOfTheGods
       VISIBILITY_EXTEND_DURATION  = 120
       VISIBILITY_EXTEND_FREQUENCY = 60
 
+      def initialize(*args)
+        super
+        @message_mux = Monitor.new
+      end
+
       def poller
         @poller ||= Aws::SQS::QueuePoller.new(endpoint)
       end
 
+      def inflight_messages
+        @message_mux.synchronize { @inflight_messages ||= [] }
+      end
+
+      def inflight_messages=(val)
+        @message_mux.synchronize { @inflight_messages = val }
+      end
+
+      def shutting_down?
+        @shutdown || false
+      end
+
+      # Basic Shutdown behavior:
+      #   Allow in-progress message to finish working.
+      #   Reset visbility timeout to all un-executed messages (from current message to end of array) to 0 so they move
+      #   to other works
+      #
+      #   Break from polling
+      def shutdown!
+        say Logger::INFO, 'Shutdown command received'
+        @shutdown = true
+      end
+
       def poll_options
         (options[:poll_options] || {}).merge(skip_delete: true)
       end
 
       def work_off(&blk)
-        poller.poll(poll_options) do |messages, _stats|
-          messages = Array.wrap(messages)
+        poller.before_request { |_stats| throw :stop_polling if shutting_down? }
 
-          working_messages(messages) do
-            completed = messages.select { |msg| work_message(msg, &blk) }
-            poller.delete_messages(completed) unless completed.empty?
+        poller.poll(poll_options) do |messages, _stats|
+          self.inflight_messages = messages = Array.wrap(messages)
+
+          working_messages do
+            completion_results = messages.group_by do |msg|
+              # We return false if we are shutting down so the messages are not deleted.
+              # It is also possible that this process has already releases these SQS messages
+              # back in to the queue so they may be picked up by another process.
+              #
+              # Work message returns true if the messager should be considered successful
+              shutting_down? ? :shutdown : work_message(msg, &blk)
+            end
+
+            poller.delete_messages(completion_results[true]) unless completion_results.fetch(true, []).empty?
+            # Messages skipped due to shutdowns get their visibility set back to 0 so they restart
+            # normal failed jobs will be left in queue until their visibility timeout expires to indicate a backoff
+            set_message_visibility(completion_results[:shutdown], 0) unless completion_results.fetch(:shutdown,
+                                                                                                     []).empty?
           end
         end
       end
@@ -54,9 +96,9 @@ module HermesMessengerOfTheGods
       def has_pending_work?
         data = queue_data.attributes
 
-        approximate_pending_messages = data["ApproximateNumberOfMessages"].to_i -
-          data["ApproximateNumberOfMessagesNotVisible"].to_i -
-          data["ApproximateNumberOfMessagesDelayed"].to_i
+        approximate_pending_messages = data['ApproximateNumberOfMessages'].to_i -
+                                       data['ApproximateNumberOfMessagesNotVisible'].to_i -
+                                       data['ApproximateNumberOfMessagesDelayed'].to_i
 
         # Just in case the math is off
         approximate_pending_messages > 0
@@ -77,38 +119,40 @@ module HermesMessengerOfTheGods
         message_body
       end
 
-      def working_messages(messages)
-        thread = start_visibility_update_thread(messages)
-
+      def working_messages
+        thread = start_visibility_update_thread
         yield
       ensure
         thread&.terminate
       end
 
-      def start_visibility_update_thread(messages)
+      def start_visibility_update_thread
         start_work_time = Time.now
         Thread.new do
           loop do
             new_time = (Time.now - start_work_time) + VISIBILITY_EXTEND_DURATION
-            queue.change_message_visibility_batch(
-              entries: messages.collect do |message|
-                {
-                  id: SecureRandom.uuid,
-                  receipt_handle: message.receipt_handle,
-                  visibility_timeout: new_time
-                }
-              end
-            )
-
+            set_message_visibility(inflight_messages, new_time)
             sleep VISIBILITY_EXTEND_FREQUENCY
           rescue StandardError => e
-            STDERR.puts 'Error received trying to extend visibility'
-            STDERR.puts e.message
+            warn 'Error received trying to extend visibility'
+            warn e.message
 
             raise
           end
         end
       end
+
+      def set_message_visibility(messages, new_time)
+        queue.change_message_visibility_batch(
+          entries: messages.collect do |message|
+            {
+              id: SecureRandom.uuid,
+              receipt_handle: message.receipt_handle,
+              visibility_timeout: new_time
+            }
+          end
+        )
+      end
     end
   end
 end

data/lib/hermes_messenger_of_the_gods/testing/array_endpoint.rb

@@ -15,6 +15,8 @@ module HermesMessengerOfTheGods
         end
       end
 
+      attr_accessor :on_receive
+
       def received
         self.class.received[endpoint]
       end
@@ -29,7 +31,9 @@ module HermesMessengerOfTheGods
 
       def do_transmit(msg, opts, raw_message = {})
         pub_opts = fetch_option(:publish_options, raw_message) || {}
-        received << { message: msg, options: opts.merge(pub_opts) }
+        new_msg = { message: msg, options: opts.merge(pub_opts) }
+        received << new_msg
+        on_receive.call(new_msg) if on_receive
         "Recorded msg ##{self.class.received[endpoint].length} for #{endpoint}"
       end
     end

data/lib/hermes_messenger_of_the_gods/version.rb

@@ -1,3 +1,3 @@
 module HermesMessengerOfTheGods
-  VERSION = '2.1.0'
+  VERSION = '2.3.0'
 end

data/packageGems.sh

@@ -4,7 +4,7 @@ docker run --rm -t -i \
   -v $HOME/.ssh:/root/.ssh \
   --mount type=bind,source="$(pwd)",target=/app \
   --workdir=/app/api \
-  getterminus/ruby-docker-images:2.6-je-20190204 \
+  getterminus/ruby-docker-images:2.7-je-20210920 \
     /bin/bash -c "
       bundle config specific_platform true && \
       bundle lock --add-platform x86_64-linux && \

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: hermes_messenger_of_the_gods
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Brian Malinconico
 - Stevie Vines
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-23 00:00:00.000000000 Z
+date: 2022-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -81,6 +81,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -151,20 +165,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -188,11 +188,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".drone.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile
@@ -221,54 +222,59 @@ files:
 - lib/hermes_messenger_of_the_gods/testing/rspec_helpers.rb
 - lib/hermes_messenger_of_the_gods/version.rb
 - packageGems.sh
-- vendor/cache/activemodel-6.1.1.gem
-- vendor/cache/activesupport-6.1.1.gem
-- vendor/cache/addressable-2.7.0.gem
-- vendor/cache/aws-eventstream-1.1.0.gem
-- vendor/cache/aws-partitions-1.417.0.gem
-- vendor/cache/aws-sdk-core-3.111.2.gem
-- vendor/cache/aws-sdk-sns-1.37.0.gem
-- vendor/cache/aws-sdk-sqs-1.35.0.gem
-- vendor/cache/aws-sigv4-1.2.2.gem
-- vendor/cache/codeclimate-test-reporter-1.0.9.gem
+- vendor/cache/activemodel-6.1.4.1.gem
+- vendor/cache/activesupport-6.1.4.1.gem
+- vendor/cache/addressable-2.8.0.gem
+- vendor/cache/aws-eventstream-1.2.0.gem
+- vendor/cache/aws-partitions-1.509.0.gem
+- vendor/cache/aws-sdk-core-3.121.1.gem
+- vendor/cache/aws-sdk-sns-1.45.0.gem
+- vendor/cache/aws-sdk-sqs-1.44.0.gem
+- vendor/cache/aws-sigv4-1.4.0.gem
+- vendor/cache/codeclimate-test-reporter-1.0.7.gem
 - vendor/cache/coderay-1.1.3.gem
-- vendor/cache/concurrent-ruby-1.1.8.gem
+- vendor/cache/concurrent-ruby-1.1.9.gem
+- vendor/cache/coveralls_reborn-0.22.0.gem
 - vendor/cache/crack-0.4.5.gem
 - vendor/cache/diff-lcs-1.4.4.gem
-- vendor/cache/docile-1.1.5.gem
-- vendor/cache/google-protobuf-3.14.0-universal-darwin.gem
-- vendor/cache/google-protobuf-3.14.0-x86_64-linux.gem
-- vendor/cache/googleapis-common-protos-types-1.0.5.gem
-- vendor/cache/grpc-1.35.0-universal-darwin.gem
-- vendor/cache/grpc-1.35.0-x86_64-linux.gem
+- vendor/cache/docile-1.4.0.gem
+- vendor/cache/google-protobuf-3.18.0-universal-darwin.gem
+- vendor/cache/google-protobuf-3.18.0-x86_64-linux.gem
+- vendor/cache/googleapis-common-protos-types-1.2.0.gem
+- vendor/cache/grpc-1.40.0-universal-darwin.gem
+- vendor/cache/grpc-1.40.0-x86_64-linux.gem
 - vendor/cache/hashdiff-1.0.1.gem
-- vendor/cache/i18n-1.8.7.gem
+- vendor/cache/i18n-1.8.10.gem
 - vendor/cache/jmespath-1.4.0.gem
-- vendor/cache/json-2.5.1.gem
 - vendor/cache/memory_profiler-1.0.0.gem
 - vendor/cache/method_source-1.0.0.gem
-- vendor/cache/minitest-5.14.3.gem
-- vendor/cache/pry-0.13.1.gem
+- vendor/cache/minitest-5.14.4.gem
+- vendor/cache/pry-0.14.1.gem
 - vendor/cache/public_suffix-4.0.6.gem
 - vendor/cache/rake-10.5.0.gem
-- vendor/cache/rexml-3.2.4.gem
+- vendor/cache/rexml-3.2.5.gem
 - vendor/cache/rspec-3.10.0.gem
 - vendor/cache/rspec-core-3.10.1.gem
 - vendor/cache/rspec-expectations-3.10.1.gem
-- vendor/cache/rspec-mocks-3.10.1.gem
-- vendor/cache/rspec-support-3.10.1.gem
-- vendor/cache/simplecov-0.13.0.gem
-- vendor/cache/simplecov-html-0.10.2.gem
+- vendor/cache/rspec-mocks-3.10.2.gem
+- vendor/cache/rspec-support-3.10.2.gem
+- vendor/cache/simplecov-0.21.2.gem
+- vendor/cache/simplecov-html-0.12.3.gem
+- vendor/cache/simplecov-lcov-0.8.0.gem
+- vendor/cache/simplecov_json_formatter-0.1.3.gem
+- vendor/cache/sync-0.5.0.gem
+- vendor/cache/term-ansicolor-1.7.1.gem
 - vendor/cache/thor-1.1.0.gem
-- vendor/cache/timecop-0.9.2.gem
+- vendor/cache/timecop-0.9.4.gem
+- vendor/cache/tins-1.29.1.gem
 - vendor/cache/tzinfo-2.0.4.gem
-- vendor/cache/webmock-3.11.1.gem
+- vendor/cache/webmock-3.14.0.gem
 - vendor/cache/zeitwerk-2.4.2.gem
-homepage: 
+homepage:
 licenses: []
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -283,8 +289,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Create and receive messages like a god!
 test_files: []

data/.drone.yml

@@ -1,9 +0,0 @@
-
-pipeline:
-  test:
-    image: getterminus/ruby-ci-image:2.6-je-20190205
-    group: test
-    secrets: [ CODECOV_TOKEN ]
-    commands:
-      - bundle --local
-      - bundle exec rspec