hermes_messenger_of_the_gods 2.4.2 → 3.0.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3ad7cd6b63d696cd8e765e69def47efa4496c43e1573b9dc80a966f770df0da
-  data.tar.gz: 627776d0de4ac041fc22b6cb35c3fe2f8ee2d873375599c6136b54d00bc46d29
+  metadata.gz: 031b674be25f17c3c598825717fd068d52dba3130baaf7fee5780917dcf6fe18
+  data.tar.gz: 64909cbd8561ce4b82e7cfc3d3ddc38bc9e283b85ff99ce79dc46babd055c9eb
 SHA512:
-  metadata.gz: 56035ce71cfc68f0e593a6ffc81236ef8512ab186f459f1b6d29f6322e161862951ba4fe82210cdee8ab189023ef4ef7fd31452f6f4df822be1d39578cfc79e9
-  data.tar.gz: 74a55d8d43193c40f7e0e345a8dfe81fc72c3674202ed29c997259f30acf8ebda468e6b920fe88e8dbc28b5018eabf2e208cf23ae533c5f592442e60c69941af
+  metadata.gz: 3f829456b20fb7ed1cef756c6f8ac38fb9e214c4fa7e39bf562d9fbbda8051b660b109211e10da747590ef70e56fa2160525e59b01c2d090c01cce8d837246e1
+  data.tar.gz: 471bafec3eed05e291dee33d3ed73ed287a289519747264839e881ec439c777a42c498b1ae4da3d53554846bdfd06f1a298975edab96196a16ff9090786f0132

data/.github/workflows/test.yml

@@ -7,10 +7,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        rubyimage: ["getterminus/ruby-docker-images:3.0-je-20210920", "getterminus/ruby-docker-images:2.7-je-20210920",]
+        rubyVersion: ["2.7.8", "3.0.4", "3.1.2", "3.2.2"]
     runs-on: ubuntu-latest
-    container:
-      image: ${{ matrix.rubyimage }}
     steps:
       - name: Install SSH Key
         uses: webfactory/ssh-agent@v0.5.2
@@ -21,7 +19,11 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: ${{ github.workspace }}/src/github.com/${{ github.repository }}
-
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.rubyVersion }}
+          bundler-cache: true
+          cache-version: "${{ matrix.rubyVersion }}-1"
       - name: Run Test
         working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }}
         run: >

data/README.md

@@ -105,51 +105,13 @@ class Foo
 
   attr_accessor :name
   validates :name, presence: true
-  self.endpoints = {
-    default: sns_endpoint("sns:arn", jitter: false),
-    other: sns_endpoint('...')
-  }
+  self.endpoints = sns_endpoint("sns:arn", jitter: false)
 end
 
 Foo.new(name: 'omg').dispatch!
 # => true
 ```
 
-Dispatches to each endpoint is done in a thread to allow for better performance.
-
-##### Targeting an endpoint
-You may target only a specific endpoint by passing the `endpoints: ['']` option
-during message creation. Using the message above:
-`Foo.new(name: 'omg', endpoints: [:other]).dispatch!` would dispatch only to
-the other endpoint.
-
-
-#### Async message dispatch
-All messages contain a `dispatch_async(!)` option in addition to typical deferral.
-Using an async dispatch runs the dispatch in a thread. Since dispatches tend to
-be I/O heavy this is an ideal option if you do not care about capturing the
-return information or error status. You can check to determine how many messages
-are in-flight using the helper method
-`HermesMessengerOfTheGods.async_dispatches_in_progress` in the testing helpers
-a method named `wait_for_async_dispatches` is avaialbe which will wait for up
-to 5 seconds for all messages to be "delivered".
-
-### Endpoint responses
-When an endpoint reports successful transmission, the response is recorded in
-the `message.successes` hash. This hash is keyed to correspond to the
-configuration `endpoints` hash.
-
-### Endpoint Failures
-When an endpoint fails (raises an error) the final error raised will be stored
-in the `message.dispatch_errors` hash. This hash is keyed to correspond to the
-`endponts` configuration hash
-
-Failure Exceptions:
-* When all endpoints fail a `HermesMessengerOfTheGods::MessageDispatchTotalFailure` is raised
-* When only some endpoints fail a `HermesMessengerOfTheGods::MessageDispatchPartialFailure` is raised
-
-Both of these exepctions inherit from `HermesMessengerOfTheGods::MessageDispatchFailed`
-
 ##  The Worker
 The worker provides a generic class to provide access for reading from
 endpoints. Only some endpoints actually provide the ability to read messages.
@@ -218,9 +180,7 @@ class MessageType
 end
 
 class AwesomeWorker < HermesMessengerOfTheGods::Worker
-  self.endpoint = {
-    default: sns_endpoint("sns::arn", jitter: false)
-  }
+  self.endpoint = sns_endpoint("sns::arn", jitter: false)
   self.deserialize_with = MessageType
   self.deserialize_method = :omg_dude
 end
@@ -247,7 +207,7 @@ and the message into a single class.
 class MonoMessageWorker
   include HermesMessengerOftheGods::Concerns::MonoMessage
 
-  self.endpoints = { default: sqs_endpoint("arn") } # Only 1 endpoint, options are supported
+  self.endpoint = sqs_endpoint("arn")
 
   def perform
     # Do magic Here
@@ -260,8 +220,8 @@ You may then start the message execution with:
 `./bin/fly_hermes start --pool=MonoMessageWorker--1`
 
 
-## Protocol Buffers
-Hermes supports using ProtoBuffers over the wire using the
+## rotocol Buffers
+Hermes supports using rotoBuffers over the wire using the
 `HermesMessengerOfTheGods::Concerns::GrpcProtobuf` mixin. Currently only JSON
 encoding is allowed.
 
@@ -371,36 +331,6 @@ array.
   * `:jsonify` - converts the trasnmitted data to json prior to transmission
   * `:from_sns` - reads and converts from JSON the serialized Message key
 
-## Instrumentation
-HMOTG Uses `ActiveSupport::Notifications` to provide internal instrumentation
-
-All messages emited include the emitter in their payload if possible:
-* Messages prefixed with `hermes_messenger_of_the_gods.worker` include the
-  `worker` key.
-* Messages prefixed with `hermes_messenger_of_the_gods.message` include the
-  `message` key.
-* Messages prefixed with `hermes_messenger_of_the_gods.endpoint` include the
-  `endpoint` key.
-
-**NOTE:** All messages are prefixed with `hermes_messenger_of_the_gods` but it
-was omitted here for brevity.
-
-| Message                     | Is Timed | Description                                                          | Payload Objects                                                                                                                                                                            |
-| ------                      | -------- | -----------                                                          | ---------------                                                                                                                                                                            |
-| `worker.starting`           |          | Called when the worker starts up                                     |                                                                                                                                                                                            |
-| `worker.starting_job`       |          | Called before work begins on a job                                   | `job` - the deserialize job being run                                                                                                                                                      |
-| `worker.run_job`            | ✓        | Measured the execution of the message                                | `job` - the deserialize job being run                                                                                                                                                      |
-| `worker.failure`            |          | Invoked when a message raises an error during execution              | `job` - the deserialize job being run <br> `error` - the exception raised                                                                                                                  |
-| `worker.fatal_error`        |          | Raised when an unhandled exception causes the worker to die          | `exception` - The exceptoin that caused the worker to die                                                                                                                                  |
-| `worker.deserialization`    | ✓        | Called during message deserialization off the endpoint               | `job` - the raw job received from the endpoint                                                                                                                                             |
-| `message.dispatch`          | ✓        | Called when a message complets the dispatch process                  |                                                                                                                                                                                            |
-| `message.dispatch_failure`  |          | Called once for every failed enpoint                                 | `exception` - the error raised <br> `endpoint_name` - the name of the failed endpoint                                                                                                      |
-| `endpoint.dispatch`         | ✓        | Called around the actual dispatch for each endpoint                  |                                                                                                                                                                                            |
-| `endpoint.dispatch_failure` |          | Called each time the endpoint fails to dispatch                      | `try` - The number of times the endpoint has tried to dispatch so far. First call is 1. <br> `exception` - The error that was raised causing the failure                                   |
-| `endpoint.final_failure`    |          | Called on the last exception to before the endpoint gives up         | `try` - The number of times the endpoint has tried to dispatch so far. At this point `try` equals the `:retries` config . <br> `exception` - The error that was raised causing the failure |
-| `endpoint.read_failure`     |          | An inbound message could not be handled (Malformed JSON for example) | `exception` - The unhandled error                                                                                                                                                          |
-
-
 ## Global Configuration
 When configuring HMOTG you can set global configuration opions like so:
 
@@ -433,24 +363,17 @@ parameter, or a block which is expected to return the message.
 Additionally the `say` method is provided which takes the desired level contsant
 as the first parameter. `say(Logger::ERROR, "foo")`
 
-### Outputters
-Hermes ships and attaches a Basic logger for by default. The primary warning is
-that the basic output will default to the `inspect` output of your message. If
-your message contains a significant number of, or complex, instance variables
-There may be more info logged then is necessary to identify the message.
-
-As such you can overwrite the `inspect` method, or create the custom
-`to_log_s` method which will be used instead.
-
-Example regular output:
-```
-I, [date stamp]  INFO -- Worker WorkerClass::0(pid: 8): Starting Job <MessageClass: {:instance_var_1 => 123, :instance_var_2=5678}>
-I, [date stamp]  INFO -- Worker WorkerClass::0(pid: 8): Finished Job <MessageClass: {:instance_var_1 => 123, :instance_var_2=5678}>
-```
-
-Don't want any output such as during a test suite, or want to write your own?
-You can turn off the Basic outputter via
-`HermesMessengerOfTheGods::Output::Basic.unsubscribe!`
+## Upgrading
+### V2 -> V3
+Upgrading to V2 to V3 includes the following breaking changes:
+
+1) The worker / message `endpoints` setting has been changed to just `endpoint` and takes a single endpoint definition.
+2) Outputters are no longer used. You need to log anything you want or need.
+3) There is no longer instrumentation outputted using ActiveSupport::Notifications
+4) Async Dispatches are now removed (Test helper `wait_for_async_dispatches` also removed)
+5) Partial and Total failure errors are no longer used, the superclass MessageDispatchFailed is used.
+6) Dispatch Errors / Successes are no longer included
+7) Callback system has been removed
 
 ## Development
 

data/lib/hermes_messenger_of_the_gods/concerns/message.rb

@@ -4,26 +4,19 @@ require 'active_model'
 require 'active_support'
 require 'active_support/core_ext/hash'
 
-require 'hermes_messenger_of_the_gods/concerns/base'
-
 module HermesMessengerOfTheGods
   module Concerns
     module Message
       extend ActiveSupport::Concern
 
       include ActiveModel::Model
-      include HermesMessengerOfTheGods::Concerns::Base
-
-      attr_accessor :targeted_endpoints, :original_message, :source_endpoint
+      include EndpointBuilder::Helpers
+      include LoggingHelpers
 
-      def initialize(options = {})
-        @monitor = Monitor.new
-        self.targeted_endpoints = options.delete(:endpoints)
-        super
-      end
+      attr_accessor :original_message, :source_endpoint
 
       def validate!
-        raise HermesMessengerOfTheGods::ValidationError, self unless valid?
+        raise HermesMessengerOfTheGods::ValidationError.new(self) unless valid?
       end
 
       def dispatch
@@ -32,24 +25,6 @@ module HermesMessengerOfTheGods
         false
       end
 
-      def dispatch_async
-        HermesMessengerOfTheGods.increment_async_dispatches_in_progress
-        Thread.new do
-          dispatch
-        ensure
-          HermesMessengerOfTheGods.decrement_async_dispatches_in_progress
-        end
-      end
-
-      def dispatch_async!
-        HermesMessengerOfTheGods.increment_async_dispatches_in_progress
-        Thread.new do
-          dispatch!
-        ensure
-          HermesMessengerOfTheGods.decrement_async_dispatches_in_progress
-        end
-      end
-
       def retry_at(future_time_or_seconds_in_future)
         raise 'unable to set visiblity' if source_endpoint.nil? || original_message.nil?
         raise "endpoint type doesn't support setting execution time" unless source_endpoint.respond_to?(:set_reexecution_time)
@@ -58,60 +33,20 @@ module HermesMessengerOfTheGods
       end
 
       def dispatch!(endpoint_args: {})
-        run_callbacks :dispatch do
-          validate!
-
-          endpoints.collect do |ep_name, endpoint|
-            next if targeted_endpoints && !targeted_endpoints.include?(ep_name)
-
-            begin
-              endpoint.dispatch!(self, endpoint_args) unless HermesMessengerOfTheGods.config.stub_dispatch
-              register_success(ep_name, endpoint.result)
-            rescue StandardError => e
-              say_error(e)
-              register_failure(ep_name, e)
-            ensure
-              endpoint.teardown
-            end
-          end
+        validate!
 
-          unless dispatch_errors.empty?
-            klass = if successes.empty?
-                      HermesMessengerOfTheGods::MessageDispatchTotalFailure
-                    else
-                      HermesMessengerOfTheGods::MessageDispatchPartialFailure
-                    end
-
-            ex = klass.new
-            ex.exceptions = dispatch_errors
-            raise ex
-          end
+        begin
+          endpoint.dispatch!(self, endpoint_args) unless HermesMessengerOfTheGods.config.stub_dispatch
+        rescue StandardError => e
+          say_error(e)
+          raise HermesMessengerOfTheGods::MessageDispatchFailed, e.message
+        ensure
+          endpoint.teardown
         end
 
         true
       end
 
-      def register_failure(ep_name, error)
-        @monitor.synchronize do
-          instrument(:dispatch_failure, endpoint_name: ep_name, exception: error)
-          dispatch_errors[ep_name] = error
-        end
-      end
-
-      def register_success(ep_name, return_value)
-        @monitor.synchronize do
-          successes[ep_name] = return_value
-        end
-      end
-
-      def dispatch_errors
-        @dispatch_errors ||= {}
-      end
-
-      def successes
-        @successes ||= {}
-      end
-
       def _build_for_transmission
         to_message
       end
@@ -131,14 +66,16 @@ module HermesMessengerOfTheGods
       end
 
       included do
-        class_attribute :_defined_attributes, :_endpoints, :_circuit_breaker_errors, :_max_consecutive_failures
-
-        define_model_callbacks :dispatch
-
-        around_dispatch { |_, blk| instrument(:dispatch, &blk) }
+        class_attribute :_defined_attributes, :_endpoint, :_circuit_breaker_errors, :_max_consecutive_failures
       end
 
       class_methods do
+        def send_messages(messages, options = {})
+          raise ArgumentError, "All messages must be #{self.class}" unless messages.all?{|m| m.is_a?(self) }
+          messages.map(&:validate!)
+          messages.first.endpoint.bulk_dispatch!(messages, options)
+        end
+
         def from_message(attrs = {}, &blk)
           attrs = attrs.slice(*_defined_attributes.map(&:to_s))
           new(attrs, &blk)
@@ -150,8 +87,8 @@ module HermesMessengerOfTheGods
           super
         end
 
-        def endpoints
-          _endpoints
+        def endpoint
+          _endpoint
         end
 
         def circuit_breaker_errors
@@ -185,10 +122,8 @@ module HermesMessengerOfTheGods
           self._circuit_breaker_errors = val
         end
 
-        def endpoints=(val)
-          raise 'Endpoints expects a hash {endpoint_name: endpoint_handler}' unless val.is_a?(Hash)
-
-          self._endpoints = val
+        def endpoint=(val)
+          self._endpoint = val
         end
       end
     end

data/lib/hermes_messenger_of_the_gods/concerns/mono_message.rb

@@ -10,12 +10,14 @@ module HermesMessengerOfTheGods
     module MonoMessage
       extend ActiveSupport::Concern
 
-      include HermesMessengerOfTheGods::Concerns::Message
+      def self.included(other)
+        other.include HermesMessengerOfTheGods::Concerns::Message
+      end
 
       class_methods do
-        def endpoints=(*args)
+        def endpoint=(*args)
           super
-          worker_klass.endpoints = endpoints
+          worker_klass.endpoint = endpoint
         end
 
         def circuit_breaker_errors=(*args)
@@ -40,7 +42,7 @@ module HermesMessengerOfTheGods
           me = self
           @worker_klass ||= Class.new do
             include HermesMessengerOfTheGods::Concerns::Worker
-            self.endpoints = me.endpoints
+            self.endpoint = me.endpoint
             self.circuit_breaker_errors = me.circuit_breaker_errors
             self.max_consecutive_failures = me.max_consecutive_failures
             self.deserialize_with = me

data/lib/hermes_messenger_of_the_gods/concerns/worker.rb

@@ -2,7 +2,6 @@
 
 require 'active_support'
 
-require_relative './base'
 require_relative '../status_server'
 
 module HermesMessengerOfTheGods
@@ -10,7 +9,8 @@ module HermesMessengerOfTheGods
     module Worker
       extend ActiveSupport::Concern
 
-      include HermesMessengerOfTheGods::Concerns::Base
+      include EndpointBuilder::Helpers
+      include LoggingHelpers
 
       included do
         attr_writer :name
@@ -37,7 +37,6 @@ module HermesMessengerOfTheGods
           trap(:TERM) { endpoint.shutdown! }
           trap(:INT) { endpoint.shutdown! }
 
-          instrument(:starting)
           self.consecutive_failures = 0
           endpoint.work_off do |job, original_message|
             self.last_task_performed = Time.now
@@ -45,7 +44,6 @@ module HermesMessengerOfTheGods
             built_job = deserialize(job)
             built_job.original_message = original_message
             built_job.source_endpoint = endpoint
-            instrument(:starting_job, job: built_job)
             built_job.validate! if built_job.respond_to?(:validate!)
             run_job(built_job)
             handle_success(built_job, job)
@@ -55,7 +53,6 @@ module HermesMessengerOfTheGods
             max_consecutive_failures = self.class.max_consecutive_failures
 
             if max_consecutive_failures.is_a?(Integer) && HermesMessengerOfTheGods.config.kill_on_consecutive_failures && consecutive_failures >= max_consecutive_failures
-              instrument(:consecutive_failure_shutdown)
               return Process.kill('TERM', Process.getpgid(Process.ppid))
             end
 
@@ -72,46 +69,35 @@ module HermesMessengerOfTheGods
             throw(:skip_delete, true)
           end
         rescue StandardError => e
-          instrument(:fatal_error, exception: e)
           raise
         ensure
           endpoint.teardown
         end
 
         def run_job(job)
-          instrument(:run_job, job: job) do
-            if respond_to?(:perform)
-              perform(job)
-            elsif job.respond_to?(:perform)
-              job.perform
-            else
-              raise 'You need to define a run_job method in the worker, or a perform method on the message'
-            end
+          if respond_to?(:perform)
+            perform(job)
+          elsif job.respond_to?(:perform)
+            job.perform
+          else
+            raise 'You need to define a run_job method in the worker, or a perform method on the message'
           end
         end
 
         def handle_success(built_job, raw_job)
-          instrument(:success, job: built_job)
           endpoint.handle_success(raw_job)
           self.consecutive_failures = 0
         end
 
         def handle_failure(job, e)
-          instrument(:failure, job: job, error: e)
           endpoint.handle_failure(job, e)
           self.consecutive_failures ||= 0
           self.consecutive_failures += 1
         end
 
         def deserialize(raw_job)
-          instrument(:deserialization, job: raw_job) do
-            deserialize_method = self.class.deserialize_method || :from_message
-            self.class.deserialize_with.send(deserialize_method, raw_job)
-          end
-        end
-
-        def endpoint
-          @endpoint ||= endpoints.values.first
+          deserialize_method = self.class.deserialize_method || :from_message
+          self.class.deserialize_with.send(deserialize_method, raw_job)
         end
 
         def log_message_prefix
@@ -124,7 +110,7 @@ module HermesMessengerOfTheGods
       end
 
       class_methods do
-        attr_reader :endpoints
+        attr_reader :endpoint
         attr_accessor :deserialize_with, :deserialize_method
 
         def build_worker
@@ -166,11 +152,10 @@ module HermesMessengerOfTheGods
           @circuit_breaker_errors = val
         end
 
-        def endpoints=(val)
+        def endpoint=(val)
           raise 'Expected an endpoint' unless val.is_a?(Hash)
-          raise 'Workers can only have one defined endpoint' unless val.length == 1
 
-          @endpoints = val
+          @endpoint = val
         end
       end
     end

data/lib/hermes_messenger_of_the_gods/endpoint_builder.rb

@@ -12,8 +12,8 @@ module HermesMessengerOfTheGods
       extend ActiveSupport::Concern
 
       included do
-        def endpoints
-          @endpoints ||= self.class.build_endpoints
+        def endpoint
+          @endpoint ||= self.class.build_endpoint
         end
       end
 
@@ -36,10 +36,8 @@ module HermesMessengerOfTheGods
           end
         end
 
-        def build_endpoints
-          endpoints.each.with_object({}) do |(k, v), hsh|
-            hsh[k] = EndpointBuilder.build(k, v[:klass], *v[:args])
-          end
+        def build_endpoint
+          EndpointBuilder.build('default', endpoint[:klass], *endpoint[:args])
         end
       end
     end

data/lib/hermes_messenger_of_the_gods/endpoints/base.rb

@@ -1,13 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../concerns/base'
-
 module HermesMessengerOfTheGods
   module Endpoints
     class Base
-      extend ActiveModel::Callbacks
-
-      include HermesMessengerOfTheGods::Concerns::Base
 
       DEFAULT_OPTIONS = { retries: 3, jitter: true, backoff: :linear, jsonify: true }.freeze
       DEFAULT_RETRYS = {
@@ -17,10 +12,6 @@ module HermesMessengerOfTheGods
 
       attr_accessor :options, :endpoint, :errors, :result, :name
 
-      define_model_callbacks :dispatch
-
-      around_dispatch { |_, blk| instrument(:dispatch, &blk) }
-
       def initialize(name, endpoint, options = {})
         self.name = name
         self.options = self.class::DEFAULT_OPTIONS.merge(options)
@@ -34,22 +25,30 @@ module HermesMessengerOfTheGods
         false
       end
 
+      def _transmit_payload(message, options)
+        to_transmit_payload(transform_message(message), message, options)
+      end
+
+      def bulk_dispatch!(messages, options)
+        payloads = messages.map do |message|
+          _transmit_payload(message, options)
+        end
+
+        bulk_transmit(payloads)
+      end
+
       def dispatch!(message, options = {})
-        run_callbacks :dispatch do
-          retry_number = 0
-          begin
-            self.result = transmit(transform_message(message), message, options)
-          rescue StandardError => e
-            errors << e
-            retry_number += 1
-            if retry_number < max_retries && retry_from(e)
-              instrument(:dispatch_failure, try: retry_number, exception: e)
-              backoff(retry_number)
-              retry
-            else
-              instrument(:final_failure, try: retry_number, exception: e)
-              raise
-            end
+        retry_number = 0
+        begin
+          self.result = transmit(_transmit_payload(message, options))
+        rescue StandardError => e
+          errors << e
+          retry_number += 1
+          if retry_number < max_retries && retry_from(e)
+            backoff(retry_number)
+            retry
+          else
+            raise
           end
         end
 

data/lib/hermes_messenger_of_the_gods/endpoints/sns.rb

@@ -6,18 +6,43 @@ require 'json'
 module HermesMessengerOfTheGods
   module Endpoints
     class Sns < Base
-      def sns_topic
-        @sns_topic ||= Aws::SNS::Topic.new(endpoint,
-                                           {
-                                             client: HermesMessengerOfTheGods.configuration.sns_client,
-                                           }.merge(options[:client_options] || {}))
+      def sns_client
+        HermesMessengerOfTheGods.configuration.sns_client
       end
 
-      def transmit(message, raw_message, dispatch_options = {})
+      def to_transmit_payload(message, raw_message, dispatch_options = {})
         pub_opts = fetch_option(:publish_options, raw_message) || {}
 
         message = JSON.dump(message) if options[:jsonify]
-        sns_topic.publish(pub_opts.merge(dispatch_options, message: message))
+
+        pub_opts.merge(dispatch_options, message: message)
+      end
+
+      def transmit(payload)
+        bulk_transmit([payload])
+      end
+
+      def bulk_transmit(payloads)
+        # Batch Publish requires a message id, so we'll generate one if it's not set
+        all_publish_batch_request_entries = payloads.map do |payload|
+          {id: SecureRandom.uuid}.merge(payload)
+        end
+
+        failed_msgs = []
+
+        all_publish_batch_request_entries.each_slice(10) do |publish_batch_request_entries|
+          resp = sns_client.publish_batch(topic_arn: endpoint, publish_batch_request_entries: publish_batch_request_entries)
+          failed_msgs.concat(resp.failed)
+        end
+
+        if failed_msgs.any?
+          all_sender_fault = failed_msgs.all?(&:sender_fault)
+          raise  FatalError, "Error in dispatching: #{failed_msgs[0].message}" if all_sender_fault
+
+          raise "Some messages failed to send due to recoverable error #{failed_msgs[0].message}"
+        end
+
+        true
       end
     end
   end

data/lib/hermes_messenger_of_the_gods/endpoints/sqs.rb

@@ -111,10 +111,13 @@ module HermesMessengerOfTheGods
 
         !skip_delete
       rescue StandardError => e
-        instrument(:read_failure, exception: e)
         false
       end
 
+      def sqs_client
+        HermesMessengerOfTheGods.configuration.sqs_client
+      end
+
       def queue
         @queue ||= Aws::SQS::Queue.new(
           endpoint,
@@ -139,11 +142,38 @@ module HermesMessengerOfTheGods
         approximate_pending_messages.positive?
       end
 
-      def transmit(message, raw_message, dispatch_options = {})
+      def to_transmit_payload(message, raw_message, dispatch_options = {})
         send_opts = fetch_option(:send_options, raw_message) || {}
 
         message = JSON.dump(message) if options[:jsonify]
-        queue.send_message(send_opts.merge(dispatch_options, message_body: message))
+
+        send_opts.merge(dispatch_options, message_body: message)
+      end
+
+      def transmit(payload)
+        bulk_transmit([payload])
+      end
+
+      def bulk_transmit(payloads)
+        all_entries = payloads.map! do |payload|
+          {id: SecureRandom.uuid}.merge(payload)
+        end
+
+        failed_msgs = []
+
+        all_entries.each_slice(10) do |entries|
+          resp = sqs_client.send_message_batch(queue_url: endpoint, entries: entries)
+          failed_msgs.concat(resp.failed)
+        end
+
+        if failed_msgs.any?
+          all_sender_fault = failed_msgs.failed.all?(&:sender_fault)
+          raise  FatalError, "Error in dispatching: #{failed_msgs[0].message}" if all_sender_fault
+
+          raise "Some messages failed to send due to recoverable error #{failed_msgs[0].message}"
+        end
+
+        true
       end
 
       private

data/lib/hermes_messenger_of_the_gods/exceptions.rb

@@ -1,27 +1,14 @@
 # frozen_string_literal: true
 
-module HermesMessengerOfTheGods
-  if defined?(ActiveModel::ValidationError)
-    class ValidationError < ActiveModel::ValidationError; end
-  else
-    class ValidationError < StandardError
-      attr_reader :model
+require 'active_model'
 
-      def initialize(model)
-        @model = model
-        errors = @model.errors.full_messages.join(', ')
-        super(errors)
-      end
-    end
-  end
+module HermesMessengerOfTheGods
+  class ValidationError < ActiveModel::ValidationError; end
 
   class MessageDispatchFailed < StandardError
     attr_accessor :exceptions
   end
 
-  class MessageDispatchTotalFailure < MessageDispatchFailed; end
-  class MessageDispatchPartialFailure < MessageDispatchFailed; end
-
   module Endpoints
     class FatalError < RuntimeError; end
   end

data/lib/hermes_messenger_of_the_gods/testing/array_endpoint.rb

@@ -4,8 +4,6 @@ module HermesMessengerOfTheGods
   module Endpoints
     class Local < Base
       class << self
-        attr_accessor :transmit_method
-
         def clear_queue!
           @received = nil
         end
@@ -21,20 +19,23 @@ module HermesMessengerOfTheGods
         self.class.received[endpoint]
       end
 
-      def transmit(msg, raw_msg, opts = {})
-        if self.class.transmit_method
-          self.class.transmit_method.call(self, msg, raw_msg)
-        else
-          do_transmit(msg, opts, raw_msg)
-        end
+      def to_transmit_payload(message, raw_message, dispatch_options = {})
+        { message: message, raw_message: raw_message, options: dispatch_options }
+      end
+
+      def transmit(payload)
+        bulk_transmit([payload])
       end
 
-      def do_transmit(msg, opts, raw_message = {})
-        pub_opts = fetch_option(:publish_options, raw_message) || {}
-        new_msg = { message: msg, options: opts.merge(pub_opts) }
-        received << new_msg
-        on_receive&.call(new_msg)
-        "Recorded msg ##{self.class.received[endpoint].length} for #{endpoint}"
+      def bulk_transmit(payloads)
+        payloads.each do |msg|
+          pub_opts = fetch_option(:publish_options, msg[:raw_message]) || {}
+
+          new_msg = { message: msg[:message], options: msg[:options].merge(pub_opts) }
+          received << new_msg
+          on_receive&.call(new_msg)
+          "Recorded msg ##{self.class.received[endpoint].length} for #{endpoint}"
+        end
       end
     end
   end
@@ -44,7 +45,6 @@ if defined?(RSpec)
   RSpec.configure do |config|
     config.after(:each) do
       HermesMessengerOfTheGods::Endpoints::Local.clear_queue!
-      HermesMessengerOfTheGods::Endpoints::Local.transmit_method = nil
     end
   end
 end

data/lib/hermes_messenger_of_the_gods/testing/dispatch_matcher.rb

@@ -17,7 +17,6 @@ RSpec::Matchers.define :dispatch do |message|
     return false if @matched_early
 
     proc.call
-    wait_for_async_dispatches
 
     !@endpoint.received.detect do |m|
       a_hash_including(message).matches?(m[:message]) && (

data/lib/hermes_messenger_of_the_gods/testing/rspec_helpers.rb

@@ -45,14 +45,6 @@ module HermesMessengerOfTheGods
           build_fake_endpoint(klass, endpoint, *args)
         end
       end
-
-      def wait_for_async_dispatches
-        return if HermesMessengerOfTheGods.async_dispatches_in_progress.zero?
-
-        Timeout.timeout(5) do
-          sleep 0.01 until HermesMessengerOfTheGods.async_dispatches_in_progress.zero?
-        end
-      end
     end
   end
 end

data/lib/hermes_messenger_of_the_gods/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HermesMessengerOfTheGods
-  VERSION = '2.4.2'
+  VERSION = '3.0.0.rc2'
 end

data/lib/hermes_messenger_of_the_gods.rb

@@ -8,7 +8,6 @@ require 'hermes_messenger_of_the_gods/configuration'
 require 'hermes_messenger_of_the_gods/logging_helpers'
 require 'hermes_messenger_of_the_gods/endpoint_builder'
 
-require 'hermes_messenger_of_the_gods/concerns/base'
 require 'hermes_messenger_of_the_gods/concerns/message'
 require 'hermes_messenger_of_the_gods/concerns/worker'
 require 'hermes_messenger_of_the_gods/concerns/mono_message'
@@ -16,8 +15,6 @@ require 'hermes_messenger_of_the_gods/concerns/grpc_protobuf'
 
 require 'hermes_messenger_of_the_gods/endpoints'
 
-require 'hermes_messenger_of_the_gods/output/basic'
-
 module HermesMessengerOfTheGods
   class << self
     def configuration
@@ -29,29 +26,5 @@ module HermesMessengerOfTheGods
       configuration
     end
     alias config configure
-
-    def async_dispatches_in_progress
-      @async_dispatches_in_progress ||= 0
-    end
-
-    def increment_async_dispatches_in_progress
-      monitor.synchronize do
-        self.async_dispatches_in_progress += 1
-      end
-    end
-
-    def decrement_async_dispatches_in_progress
-      monitor.synchronize do
-        self.async_dispatches_in_progress -= 1
-      end
-    end
-
-    private
-
-    attr_writer :async_dispatches_in_progress
-
-    def monitor
-      @monitor ||= Monitor.new
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hermes_messenger_of_the_gods
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 3.0.0.rc2
 platform: ruby
 authors:
 - Brian Malinconico
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-27 00:00:00.000000000 Z
+date: 2023-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -217,7 +217,6 @@ files:
 - exe/fly_hermes
 - hermes_messenger_of_the_gods.gemspec
 - lib/hermes_messenger_of_the_gods.rb
-- lib/hermes_messenger_of_the_gods/concerns/base.rb
 - lib/hermes_messenger_of_the_gods/concerns/grpc_protobuf.rb
 - lib/hermes_messenger_of_the_gods/concerns/message.rb
 - lib/hermes_messenger_of_the_gods/concerns/mono_message.rb
@@ -230,7 +229,6 @@ files:
 - lib/hermes_messenger_of_the_gods/endpoints/sqs.rb
 - lib/hermes_messenger_of_the_gods/exceptions.rb
 - lib/hermes_messenger_of_the_gods/logging_helpers.rb
-- lib/hermes_messenger_of_the_gods/output/basic.rb
 - lib/hermes_messenger_of_the_gods/status_server.rb
 - lib/hermes_messenger_of_the_gods/testing/array_endpoint.rb
 - lib/hermes_messenger_of_the_gods/testing/dispatch_matcher.rb
@@ -252,9 +250,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.2.22
 signing_key:

data/lib/hermes_messenger_of_the_gods/concerns/base.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support'
-require 'active_support/core_ext/object/blank'
-
-module HermesMessengerOfTheGods
-  module Concerns
-    module Base
-      extend ActiveSupport::Concern
-
-      include EndpointBuilder::Helpers
-      include LoggingHelpers
-
-      included do
-        def instrument(name, payload = {}, &blk)
-          name = [
-            'hermes_messenger_of_the_gods',
-            notification_prefix,
-            name,
-          ].reject(&:blank?).join('.')
-
-          payload[_instrument_key] ||= self if _instrument_key
-          ActiveSupport::Notifications.instrument(name, payload, &blk)
-        end
-
-        def notification_prefix
-          _instrument_key
-        end
-
-        def _instrument_key
-          if is_a?(HermesMessengerOfTheGods::Concerns::Worker)
-            :worker
-          elsif is_a?(HermesMessengerOfTheGods::Concerns::Message)
-            :message
-          elsif is_a?(HermesMessengerOfTheGods::Endpoints::Base)
-            :endpoint
-          end
-        end
-      end
-    end
-  end
-end

data/lib/hermes_messenger_of_the_gods/output/basic.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-# Worker Output
-module HermesMessengerOfTheGods
-  module Output
-    module Basic
-      @subscriptions = []
-
-      def self.unsubscribe!
-        @subscriptions.map { |s| ActiveSupport::Notifications.unsubscribe(s) }
-      end
-
-      def self.subscribe!
-        unsubscribe!
-
-        @subscriptions = [
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.starting') do |*_, payload|
-            payload[:worker].say_info { 'Starting Worker' }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.run_job') do |_, start, finish, _, payload|
-            payload[:worker].say_debug { "Completed #{to_log_s(payload[:job])} in #{finish - start}s" }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.starting_job') do |*_, payload|
-            payload[:worker].say_info { "Starting Job #{to_log_s(payload[:job])}" }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.success') do |*_, payload|
-            payload[:worker].say_info { "Finished Job #{to_log_s(payload[:job])}" }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.failure') do |*_, payload|
-            payload[:worker].say_error { "Error in #{to_log_s(payload[:job])}: #{payload[:error].message}" }
-            payload[:worker].say_debug { "backtrace: #{payload[:error].backtrace}" }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.worker.fatal_error') do |*_, payload|
-            payload[:worker].say_error { "Fatal Error: #{to_log_s(payload[:job])}: #{payload[:exception].message}" }
-            payload[:worker].say_debug { "backtrace: #{payload[:exception].backtrace}"  }
-          end,
-          # Message Output
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.message.dispatch') do |_, _start, _finish, _, payload|
-            payload[:message].say_debug { "Dispatch complete in #{to_log_s(payload[:job])}s" }
-          end,
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.message.dispatch_failure') do |*_, payload|
-            payload[:message].say_error { "Dispatch failure to #{to_log_s(payload[:job])}: #{payload[:exception].inspect}" }
-          end,
-          # Endpoint Output
-          ActiveSupport::Notifications.subscribe('hermes_messenger_of_the_gods.endpoint.dispatch') do |_, start, finish, _, payload|
-            payload[:endpoint].say_debug { "Dispatch complete in #{finish - start}s #{' FAILED' if payload.key?(:exception)}" }
-          end,
-          ActiveSupport::Notifications.subscribe(/hermes_messenger_of_the_gods.endpoint.(dispatch|final)_failure/) do |name, *_, payload|
-            payload[:endpoint].say_error { "Dispatch #{'final ' if name.include?('final_')} failure ##{payload[:try]} to #{payload[:endpoint_name]}: #{payload[:exception].inspect}" }
-          end,
-          ActiveSupport::Notifications.subscribe(/hermes_messenger_of_the_gods.endpoint.read_failure/) do |_name, *_, payload|
-            payload[:endpoint].say_error { "A message was received that could not be decoded: #{payload[:exception].message}" }
-          end,
-        ]
-      end
-      subscribe!
-
-      def self.to_log_s(item)
-        if item.respond_to?(:to_log_s)
-          item.to_log_s
-        else
-          item.inspect
-        end
-      end
-    end
-  end
-end