hermes_messenger_of_the_gods 2.0.0

data/lib/hermes_messenger_of_the_gods/concerns/mono_message.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'active_support'
+
+require 'hermes_messenger_of_the_gods/concerns/message'
+require 'hermes_messenger_of_the_gods/concerns/worker'
+
+module HermesMessengerOfTheGods
+  module Concerns
+    module MonoMessage
+      extend ActiveSupport::Concern
+
+      include HermesMessengerOfTheGods::Concerns::Message
+
+      class_methods do
+        def endpoints=(*args)
+          super
+          worker_klass.endpoints = endpoints
+        end
+
+        def circuit_breaker_errors=(*args)
+          super
+          worker_klass.circuit_breaker_errors = circuit_breaker_errors
+        end
+
+        def max_consecutive_failures=(*args)
+          super
+          worker_klass.max_consecutive_failures = max_consecutive_failures
+        end
+
+        def build_worker
+          worker_klass.build_worker
+        end
+
+        def work_off
+          build_worker.work_off
+        end
+
+        def worker_klass
+          me = self
+          @worker_klass ||= Class.new do
+            include HermesMessengerOfTheGods::Concerns::Worker
+            self.endpoints = me.endpoints
+            self.circuit_breaker_errors = me.circuit_breaker_errors
+            self.max_consecutive_failures = me.max_consecutive_failures
+            self.deserialize_with = me
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hermes_messenger_of_the_gods/concerns/worker.rb

@@ -0,0 +1,173 @@
+require 'active_support'
+
+require_relative './base'
+require_relative '../status_server'
+
+module HermesMessengerOfTheGods
+  module Concerns
+    module Worker
+      extend ActiveSupport::Concern
+
+      include HermesMessengerOfTheGods::Concerns::Base
+
+      included do
+        attr_writer :name
+        attr_accessor :consecutive_failures, :last_task_performed
+
+        def health_check_enabled?
+          ENV.fetch('ENABLE_HERMES_HEALTH_CHECK', 'false') != 'false'
+        end
+
+        def expected_work_frequency
+          ENV.fetch('HERMES_MINIMUM_WORK_FREQUENCY', 60).to_i
+        end
+
+        def healthy?
+          !endpoint.has_pending_work? || (
+            last_task_performed &&
+            Time.now - last_task_performed < expected_work_frequency
+          )
+        end
+
+        def work_off
+          WorkerStatusServer.start!(worker: self) if health_check_enabled?
+
+          instrument(:starting)
+          self.consecutive_failures = 0
+          endpoint.work_off do |job, original_message|
+            self.last_task_performed = Time.now
+
+            built_job = deserialize(job, original_message)
+            instrument(:starting_job, job: built_job)
+            built_job.validate! if built_job.respond_to?(:validate!)
+            run_job(built_job)
+            handle_success(built_job, job)
+          rescue StandardError => e
+            handle_failure(job, e)
+
+            max_consecutive_failures = self.class.max_consecutive_failures
+
+            if max_consecutive_failures.is_a?(Integer) && HermesMessengerOfTheGods.config.kill_on_consecutive_failures && consecutive_failures >= max_consecutive_failures
+              instrument(:consecutive_failure_shutdown)
+              return Process.kill('TERM', Process.getpgid(Process.ppid))
+            end
+
+            circuit_breaker_error = self.class.circuit_breaker_errors[e.class]
+
+            matching_circuit_breaker_error = self.class.circuit_breaker_errors.keys.detect { |potential_circuit_breaker| e.is_a?(potential_circuit_breaker) }
+
+            if matching_circuit_breaker_error
+              circuit_breaker_error = self.class.circuit_breaker_errors[matching_circuit_breaker_error]
+              sleep_time = circuit_breaker_error[:sleep] || 0
+              Kernel.sleep(sleep_time) unless sleep_time.zero?
+              raise e if !!circuit_breaker_error[:fatal]
+            end
+            throw(:skip_delete, true)
+          end
+        rescue StandardError => e
+          instrument(:fatal_error, exception: e)
+          raise
+        ensure
+          endpoint.teardown
+        end
+
+        def run_job(job)
+          instrument(:run_job, job: job) do
+            if respond_to?(:perform)
+              perform(job)
+            elsif job.respond_to?(:perform)
+              job.perform
+            else
+              raise 'You need to define a run_job method in the worker, or a perform method on the message'
+            end
+          end
+        end
+
+        def handle_success(built_job, raw_job)
+          instrument(:success, job: built_job)
+          endpoint.handle_success(raw_job)
+          self.consecutive_failures = 0
+        end
+
+        def handle_failure(job, e)
+          instrument(:failure, job: job, error: e)
+          endpoint.handle_failure(job, e)
+          self.consecutive_failures ||= 0
+          self.consecutive_failures += 1
+        end
+
+        def deserialize(raw_job, original_message)
+          instrument(:deserialization, job: raw_job) do
+            deserialize_method = self.class.deserialize_method || :from_message
+            self.class.deserialize_with.send(deserialize_method, raw_job).tap do |obj|
+              obj.instance_variable_set :@original_message, original_message
+            end
+          end
+        end
+
+        def endpoint
+          @endpoint ||= endpoints.values.first
+        end
+
+        def log_message_prefix
+          "Worker #{name}(pid: #{$PROCESS_ID})"
+        end
+
+        def name
+          @name ||= self.class.to_s
+        end
+      end
+
+      class_methods do
+        attr_reader :endpoints
+        attr_accessor :deserialize_with, :deserialize_method
+
+        def build_worker
+          new
+        end
+
+        def work_off
+          build_worker.work_off
+        end
+
+        def max_consecutive_failures
+          @max_consecutive_failures
+        end
+
+        def max_consecutive_failures=(val)
+          raise 'Expected an Integer' unless val.is_a?(Integer) || val.is_a?(NilClass)
+
+          @max_consecutive_failures = val
+        end
+
+        def circuit_breaker_errors
+          @circuit_breaker_errors || {}
+        end
+
+        def circuit_breaker_errors=(val)
+          val ||= {}
+          raise 'Expected a hash' unless val.is_a?(Hash)
+
+          val.each do |err, _actions|
+            val[err][:sleep] ||= 0
+            val[err][:fatal] ||= false
+          end
+
+          val.each do |_err, actions|
+            raise ':sleep must be a number' unless actions[:sleep].is_a?(Numeric)
+            raise ':fatal must be a boolean' unless [true, false].include?(actions[:fatal])
+          end
+
+          @circuit_breaker_errors = val
+        end
+
+        def endpoints=(val)
+          raise 'Expected an endpoint' unless val.is_a?(Hash)
+          raise 'Workers can only have one defined endpoint' unless val.length == 1
+
+          @endpoints = val
+        end
+      end
+    end
+  end
+end

data/lib/hermes_messenger_of_the_gods/configuration.rb

@@ -0,0 +1,29 @@
+require 'logger'
+
+module HermesMessengerOfTheGods
+  class Configuration
+    class << self
+      def attr_accessor_with_default(meth, default)
+        attr_writer meth
+
+        define_method(meth) do
+          if instance_variable_defined?("@#{meth}")
+            instance_variable_get("@#{meth}")
+          else
+            instance_variable_set("@#{meth}", default)
+          end
+        end
+      end
+    end
+
+    # Log location for HMOTG info messages. Should respond like your
+    # typical Ruby Logger defaults to logging via STDOUT
+    attr_accessor_with_default :logger, Logger.new(STDOUT)
+
+    attr_accessor_with_default :quiet, false
+
+    attr_accessor_with_default :stub_dispatch, false
+
+    attr_accessor_with_default :kill_on_consecutive_failures, false
+  end
+end

data/lib/hermes_messenger_of_the_gods/endpoint_builder.rb

@@ -0,0 +1,41 @@
+require 'active_support'
+
+module HermesMessengerOfTheGods
+  module EndpointBuilder
+    def self.build(name, klass, *args)
+      klass.new(name, *args)
+    end
+
+    module Helpers
+      extend ActiveSupport::Concern
+
+      included do
+        def endpoints
+          @endpoints ||= self.class.build_endpoints
+        end
+      end
+
+      class_methods do
+        def is_valid_endpoint_creation_method(method)
+          method.to_s =~ /(\w+)_endpoint\z/ &&
+            ("HermesMessengerOfTheGods::Endpoints::#{$1.camelize}".constantize rescue nil)
+        end
+
+        def method_missing(method, *args, &blk)
+          klass = is_valid_endpoint_creation_method(method)
+          if klass
+            {klass: klass, args: args}
+          else
+            super(method, *args, &blk)
+          end
+        end
+
+        def build_endpoints
+          self.endpoints.each.with_object({}) do |(k,v), hsh|
+            hsh[k] = EndpointBuilder.build(k, v[:klass], *v[:args])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hermes_messenger_of_the_gods/endpoints.rb

@@ -0,0 +1,3 @@
+require 'hermes_messenger_of_the_gods/endpoints/base'
+require 'hermes_messenger_of_the_gods/endpoints/sns'
+require 'hermes_messenger_of_the_gods/endpoints/sqs'

data/lib/hermes_messenger_of_the_gods/endpoints/base.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/base'
+
+module HermesMessengerOfTheGods
+  module Endpoints
+    class Base
+      extend ActiveModel::Callbacks
+
+      include HermesMessengerOfTheGods::Concerns::Base
+
+      DEFAULT_OPTIONS = { retries: 3, jitter: true, backoff: :linear, jsonify: true }.freeze
+      DEFAULT_RETRYS = {
+        linear: ->(n) { n }, # Wait 1 sec, then two secondns, then three...
+        exponential: ->(n) { (2**n) / (n + 1).to_f } # Wait 1, then 1.3, then 2 sec
+      }.freeze
+
+      attr_accessor :options, :endpoint, :errors, :result, :name
+
+      define_model_callbacks :dispatch
+
+      around_dispatch { |_, blk| instrument(:dispatch, &blk) }
+
+      def initialize(name, endpoint, options = {})
+        self.name = name
+        self.options = self.class::DEFAULT_OPTIONS.merge(options)
+        self.endpoint = endpoint
+        self.errors = []
+      end
+
+      def dispatch(*args)
+        dispatch!(*args)
+      rescue StandardError
+        false
+      end
+
+      def dispatch!(message, options = {})
+        run_callbacks :dispatch do
+          retry_number = 0
+          begin
+            self.result = transmit(transform_message(message), message, options)
+          rescue StandardError => e
+            errors << e
+            retry_number += 1
+            if retry_number < max_retries && retry_from(e)
+              instrument(:dispatch_failure, try: retry_number, exception: e)
+              backoff(retry_number)
+              retry
+            else
+              instrument(:final_failure, try: retry_number, exception: e)
+              raise
+            end
+          end
+        end
+
+        true
+      end
+
+      def backoff(retry_number)
+        return unless options[:backoff]
+
+        backoff_arg = if options[:backoff].respond_to?(:call)
+                        options[:backoff]
+                      else
+                        DEFAULT_RETRYS[options[:backoff]]
+                      end
+        sleep_time = backoff_arg.call(retry_number)
+
+        if sleep_time
+          sleep(sleep_time * (options[:jitter] ? rand(0.3..1) : 1))
+        end
+      end
+
+      def transform_message(message)
+        transformer_name = "to_#{self.class.to_s.demodulize.underscore}_message"
+
+        if options[:transformer]
+          options[:transformer].respond_to?(:call) ?
+            options[:transformer].call(message) :
+            message.send(options[:transformer])
+        elsif message.respond_to?(transformer_name)
+          message.public_send(transformer_name)
+        elsif message.respond_to?(:_build_for_transmission)
+          message._build_for_transmission
+        else
+          message
+        end
+      end
+
+      def retry_from(exception)
+        !exception.is_a?(HermesMessengerOfTheGods::Endpoints::FatalError)
+      end
+
+      def max_retries
+        options[:retries] || 0
+      end
+
+      def teardown; end
+
+      def handle_success(_job); end
+
+      def handle_failure(_job, e); end
+
+      def fetch_option(option, *args, &blk)
+        if options[option].respond_to?(:call)
+          options[option].call(*args, &blk)
+        else
+          options[option]
+        end || {}
+      end
+    end
+  end
+end

data/lib/hermes_messenger_of_the_gods/endpoints/sns.rb

@@ -0,0 +1,21 @@
+require 'aws-sdk-sns'
+require 'json'
+
+module HermesMessengerOfTheGods
+  module Endpoints
+    class Sns < Base
+      def sns_topic
+        @sns_topic ||= Aws::SNS::Topic.new(self.endpoint,
+                                           self.options[:client_options] || {}
+                                          )
+      end
+
+      def transmit(message, raw_message, dispatch_options = {})
+        pub_opts = fetch_option(:publish_options, raw_message) || {}
+
+        message = JSON.dump(message) if self.options[:jsonify]
+        sns_topic.publish(pub_opts.merge(dispatch_options, message: message))
+      end
+    end
+  end
+end

data/lib/hermes_messenger_of_the_gods/endpoints/sqs.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'json'
+require_relative './base'
+
+module HermesMessengerOfTheGods
+  module Endpoints
+    class Sqs < Base
+      VISIBILITY_EXTEND_DURATION  = 120
+      VISIBILITY_EXTEND_FREQUENCY = 60
+
+      def poller
+        @poller ||= Aws::SQS::QueuePoller.new(endpoint)
+      end
+
+      def poll_options
+        (options[:poll_options] || {}).merge(skip_delete: true)
+      end
+
+      def work_off(&blk)
+        poller.poll(poll_options) do |messages, _stats|
+          messages = Array.wrap(messages)
+
+          working_messages(messages) do
+            completed = messages.select { |msg| work_message(msg, &blk) }
+            poller.delete_messages(completed) unless completed.empty?
+          end
+        end
+      end
+
+      def work_message(message)
+        message_body = decode_message(message)
+
+        skip_delete = catch(:skip_delete) do
+          yield(message_body, message)
+          false
+        end
+
+        !skip_delete
+      rescue StandardError => e
+        instrument(:read_failure, exception: e)
+        false
+      end
+
+      def queue
+        @queue ||= Aws::SQS::Queue.new(endpoint, options[:client_options] || {})
+      end
+
+      def queue_data
+        queue.reload.data
+      end
+
+      def has_pending_work?
+        data = queue_data.attributes
+
+        approximate_pending_messages = data["ApproximateNumberOfMessages"].to_i -
+          data["ApproximateNumberOfMessagesNotVisible"].to_i -
+          data["ApproximateNumberOfMessagesDelayed"].to_i
+
+        # Just in case the math is off
+        approximate_pending_messages > 0
+      end
+
+      def transmit(message, raw_message, dispatch_options = {})
+        send_opts = fetch_option(:send_options, raw_message) || {}
+
+        message = JSON.dump(message) if options[:jsonify]
+        queue.send_message(send_opts.merge(dispatch_options, message_body: message))
+      end
+
+      private
+
+      def decode_message(message)
+        message_body = JSON.parse(message.body)
+        message_body = JSON.parse(message_body['Message']) if options[:from_sns]
+        message_body
+      end
+
+      def working_messages(messages)
+        thread = start_visibility_update_thread(messages)
+
+        yield
+      ensure
+        thread&.terminate
+      end
+
+      def start_visibility_update_thread(messages)
+        start_work_time = Time.now
+        Thread.new do
+          loop do
+            new_time = (Time.now - start_work_time) + VISIBILITY_EXTEND_DURATION
+            queue.change_message_visibility_batch(
+              entries: messages.collect do |message|
+                {
+                  id: SecureRandom.uuid,
+                  receipt_handle: message.receipt_handle,
+                  visibility_timeout: new_time
+                }
+              end
+            )
+
+            sleep VISIBILITY_EXTEND_FREQUENCY
+          rescue StandardError => e
+            STDERR.puts 'Error received trying to extend visibility'
+            STDERR.puts e.message
+
+            raise
+          end
+        end
+      end
+    end
+  end
+end