hermes-rb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 796451c71d28d43f93373f55f0b0d4e81c260026775c2e65756730aabe3d80ca
-  data.tar.gz: efa0dcb057ed87302c0c558f17392d06a83db786088f4617c35d3f749bb0a834
+  metadata.gz: 104554076c3b4ad46491c12d842e3580fdf8c96ba36eefbfc2f4c66a5ba5f208
+  data.tar.gz: a5f1754803e51907feda1164ff25c048731037f4cd75f2a6eaf4db68668b6647
 SHA512:
-  metadata.gz: e64fd98f7a5083b81dfd2fa09a491f1a1d6fea938022f29cad594a1132638f41620eb98aa4602e03fae647bc070ccd067e9d82657098a63ab48a05ef5d9d580a
-  data.tar.gz: 749a9a7d7798c1f76f70ef7f19b1f44d60738806804eac9855f7ba8027fda830c5604fff4465e2f0f80fa9dd441a51d598c2eb304dc091e28bb2ae536ea262d5
+  metadata.gz: aea75486bab7ba279b788dc48ea465f3ecb96c7281d9b6cc9bc29443c738ba779fe0951f6d0a8028211cce4f64520556f085cab73aee7ba98bac01af9dc5a973
+  data.tar.gz: 10b79dce9ad5c116d6ccb31a6102b1e3732a2b0a73e328b5ef37b421e5407cc8a5cc240e6b68c11b7fd26bd2c06acb579a610ba3708d83515d5c17dc48207f50

data/Changelog.md

@@ -1,4 +1,7 @@
 # Changelog
 
+## 0.2.0
+- Implement "Safe Producer" extension allowing to retry delivery later in an automated way when publishing of the message fails.
+
 ## 0.1.0
 - Initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hermes-rb (0.1.0)
+    hermes-rb (0.2.0)
       activerecord (>= 5)
       activesupport (>= 5)
       dry-container (~> 0)
@@ -12,12 +12,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.0)
-      activesupport (= 6.1.0)
-    activerecord (6.1.0)
-      activemodel (= 6.1.0)
-      activesupport (= 6.1.0)
-    activesupport (6.1.0)
+    activemodel (6.1.1)
+      activesupport (= 6.1.1)
+    activerecord (6.1.1)
+      activemodel (= 6.1.1)
+      activesupport (= 6.1.1)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -30,21 +30,19 @@ GEM
       json
     concurrent-ruby (1.1.7)
     diff-lcs (1.4.4)
-    dry-configurable (0.11.6)
+    dry-configurable (0.12.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.4, >= 0.4.7)
-      dry-equalizer (~> 0.2)
+      dry-core (~> 0.5, >= 0.5.0)
     dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.10)
+    dry-core (0.5.0)
       concurrent-ruby (~> 1.0)
     dry-equalizer (0.3.0)
     dry-inflector (0.2.0)
-    dry-logic (1.0.8)
+    dry-logic (1.1.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.2)
-      dry-equalizer (~> 0.2)
+      dry-core (~> 0.5, >= 0.5)
     dry-struct (1.3.0)
       dry-core (~> 0.4, >= 0.4.4)
       dry-equalizer (~> 0.3)
@@ -62,11 +60,11 @@ GEM
       bunny (>= 2.15, < 2.16)
       carrot-top (~> 0.0.7)
       multi_json (~> 1.14)
-    i18n (1.8.5)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
-    json (2.3.1)
-    minitest (5.14.2)
+    json (2.5.1)
+    minitest (5.14.3)
     multi_json (1.15.0)
     pg (1.2.3)
     rack (2.2.3)
@@ -87,7 +85,7 @@ GEM
       rspec-support (~> 3.10.0)
     rspec-support (3.10.0)
     timecop (0.9.2)
-    tzinfo (2.0.3)
+    tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     vcr (5.0.0)
     zeitwerk (2.4.2)

data/README.md

@@ -136,10 +136,14 @@ If you don't care about it, you can leave it empty.
 
 13. `distributes_tracing_mapper` - an object responding to `call` method taking one argument (a hash of attributes) that has to return a hash as well. This hash will be used for assigning attributes when creating `Hermes::DistributedTrace`. The default mapper just returns the original hash. You can use it if you want to remove, for example, some sensitive info from the event's body. Optional.
 
-14. `error_notification_service` - an object responding to `capture_exception` method taking one argument (error). Used when storing distributed traces, its interface is based on `Raven` from [Sentry Raven](https://github.com/getsentry/sentry-ruby/tree/master/sentry-raven). By default `Hermes::NullErrorNotificationService` is used, which does nothing. Optional.
+14. `error_notification_service` - an object responding to `capture_exception` method taking one argument (error). Its interface is based on `Raven` from [Sentry Raven](https://github.com/getsentry/sentry-ruby/tree/master/sentry-raven). By default `Hermes::NullErrorNotificationService` is used, which does nothing. Optional.
 
 15. `database_error_handler` - `an object responding to `call` method taking one argument (error). Used when storing distributed traces. By default it uses `Hermes::DatabaseErrorHandler` which depends on `error_notification_service`, so in most cases, you will probably want to just configure `error_notification_service`. Optional.
 
+16. `enable_safe_producer` - a method requiring a job class implementing `enqueue` method that will be responsible for retrying delivery of the event later in case it fails. Check `Safe Event Producer` section for more details.
+
+17. `producer_retryable` - used when `safe_producer` was enabled via (`enable_safe_producer`). By default, it is a method retrying delivery 3 times rescuing from `StandardError` each time. The object responsible for this behavior by default is: `Hermes::Retryable.new(times: 3, errors: [StandardError])`.
+
 ## RPC
 
 If you want to handle RPC call, you need to add `rpc: true` flag. Keep in mind that RPC requires a synchronous processing and response, so you also need to set `async: false`. The routing key and correlation ID will be resolved based on the message that is published by the client. The payload that is sent back will be what event handler reutrns, so it might be a good idea to just return a hash so that you can operate on JSON easily.
@@ -226,6 +230,40 @@ It is highly recommended to use a shared database for storing traces. It's not i
 
 Since distributed tracing is a secondary feature, all exceptions coming from the database are rescued. It is highly recommended to provide `error_notification_service` to be notified about these errors. If you are not happy with that behavior and you would prefer to have errors raised, you can implement your own `database_error_handler` where you can re-raise the exception.
 
+## Safe Event Producer
+
+Most likely in your production environment you are going to have a high availability setup with more than one node, probably at least 3. This might seem like there is very little chance that something will go wrong when publishing an event to RabbitMQ and even if it happens, it will be so rare that you will handle any exceptions manually. However, operations like updating Erlang will most likely require a downtime, which will mean that you might have a lot of errors during that period. Not to mention other potential issues, even without scheduled downtime, like the entire cluster being down for random reason or timeouts.
+
+In that case, it might be a good idea to have some automated way of dealing with this kind of issues. For that purpose, you can enable a `Safe Event Producer` - by default, it's going to try publishing event 3 times, rescuing twice from `StandardError`, and if it fails after a 3rd time, it's going to use `error_notification_service` to deliver info about the error that ahppened and is going to call `enqueue` method on a specified object.
+
+To take advantage of this feature, apply the following logic in the initializer
+
+``` rb
+Hermes.configure do |config|
+  config.clock = clock
+  config.error_notification_service = Raven
+  config.enable_safe_producer(HermesRecoveryJob)
+end
+```
+
+`HermesRecoveryJob` is expected to implement `enqueue` method taking 3 arguments: `event_class_name`, `event_body` and `headers`. What happens in `enqueue` method is up to you. You can, for example, schedule publishing the message in 5 minutes from now. However, the job should call `Hermes::RetryableEventProducer.publish(event_class, event_body, headers)` to properly handle the delivery retry flow. Here is an example job class using Sidekiq:
+
+``` rb
+class HermesRecoveryJob
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :hermes_recovery
+
+  def self.enqueue(event_class, event_body, origin_headers)
+    perform_at(5.minutes.from_now, event_class, event_body, origin_headers)
+  end
+
+  def perform(event_class, event_body, origin_headers)
+    Hermes::RetryableEventProducer.publish(event_class, event_body, origin_headers)
+  end
+end
+```
+
 ## Testing
 
 ### RSpec useful stuff
@@ -395,7 +433,7 @@ RSpec.describe HermesHandlerJob do
 end
 ```
 
-## Deployment and managing consumerzs
+## Deployment and managing consumers
 
 Hermes is just an extra layer on top of [hutch](https://github.com/gocardless/hutch), refer to Hutch's docs for more info about dealing with the workers and deployment.
 

data/lib/hermes.rb

@@ -18,6 +18,11 @@ require "hermes/distributed_trace_repository"
 require "hermes/dependencies_container"
 require "hermes/null_error_notification_service"
 require "hermes/database_error_handler"
+require "hermes/retryable"
+require "hermes/retryable_event_producer"
+require "hermes/producer_error_handler"
+require "hermes/producer_error_handler/null_handler"
+require "hermes/producer_error_handler/safe_handler"
 require "active_support"
 require "active_support/core_ext/string"
 require "active_record"

data/lib/hermes/b_3_propagation_model_headers.rb

@@ -3,6 +3,22 @@ module Hermes
     attr_reader :trace_context
     private     :trace_context
 
+    def self.trace_id_key
+      "X-B3-TraceId"
+    end
+
+    def self.span_id_key
+      "X-B3-SpanId"
+    end
+
+    def self.parent_span_id_key
+      "X-B3-ParentSpanId"
+    end
+
+    def self.sampled_key
+      "X-B3-Sampled"
+    end
+
     def initialize(trace_context)
       @trace_context = trace_context
     end
@@ -13,10 +29,10 @@ module Hermes
 
     def to_h
       {
-        "X-B3-TraceId" => trace_context.trace,
-        "X-B3-ParentSpanId" => trace_context.parent_span,
-        "X-B3-SpanId" => trace_context.span,
-        "X-B3-Sampled" => ""
+        self.class.trace_id_key => trace_context.trace,
+        self.class.parent_span_id_key => trace_context.parent_span,
+        self.class.span_id_key => trace_context.span,
+        self.class.sampled_key => ""
       }
     end
   end

data/lib/hermes/configuration.rb

@@ -3,7 +3,8 @@ module Hermes
     attr_accessor :adapter, :clock, :hutch, :application_prefix, :logger,
       :background_processor, :enqueue_method, :event_handler, :rpc_call_timeout,
       :instrumenter, :distributed_tracing_database_uri, :distributed_tracing_database_table,
-      :distributes_tracing_mapper, :database_error_handler, :error_notification_service
+      :distributes_tracing_mapper, :database_error_handler, :error_notification_service, :producer_error_handler,
+      :producer_error_handler_job_class, :producer_retryable
 
     def configure_hutch
       yield hutch
@@ -38,7 +39,7 @@ module Hermes
     end
 
     def distributes_tracing_mapper=(mapper)
-      raise ArgumentError.new("mapper must espond to :call method") if !mapper.respond_to?(:call)
+      raise ArgumentError.new("mapper must respond to :call method") if !mapper.respond_to?(:call)
       @distributes_tracing_mapper = mapper
     end
 
@@ -54,6 +55,24 @@ module Hermes
       @database_error_handler || Hermes::DatabaseErrorHandler.new(error_notification_service: error_notification_service)
     end
 
+    def producer_error_handler
+      @producer_error_handler || Hermes::ProducerErrorHandler::NullHandler
+    end
+
+    def producer_retryable
+      @producer_retryable || Hermes::Retryable.new(times: 3, errors: [StandardError])
+    end
+
+    def enable_safe_producer(producer_error_handler_job_class)
+      self.producer_error_handler_job_class = producer_error_handler_job_class
+
+      @producer_error_handler = Hermes::ProducerErrorHandler::SafeHandler.new(
+        job_class: producer_error_handler_job_class,
+        error_notifier: error_notification_service,
+        retryable: producer_retryable
+      )
+    end
+
     class HutchConfig
       attr_accessor :uri
     end

data/lib/hermes/dependencies_container.rb

@@ -64,5 +64,17 @@ module Hermes
     def self.database_error_handler
       config.database_error_handler
     end
+
+    def self.producer_error_handler
+      config.producer_error_handler
+    end
+
+    def self.producer_retryable
+      config.producer_retryable
+    end
+
+    def self.objects_resolver
+      Object
+    end
   end
 end

data/lib/hermes/event_producer.rb

@@ -2,8 +2,8 @@ module Hermes
   class EventProducer
     extend Forwardable
 
-    attr_reader :publisher, :serializer, :distributed_trace_repository, :config
-    private     :publisher, :serializer, :distributed_trace_repository, :config
+    attr_reader :publisher, :serializer, :distributed_trace_repository, :producer_error_handler, :config
+    private     :publisher, :serializer, :distributed_trace_repository, :producer_error_handler, :config
 
     def self.publish(event, properties = {}, options = {})
       build.publish(event, properties, options)
@@ -14,19 +14,23 @@ module Hermes
         publisher: Hermes::DependenciesContainer["publisher"],
         serializer: Hermes::DependenciesContainer["serializer"],
         distributed_trace_repository: Hermes::DependenciesContainer["distributed_trace_repository"],
+        producer_error_handler: Hermes::DependenciesContainer["producer_error_handler"],
         config: Hermes::DependenciesContainer["config"]
       )
     end
 
-    def initialize(publisher:, serializer:, distributed_trace_repository:, config:)
+    def initialize(publisher:, serializer:, distributed_trace_repository:, producer_error_handler:, config:)
       @publisher = publisher
       @serializer = serializer
       @distributed_trace_repository = distributed_trace_repository
+      @producer_error_handler = producer_error_handler
       @config = config
     end
 
     def publish(event, properties = {}, options = {})
-      publish_event(event, properties, options).tap { store_trace(event) }
+      producer_error_handler.call(event) do
+        publish_event(event, properties, options).tap { store_trace(event) }
+      end
     end
 
     private

data/lib/hermes/producer_error_handler.rb

@@ -0,0 +1,4 @@
+module Hermes
+  module ProducerErrorHandler
+  end
+end

data/lib/hermes/producer_error_handler/null_handler.rb

@@ -0,0 +1,9 @@
+module Hermes
+  module ProducerErrorHandler
+    class NullHandler
+      def self.call(*)
+        yield
+      end
+    end
+  end
+end

data/lib/hermes/producer_error_handler/safe_handler.rb

@@ -0,0 +1,21 @@
+module Hermes
+  module ProducerErrorHandler
+    class SafeHandler
+      attr_reader :job_class, :error_notifier, :retryable
+      private     :job_class, :error_notifier, :retryable
+
+      def initialize(job_class:, error_notifier:, retryable:)
+        @job_class = job_class
+        @error_notifier = error_notifier
+        @retryable = retryable
+      end
+
+      def call(event)
+        retryable.perform { yield }
+      rescue => error
+        error_notifier.capture_exception(error)
+        job_class.enqueue(event.class.name, event.as_json, event.origin_headers)
+      end
+    end
+  end
+end

data/lib/hermes/publisher_factory.rb

@@ -1,7 +1,6 @@
 require "hermes/publisher/hutch_adapter"
 require "hermes/publisher/in_memory_adapter"
 
-
 module Hermes
   class PublisherFactory
     def self.build(adapter)

data/lib/hermes/rb/version.rb

@@ -1,5 +1,5 @@
 module Hermes
   module Rb
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/hermes/retryable.rb

@@ -0,0 +1,27 @@
+module Hermes
+  class Retryable
+    attr_reader :times, :errors, :before_retry
+    private     :times, :errors, :before_retry
+
+    def initialize(times:, errors: [], before_retry: ->(_error) {})
+      @times = times
+      @errors = errors
+      @before_retry = before_retry
+    end
+
+    def perform
+      executed = 0
+      begin
+        executed += 1
+        yield
+      rescue *errors => error
+        if executed < times
+          before_retry.call(error)
+          retry
+        else
+          raise error
+        end
+      end
+    end
+  end
+end

data/lib/hermes/retryable_event_producer.rb

@@ -0,0 +1,24 @@
+module Hermes
+  class RetryableEventProducer
+    def self.publish(event_class, event_body, origin_headers)
+      new(
+        objects_resolver: Hermes::DependenciesContainer["objects_resolver"],
+        event_producer: Hermes::DependenciesContainer["event_producer"]
+      ).publish(event_class, event_body, origin_headers)
+    end
+
+    attr_reader :objects_resolver, :event_producer
+    private     :objects_resolver, :event_producer
+
+    def initialize(objects_resolver:, event_producer:)
+      @objects_resolver = objects_resolver
+      @event_producer = event_producer
+    end
+
+    def publish(event_class, event_body, origin_headers)
+      event = objects_resolver.const_get(event_class).new(event_body.deep_symbolize_keys)
+      event.origin_headers = origin_headers.except(Hermes::B3PropagationModelHeaders.span_id_key)
+      event_producer.publish(event)
+    end
+  end
+end

data/lib/hermes/trace_context.rb

@@ -12,7 +12,7 @@ module Hermes
     end
 
     def trace
-      @trace ||= origin_event_headers.fetch("X-B3-TraceId", SecureRandom.hex(32))
+      @trace ||= origin_event_headers.fetch(Hermes::B3PropagationModelHeaders.trace_id_key, SecureRandom.hex(32))
     end
 
     def span
@@ -20,7 +20,7 @@ module Hermes
     end
 
     def parent_span
-      origin_event_headers.fetch("X-B3-SpanId", nil)
+      origin_event_headers.fetch(Hermes::B3PropagationModelHeaders.span_id_key, nil)
     end
 
     def service

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hermes-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-01 00:00:00.000000000 Z
+date: 2021-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -214,12 +214,17 @@ files:
 - lib/hermes/logger.rb
 - lib/hermes/null_error_notification_service.rb
 - lib/hermes/null_instrumenter.rb
+- lib/hermes/producer_error_handler.rb
+- lib/hermes/producer_error_handler/null_handler.rb
+- lib/hermes/producer_error_handler/safe_handler.rb
 - lib/hermes/publisher.rb
 - lib/hermes/publisher/hutch_adapter.rb
 - lib/hermes/publisher/in_memory_adapter.rb
 - lib/hermes/publisher_factory.rb
 - lib/hermes/rb.rb
 - lib/hermes/rb/version.rb
+- lib/hermes/retryable.rb
+- lib/hermes/retryable_event_producer.rb
 - lib/hermes/rpc_client.rb
 - lib/hermes/serializer.rb
 - lib/hermes/support/matchers/publish_async_message.rb
@@ -247,7 +252,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: A messenger of gods, delivering them via RabbitMQ with a little help from