hermes-rb 0.3.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc074c27cdac75e904004b2bd84d3d633f669251f5476065a9691979df5334da
-  data.tar.gz: 0255b07f12767b699b9bffb006ef00f8fc782f72c394bc5ab0622089441cf43b
+  metadata.gz: 8d9a8e1af4d989ab98014db38015a8a00f24c2c7a4ab0d7724c849e092812855
+  data.tar.gz: 7038b16132b50567e00ebd2398054907f8e17dccc6e15652a7367f1dbd76299e
 SHA512:
-  metadata.gz: 07d4527111544947adfb5222634df4b8a6f02169e612174a1de52723eeafc51a30daf3515b341f532244bf87e106cb204b6f7b5142f043e8aa24917a8e6374bc
-  data.tar.gz: 9d2f1c37b34b417c1fc9340590546f38608d67af7d1efb77d8a8853ea9827e0b6a19b43cfaad465867be52032bffe1446a2ef9280253d12896d124fbde6e0502
+  metadata.gz: 432bfa0092f9dbeaad563aadea388c542a8daf06c4708dca3df620806b8c678350c88b27838d2c6fd06eda9ce84398c5eb3d770efca21eff63a528161e46a3d9
+  data.tar.gz: ba182e98338361a3024829c2e7872bf935d28769a3c69f373ecbd4a3f6676040f6a2ecd3b2753f7683e0e799484dbbd178d695679a36fb0440a25575015f12f3

data/Changelog.md

@@ -2,6 +2,22 @@
 
 ## Master
 
+## 0.6.1
+- Extend the list of sensitive keywords for Hermes::Logger::ParamsFilter
+
+## 0.6.0
+- Add Health check
+
+## 0.5.0
+- Add Datadog tracer
+- Allow to provide a custom tracer
+
+## 0.4.0
+- Make it possible to provide Hutch consumer config
+
+## 0.3.1
+- Fix filtering params for logs for nil values in sensitive attributes
+
 ## 0.3.0
 - Reorganize Hutch config - instead of setting config values when connecting to Hutch when using publisher, do it right after the initialization.
 - Make params filter customizable for the logger

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hermes-rb (0.3.0)
+    hermes-rb (0.6.1)
       activerecord (>= 5)
       activesupport (>= 5)
       dry-container (~> 0)
@@ -12,12 +12,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activerecord (6.1.2.1)
-      activemodel (= 6.1.2.1)
-      activesupport (= 6.1.2.1)
-    activesupport (6.1.2.1)
+    activemodel (6.1.3.2)
+      activesupport (= 6.1.3.2)
+    activerecord (6.1.3.2)
+      activemodel (= 6.1.3.2)
+      activesupport (= 6.1.3.2)
+    activesupport (6.1.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,25 +28,27 @@ GEM
       amq-protocol (~> 2.3, >= 2.3.1)
     carrot-top (0.0.7)
       json
-    concurrent-ruby (1.1.8)
+    concurrent-ruby (1.1.9)
+    ddtrace (0.45.0)
+      msgpack
     diff-lcs (1.4.4)
-    dry-configurable (0.12.0)
+    dry-configurable (0.12.1)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5.0)
-    dry-container (0.7.2)
+    dry-container (0.8.0)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.5.0)
+    dry-core (0.6.0)
       concurrent-ruby (~> 1.0)
     dry-inflector (0.2.0)
-    dry-logic (1.1.0)
+    dry-logic (1.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
     dry-struct (1.4.0)
       dry-core (~> 0.5, >= 0.5)
       dry-types (~> 1.5)
       ice_nine (~> 0.11)
-    dry-types (1.5.0)
+    dry-types (1.5.1)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.5, >= 0.5)
@@ -57,11 +59,12 @@ GEM
       bunny (>= 2.15, < 2.16)
       carrot-top (~> 0.0.7)
       multi_json (~> 1.14)
-    i18n (1.8.9)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     json (2.5.1)
-    minitest (5.14.3)
+    minitest (5.14.4)
+    msgpack (1.4.2)
     multi_json (1.15.0)
     newrelic_rpm (6.15.0)
     pg (1.2.3)
@@ -93,6 +96,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler
+  ddtrace
   hermes-rb!
   newrelic_rpm
   pg

data/README.md

@@ -37,7 +37,8 @@ Rails.application.config.to_prepare do
     config.configure_hutch do |hutch|
       hutch.uri = ENV.fetch("HUTCH_URI")
       hutch.force_publisher_confirms = true
-      hutch.enable_http_api_use = false 
+      hutch.enable_http_api_use = false
+      hutch.tracer = MyOwnCustomTracerIfIWantToDoSomethingCrazy
     end
     config.distributed_tracing_database_uri = ENV.fetch("DISTRIBUTED_TRACING_DATABASE_URI", nil)
     config.error_notification_service = Raven
@@ -45,8 +46,15 @@ Rails.application.config.to_prepare do
 
   event_handler.handle_events do
     handle Events::Example::Happened, with: Example::HappenedHandler
-
     handle Events::Example::SyncCallHappened, with: Example::SyncCallHappenedHandler, async: false
+
+    extra_consumer_config = -> do
+      classic_queue
+      quorum_queue initial_group_size: 3
+      arguments "x-max-length" => 10
+    end
+    handle Events::Example::SomethingHappenedWithExtraConsumerConfig, with: Example::SomethingHappenedWithExtraConsumerConfigHandler,
+      consumer_config: extra_consumer_config
   end
 
   # if you care about distributed tracing
@@ -79,12 +87,13 @@ end
 
 If you know what you are doing, you don't necessarily have to process things in the background. As long as the class implements the expected interface, you can do anything you want.
 
-5. `event_handler` - an instance of event handler/storage, just use what is shown in the example.
+5. `event_handler` - an instance of event handler/storage, just use what is shown in the example. Notice that you can also pass extra consumer config lambda that will be evaluated within the context of Hutch consumer.
 6. `clock` - a clock object that is time-zone aware, implementing `now` method.
 7. `configure_hutch` - a way to configure Hutch:
    - `uri` - the URI for RabbitMQ, required.
    - `force_publisher_confirms` - defaults to `true`
    - `enable_http_api_use` - defaults to `false`
+   - `tracer` - defaults to `Hermes::Tracers::Datadog` if you use Datadog, `Hutch::Tracers::NewRelic` for NewRelic and `Hutch::Tracers::NullTracer` if you use neither Datadog, nor NewRelic. Check APM section for more details if you want to provide a custom tracer.
 8. `event_handler.handle_events` - that's how you declare events and their handlers. The event handler is an object that responds to `call` method and takes `event` as an argument. All events should ideally be subclasses of `Hermes::BaseEvent`
 
 This class inherits from `Dry::Struct`, so getting familiar with [dry-struct gem](https://dry-rb.org/gems/dry-struct/) would be beneficial. Here is an example event:
@@ -139,7 +148,7 @@ If you don't care about it, you can leave it empty.
 
 12. `distributed_tracing_database_table` - Table name for storing traces, by default it's `hermes_distributed_traces`. Optional.
 
-13. `distributes_tracing_mapper` - an object responding to `call` method taking one argument (a hash of attributes) which must return a hash as well. This hash will be used for assigning attributes when creating `Hermes::DistributedTrace`. It defaults to `Hermes::DistributedTrace::Mapper`, which uses `logger_params_filter` to remove sensitive info (this config option is covered below). You can either provide a custom mapper or pass a custom params filter, for example: `Hermes::DistributedTrace::Mapper.new(params_filter: custom_params_filter)` 
+13. `distributes_tracing_mapper` - an object responding to `call` method taking one argument (a hash of attributes) which must return a hash as well. This hash will be used for assigning attributes when creating `Hermes::DistributedTrace`. It defaults to `Hermes::DistributedTrace::Mapper`, which uses `logger_params_filter` to remove sensitive info (this config option is covered below). You can either provide a custom mapper or pass a custom params filter, for example: `Hermes::DistributedTrace::Mapper.new(params_filter: custom_params_filter)`
 
 14. `error_notification_service` - an object responding to `capture_exception` method taking one argument (error). Its interface is based on `Raven` from [Sentry Raven](https://github.com/getsentry/sentry-ruby/tree/master/sentry-raven). By default `Hermes::NullErrorNotificationService` is used, which does nothing. Optional.
 
@@ -151,7 +160,7 @@ If you don't care about it, you can leave it empty.
 
 18. `logger_params_filter` - a service used as params filter for logger, to make sure no sensitive data will be logged. It defaults to `Hermes::Logger::ParamsFilter` which already performs some filtering but it might not be enough in your case. If you are not satisfied with the defaults, you have 2 options, which are especially simple in Rails apps:
    - provide custom array of sensitive attributes and still use a default filter: `Hermes::Logger::ParamsFilter.new(sensitive_keywords: Rails.application.config.filter_parameters)`.
-   - provide custom filter object, which responds to `call` method and takes 2 arguments: attribute name and its value and performs mutation by using `gsub!` (don't worry, the entire body is cloned before passing it to the filter, so nothing unexpected will happen). This is compatible with the interface of `Rails.application.config.filter_parameters` when you use a custom filter there. In such case, you can do something like this: `Rails.application.config.filter_parameters = [Proc.new { |k, v| do_something_custom_here(k, v) }]` and then just assign `Rails.application.config.filter_parameters.first` in the Hermes config.  
+   - provide custom filter object, which responds to `call` method and takes 2 arguments: attribute name and its value and performs mutation by using `gsub!` (don't worry, the entire body is cloned before passing it to the filter, so nothing unexpected will happen). This is compatible with the interface of `Rails.application.config.filter_parameters` when you use a custom filter there. In such case, you can do something like this: `Rails.application.config.filter_parameters = [Proc.new { |k, v| do_something_custom_here(k, v) }]` and then just assign `Rails.application.config.filter_parameters.first` in the Hermes config.
 ## RPC
 
 If you want to handle RPC call, you need to add `rpc: true` flag. Keep in mind that RPC requires a synchronous processing and response, so you also need to set `async: false`. The routing key and correlation ID will be resolved based on the message that is published by the client. The payload that is sent back will be what event handler reutrns, so it might be a good idea to just return a hash so that you can operate on JSON easily.
@@ -180,9 +189,11 @@ parsed_response_hash = Hermes::RpcClient.new(rpc_call_timeout: 10).call(event)
 
 If the request timeouts, `Hermes::RpcClient::RpcTimeoutError` will be raised.
 
-## NewRelic integration
+## APM and tracing
 
-The integration is enabled automatically if you use `newrelic_rpm` gem via `Hutch::Tracers::NewRelic`.
+The integration is enabled automatically if you use `newrelic_rpm` gem via `Hutch::Tracers::NewRelic` or via `Hermes::Tracers::Datadog` when using `ddtrace` gem.
+
+You can also provide your own tracer, as long as it implements an interface expected by [Hutch][https://github.com/ruby-amqp/hutch].
 
 ## Distributed Tracing (experimental feature, the interface might change in the future)
 
@@ -448,6 +459,33 @@ end
 
 Hermes is just an extra layer on top of [hutch](https://github.com/gocardless/hutch), refer to Hutch's docs for more info about dealing with the workers and deployment.
 
+## Health Checks
+
+If you want to perform a health check, use `Hermes::Checks::HealthCheck.check`, which checks if it's possible to connect RabbitMQ via Hutch.
+
+The interface is compliant with `health_check`[https://github.com/ianheggie/health_check] gem. If you want to add the custom health check, just add this to the config:
+
+``` rb
+config.add_custom_check("hermes") do
+  Hermes::Checks::HealthCheck.check
+end
+```
+
+You can also use `bin/health_check` file to perform healthcheck - on success, the script exits with `0` status and on failure, it logs the error and exits with `1` status.
+
+To perform the actual check:
+
+1. Via `health_check` gem:
+```
+curl -v localhost:3000/health_check/hermes.json
+```
+
+2. Via binary:
+
+```
+bundle exec hermes_health_check
+```
+
 ## CircleCI config for installing RabbitMQ
 
 Use `- image: brandembassy/rabbitmq:latest`

data/bin/hermes_health_check

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require_relative "../lib/hermes-rb"
+
+result = Hermes::Checks::HealthCheck.check
+if result.empty?
+  exit 0
+else
+  Hermes::DependenciesContainer["logger"].log_health_check_failure(result)
+  exit 1
+end

data/hermes-rb.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = ["hermes_health_check"]
   spec.require_paths = ["lib"]
 
   spec.add_dependency "dry-struct", "~> 1"
@@ -41,4 +41,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "pg"
   spec.add_development_dependency "newrelic_rpm"
+  spec.add_development_dependency "ddtrace"
 end

data/lib/hermes.rb

@@ -24,6 +24,8 @@ require "hermes/retryable_event_producer"
 require "hermes/producer_error_handler"
 require "hermes/producer_error_handler/null_handler"
 require "hermes/producer_error_handler/safe_handler"
+require "hermes/tracers"
+require "hermes/checks"
 require "active_support"
 require "active_support/core_ext/string"
 require "active_record"

data/lib/hermes/checks.rb

@@ -0,0 +1,5 @@
+module Hermes
+  module Checks
+    autoload :HealthCheck, "hermes/checks/health_check"
+  end
+end

data/lib/hermes/checks/health_check.rb

@@ -0,0 +1,14 @@
+module Hermes
+  module Checks
+    class HealthCheck
+      def self.check
+        broker = Hutch::Broker.new
+        broker.connect
+        broker.disconnect
+        ""
+      rescue StandardError => e
+        "[Hermes - #{e.message}] "
+      end
+    end
+  end
+end

data/lib/hermes/configuration.rb

@@ -4,7 +4,7 @@ module Hermes
       :background_processor, :enqueue_method, :event_handler, :rpc_call_timeout,
       :instrumenter, :distributed_tracing_database_uri, :distributed_tracing_database_table,
       :distributes_tracing_mapper, :database_error_handler, :error_notification_service, :producer_error_handler,
-      :producer_error_handler_job_class, :producer_retryable, :logger_params_filter
+      :producer_error_handler_job_class, :producer_retryable, :logger_params_filter, :tracer
 
     def configure_hutch
       yield hutch
@@ -82,14 +82,14 @@ module Hermes
       attr_reader :original_hutch_config
       private     :original_hutch_config
 
-      attr_accessor :uri, :force_publisher_confirms, :enable_http_api_use
+      attr_accessor :uri, :force_publisher_confirms, :enable_http_api_use, :tracer
 
       def initialize(original_hutch_config: Hutch::Config)
         @original_hutch_config = original_hutch_config
       end
 
       def commit_config
-        original_hutch_config.set(:tracer, Hutch::Tracers::NewRelic) if Object.const_defined?("NewRelic")
+        original_hutch_config.set(:tracer, tracer) if tracer
         original_hutch_config.set(:force_publisher_confirms, force_publisher_confirms)
         original_hutch_config.set(:uri, uri)
       end
@@ -105,6 +105,13 @@ module Hermes
 
         @enable_http_api_use ||= false
       end
+
+      def tracer
+        return @tracer if @tracer
+        return Hermes::Tracers::Datadog if Object.const_defined?("Datadog")
+        return Hutch::Tracers::NewRelic if Object.const_defined?("NewRelic")
+        Hutch::Tracers::NullTracer
+      end
     end
     private_constant :HutchConfig
   end

data/lib/hermes/consumer_builder.rb

@@ -2,7 +2,7 @@ require "hutch"
 
 module Hermes
   class ConsumerBuilder
-    def build(event_class)
+    def build(event_class, consumer_config: -> {})
       queue = queue_name_from_event(event_class)
       routing_key = event_class.routing_key
       consumer_name = consumer_name_from_event(event_class)
@@ -12,6 +12,7 @@ module Hermes
 
         consume routing_key
         queue_name queue
+        instance_exec(&consumer_config)
 
         define_method :process do |message|
           instrumenter.instrument("Hermes.Consumer.process") do

data/lib/hermes/dependencies_container.rb

@@ -84,5 +84,9 @@ module Hermes
     def self.objects_resolver
       Object
     end
+
+    def self.consumer_builder
+      Hermes::ConsumerBuilder.new
+    end
   end
 end

data/lib/hermes/event_handler.rb

@@ -5,25 +5,27 @@ module Hermes
     attr_reader :container, :consumer_builder
     private     :container, :consumer_builder
 
-    def initialize
-      @container = Dry::Container.new
-      @consumer_builder = Hermes::ConsumerBuilder.new
+    def initialize(container: Dry::Container.new, consumer_builder: Hermes::DependenciesContainer["consumer_builder"])
+      @container = container
+      @consumer_builder = consumer_builder
     end
 
     def handle_events(&block)
       instance_exec(&block)
     end
 
-    def handle(event_class, with:, async: true, rpc: false)
+    def handle(event_class, with:, async: true, rpc: false, consumer_config: -> {})
       handler = with
       options = {
         async: async,
-        rpc: rpc
+        rpc: rpc,
+        consumer_config: consumer_config
       }
-      consumer = build_consumer_for_event(event_class)
+      consumer = build_consumer_for_event(event_class, consumer_config)
 
-      registration = Registration.new(handler, consumer, options)
-      container.register(event_class, registration)
+      Registration.new(handler, consumer, options).tap do |registration|
+        container.register(event_class, registration)
+      end
     end
 
     def registration_for(event_class)
@@ -32,8 +34,8 @@ module Hermes
 
     private
 
-    def build_consumer_for_event(event_class)
-      consumer_builder.build(event_class)
+    def build_consumer_for_event(event_class, consumer_config)
+      consumer_builder.build(event_class, consumer_config: consumer_config)
     end
 
     class Registration
@@ -52,6 +54,10 @@ module Hermes
       def rpc?
         options.fetch(:rpc) == true
       end
+
+      def consumer_config
+        optons.fetch(:consumer_config)
+      end
     end
   end
 end

data/lib/hermes/logger.rb

@@ -17,6 +17,10 @@ module Hermes
       backend.info "[Hutch] published event to: #{routing_key}, properties: #{properties}, body: #{strip_sensitive_info(body)} at #{timestamp}"
     end
 
+    def log_health_check_failure(error)
+      backend.info "[Hermes] health check failed: #{error}"
+    end
+
     private
 
     def strip_sensitive_info(body)

data/lib/hermes/logger/params_filter.rb

@@ -2,7 +2,7 @@ module Hermes
   class Logger
     class ParamsFilter
       SENSITIVE_ATTRIBUTES_KEYWORDS = %w(token password credit_card card_number security_code verification_value
-        private_key signature api_key secret_key publishable_key).freeze
+        private_key signature api_key secret_key publishable_key client_key client_secret secret).freeze
       STRIPPED_VALUE = "[STRIPPED]".freeze
 
       private_constant :SENSITIVE_ATTRIBUTES_KEYWORDS, :STRIPPED_VALUE
@@ -16,7 +16,7 @@ module Hermes
       end
 
       def call(attribute, value)
-        if sensitive_keywords.any? { |sensitive_attribute| attribute.to_s.match(sensitive_attribute) }
+        if sensitive_keywords.any? { |sensitive_attribute| attribute.to_s.match(sensitive_attribute) } && value.respond_to?(:to_str)
           value.gsub!(value, stripped_value)
         end
       end

data/lib/hermes/rb/version.rb

@@ -1,5 +1,5 @@
 module Hermes
   module Rb
-    VERSION = "0.3.0"
+    VERSION = "0.6.1"
   end
 end

data/lib/hermes/tracers.rb

@@ -0,0 +1,5 @@
+module Hermes
+  module Tracers
+    autoload :Datadog, "hermes/tracers/datadog"
+  end
+end

data/lib/hermes/tracers/datadog.rb

@@ -0,0 +1,20 @@
+require "ddtrace"
+
+module Hermes
+  module Tracers
+    class Datadog
+      attr_reader :klass
+      private     :klass
+
+      def initialize(klass)
+        @klass = klass
+      end
+
+      def handle(message)
+        ::Datadog.tracer.trace(klass.class.name, service: "hermes", span_type: "rabbitmq") do
+          klass.process(message)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hermes-rb
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2021-02-15 00:00:00.000000000 Z
+date: 2021-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -192,11 +192,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ddtrace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A messenger of gods, delivering them via RabbitMQ with a little help
   from Hutch
 email:
 - dev@bookingsync.com
-executables: []
+executables:
+- hermes_health_check
 extensions: []
 extra_rdoc_files: []
 files:
@@ -210,12 +225,15 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/hermes_health_check
 - bin/setup
 - hermes-rb.gemspec
 - lib/hermes-rb.rb
 - lib/hermes.rb
 - lib/hermes/b_3_propagation_model_headers.rb
 - lib/hermes/base_event.rb
+- lib/hermes/checks.rb
+- lib/hermes/checks/health_check.rb
 - lib/hermes/configuration.rb
 - lib/hermes/consumer_builder.rb
 - lib/hermes/database_error_handler.rb
@@ -245,6 +263,8 @@ files:
 - lib/hermes/serializer.rb
 - lib/hermes/support/matchers/publish_async_message.rb
 - lib/hermes/trace_context.rb
+- lib/hermes/tracers.rb
+- lib/hermes/tracers/datadog.rb
 - untitled
 homepage: https://github.com/BookingSync/hermes-rb
 licenses: