hermes-rb 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 104554076c3b4ad46491c12d842e3580fdf8c96ba36eefbfc2f4c66a5ba5f208
-  data.tar.gz: a5f1754803e51907feda1164ff25c048731037f4cd75f2a6eaf4db68668b6647
+  metadata.gz: cc074c27cdac75e904004b2bd84d3d633f669251f5476065a9691979df5334da
+  data.tar.gz: 0255b07f12767b699b9bffb006ef00f8fc782f72c394bc5ab0622089441cf43b
 SHA512:
-  metadata.gz: aea75486bab7ba279b788dc48ea465f3ecb96c7281d9b6cc9bc29443c738ba779fe0951f6d0a8028211cce4f64520556f085cab73aee7ba98bac01af9dc5a973
-  data.tar.gz: 10b79dce9ad5c116d6ccb31a6102b1e3732a2b0a73e328b5ef37b421e5407cc8a5cc240e6b68c11b7fd26bd2c06acb579a610ba3708d83515d5c17dc48207f50
+  metadata.gz: 07d4527111544947adfb5222634df4b8a6f02169e612174a1de52723eeafc51a30daf3515b341f532244bf87e106cb204b6f7b5142f043e8aa24917a8e6374bc
+  data.tar.gz: 9d2f1c37b34b417c1fc9340590546f38608d67af7d1efb77d8a8853ea9827e0b6a19b43cfaad465867be52032bffe1446a2ef9280253d12896d124fbde6e0502

data/Changelog.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Master
+
+## 0.3.0
+- Reorganize Hutch config - instead of setting config values when connecting to Hutch when using publisher, do it right after the initialization.
+- Make params filter customizable for the logger
+- Remove sensitive info when creating distributed trace records.
+
 ## 0.2.0
 - Implement "Safe Producer" extension allowing to retry delivery later in an automated way when publishing of the message fails.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hermes-rb (0.2.0)
+    hermes-rb (0.3.0)
       activerecord (>= 5)
       activesupport (>= 5)
       dry-container (~> 0)
@@ -12,12 +12,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.1)
-      activesupport (= 6.1.1)
-    activerecord (6.1.1)
-      activemodel (= 6.1.1)
-      activesupport (= 6.1.1)
-    activesupport (6.1.1)
+    activemodel (6.1.2.1)
+      activesupport (= 6.1.2.1)
+    activerecord (6.1.2.1)
+      activemodel (= 6.1.2.1)
+      activesupport (= 6.1.2.1)
+    activesupport (6.1.2.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,7 +28,7 @@ GEM
       amq-protocol (~> 2.3, >= 2.3.1)
     carrot-top (0.0.7)
       json
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.8)
     diff-lcs (1.4.4)
     dry-configurable (0.12.0)
       concurrent-ruby (~> 1.0)
@@ -38,21 +38,18 @@ GEM
       dry-configurable (~> 0.1, >= 0.1.3)
     dry-core (0.5.0)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.3.0)
     dry-inflector (0.2.0)
     dry-logic (1.1.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.5, >= 0.5)
-    dry-struct (1.3.0)
-      dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.3)
-      dry-types (~> 1.3)
+    dry-struct (1.4.0)
+      dry-core (~> 0.5, >= 0.5)
+      dry-types (~> 1.5)
       ice_nine (~> 0.11)
-    dry-types (1.4.0)
+    dry-types (1.5.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
-      dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.3)
+      dry-core (~> 0.5, >= 0.5)
       dry-inflector (~> 0.1, >= 0.1.2)
       dry-logic (~> 1.0, >= 1.0.2)
     hutch (1.0.0)
@@ -60,12 +57,13 @@ GEM
       bunny (>= 2.15, < 2.16)
       carrot-top (~> 0.0.7)
       multi_json (~> 1.14)
-    i18n (1.8.7)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     json (2.5.1)
     minitest (5.14.3)
     multi_json (1.15.0)
+    newrelic_rpm (6.15.0)
     pg (1.2.3)
     rack (2.2.3)
     rake (13.0.1)
@@ -96,6 +94,7 @@ PLATFORMS
 DEPENDENCIES
   bundler
   hermes-rb!
+  newrelic_rpm
   pg
   rake
   rspec

data/README.md

@@ -36,6 +36,8 @@ Rails.application.config.to_prepare do
     config.instrumenter = Instrumenter
     config.configure_hutch do |hutch|
       hutch.uri = ENV.fetch("HUTCH_URI")
+      hutch.force_publisher_confirms = true
+      hutch.enable_http_api_use = false 
     end
     config.distributed_tracing_database_uri = ENV.fetch("DISTRIBUTED_TRACING_DATABASE_URI", nil)
     config.error_notification_service = Raven
@@ -79,7 +81,10 @@ If you know what you are doing, you don't necessarily have to process things in
 
 5. `event_handler` - an instance of event handler/storage, just use what is shown in the example.
 6. `clock` - a clock object that is time-zone aware, implementing `now` method.
-7. `configure_hutch` - a way to specify `hutch uri`, basically the URI for RabbitMQ.
+7. `configure_hutch` - a way to configure Hutch:
+   - `uri` - the URI for RabbitMQ, required.
+   - `force_publisher_confirms` - defaults to `true`
+   - `enable_http_api_use` - defaults to `false`
 8. `event_handler.handle_events` - that's how you declare events and their handlers. The event handler is an object that responds to `call` method and takes `event` as an argument. All events should ideally be subclasses of `Hermes::BaseEvent`
 
 This class inherits from `Dry::Struct`, so getting familiar with [dry-struct gem](https://dry-rb.org/gems/dry-struct/) would be beneficial. Here is an example event:
@@ -134,7 +139,7 @@ If you don't care about it, you can leave it empty.
 
 12. `distributed_tracing_database_table` - Table name for storing traces, by default it's `hermes_distributed_traces`. Optional.
 
-13. `distributes_tracing_mapper` - an object responding to `call` method taking one argument (a hash of attributes) that has to return a hash as well. This hash will be used for assigning attributes when creating `Hermes::DistributedTrace`. The default mapper just returns the original hash. You can use it if you want to remove, for example, some sensitive info from the event's body. Optional.
+13. `distributes_tracing_mapper` - an object responding to `call` method taking one argument (a hash of attributes) which must return a hash as well. This hash will be used for assigning attributes when creating `Hermes::DistributedTrace`. It defaults to `Hermes::DistributedTrace::Mapper`, which uses `logger_params_filter` to remove sensitive info (this config option is covered below). You can either provide a custom mapper or pass a custom params filter, for example: `Hermes::DistributedTrace::Mapper.new(params_filter: custom_params_filter)` 
 
 14. `error_notification_service` - an object responding to `capture_exception` method taking one argument (error). Its interface is based on `Raven` from [Sentry Raven](https://github.com/getsentry/sentry-ruby/tree/master/sentry-raven). By default `Hermes::NullErrorNotificationService` is used, which does nothing. Optional.
 
@@ -144,6 +149,9 @@ If you don't care about it, you can leave it empty.
 
 17. `producer_retryable` - used when `safe_producer` was enabled via (`enable_safe_producer`). By default, it is a method retrying delivery 3 times rescuing from `StandardError` each time. The object responsible for this behavior by default is: `Hermes::Retryable.new(times: 3, errors: [StandardError])`.
 
+18. `logger_params_filter` - a service used as params filter for logger, to make sure no sensitive data will be logged. It defaults to `Hermes::Logger::ParamsFilter` which already performs some filtering but it might not be enough in your case. If you are not satisfied with the defaults, you have 2 options, which are especially simple in Rails apps:
+   - provide custom array of sensitive attributes and still use a default filter: `Hermes::Logger::ParamsFilter.new(sensitive_keywords: Rails.application.config.filter_parameters)`.
+   - provide custom filter object, which responds to `call` method and takes 2 arguments: attribute name and its value and performs mutation by using `gsub!` (don't worry, the entire body is cloned before passing it to the filter, so nothing unexpected will happen). This is compatible with the interface of `Rails.application.config.filter_parameters` when you use a custom filter there. In such case, you can do something like this: `Rails.application.config.filter_parameters = [Proc.new { |k, v| do_something_custom_here(k, v) }]` and then just assign `Rails.application.config.filter_parameters.first` in the Hermes config.  
 ## RPC
 
 If you want to handle RPC call, you need to add `rpc: true` flag. Keep in mind that RPC requires a synchronous processing and response, so you also need to set `async: false`. The routing key and correlation ID will be resolved based on the message that is published by the client. The payload that is sent back will be what event handler reutrns, so it might be a good idea to just return a hash so that you can operate on JSON easily.
@@ -172,6 +180,10 @@ parsed_response_hash = Hermes::RpcClient.new(rpc_call_timeout: 10).call(event)
 
 If the request timeouts, `Hermes::RpcClient::RpcTimeoutError` will be raised.
 
+## NewRelic integration
+
+The integration is enabled automatically if you use `newrelic_rpm` gem via `Hutch::Tracers::NewRelic`.
+
 ## Distributed Tracing (experimental feature, the interface might change in the future)
 
 If you want to take advantage of distributed tracing, you need to specify `distributed_tracing_database_uri` in the config and in many cases that will be enough, although there are some cases where some extra code will be required to properly use it.
@@ -240,8 +252,7 @@ To take advantage of this feature, apply the following logic in the initializer
 
 ``` rb
 Hermes.configure do |config|
-  config.clock = clock
-  config.error_notification_service = Raven
+  config.error_notification_service = Raven # required for this use case
   config.enable_safe_producer(HermesRecoveryJob)
 end
 ```

data/hermes-rb.gemspec

@@ -40,4 +40,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "pg"
+  spec.add_development_dependency "newrelic_rpm"
 end

data/lib/hermes.rb

@@ -11,6 +11,7 @@ require "hermes/serializer"
 require "hermes/rpc_client"
 require "hermes/null_instrumenter"
 require "hermes/logger"
+require "hermes/logger/params_filter"
 require "hermes/rb"
 require "hermes/b_3_propagation_model_headers"
 require "hermes/trace_context"
@@ -50,3 +51,4 @@ module Hermes
 end
 
 require "hermes/distributed_trace"
+require "hermes/distributed_trace/mapper"

data/lib/hermes/configuration.rb

@@ -4,10 +4,11 @@ module Hermes
       :background_processor, :enqueue_method, :event_handler, :rpc_call_timeout,
       :instrumenter, :distributed_tracing_database_uri, :distributed_tracing_database_table,
       :distributes_tracing_mapper, :database_error_handler, :error_notification_service, :producer_error_handler,
-      :producer_error_handler_job_class, :producer_retryable
+      :producer_error_handler_job_class, :producer_retryable, :logger_params_filter
 
     def configure_hutch
       yield hutch
+      hutch.commit_config
     end
 
     def self.configure
@@ -44,7 +45,7 @@ module Hermes
     end
 
     def distributes_tracing_mapper
-      @distributes_tracing_mapper || ->(attributes) { attributes }
+      @distributes_tracing_mapper || Hermes::DistributedTrace::Mapper.new
     end
 
     def error_notification_service
@@ -73,8 +74,37 @@ module Hermes
       )
     end
 
+    def logger_params_filter
+      @logger_params_filter || Hermes::Logger::ParamsFilter.new
+    end
+
     class HutchConfig
-      attr_accessor :uri
+      attr_reader :original_hutch_config
+      private     :original_hutch_config
+
+      attr_accessor :uri, :force_publisher_confirms, :enable_http_api_use
+
+      def initialize(original_hutch_config: Hutch::Config)
+        @original_hutch_config = original_hutch_config
+      end
+
+      def commit_config
+        original_hutch_config.set(:tracer, Hutch::Tracers::NewRelic) if Object.const_defined?("NewRelic")
+        original_hutch_config.set(:force_publisher_confirms, force_publisher_confirms)
+        original_hutch_config.set(:uri, uri)
+      end
+
+      def force_publisher_confirms
+        return @force_publisher_confirms if defined?(@force_publisher_confirms)
+
+        @force_publisher_confirms ||= true
+      end
+
+      def enable_http_api_use
+        return @enable_http_api_use if defined?(@enable_http_api_use)
+
+        @enable_http_api_use ||= false
+      end
     end
     private_constant :HutchConfig
   end

data/lib/hermes/dependencies_container.rb

@@ -73,6 +73,14 @@ module Hermes
       config.producer_retryable
     end
 
+    def self.hutch_logger
+      Hutch.logger
+    end
+
+    def self.logger_params_filter
+      config.logger_params_filter
+    end
+
     def self.objects_resolver
       Object
     end

data/lib/hermes/distributed_trace/mapper.rb

@@ -0,0 +1,18 @@
+module Hermes
+  class DistributedTrace
+    class Mapper
+      attr_reader :params_filter
+      private     :params_filter
+
+      def initialize(params_filter: Hermes::DependenciesContainer["logger_params_filter"])
+        @params_filter = params_filter
+      end
+
+      def call(attributes)
+        attributes.deep_dup.tap do |attributes_copy|
+          attributes_copy.each { |key, value| params_filter.call(key, value) }
+        end
+      end
+    end
+  end
+end

data/lib/hermes/logger.rb

@@ -1,15 +1,12 @@
 module Hermes
   class Logger
-    SENSITIVE_ATTRIBUTES_KEYWORDS = %w(token password credit_card).freeze
-    STRIPPED_VALUE = "[STRIPPED]".freeze
+    attr_reader :backend, :logger_params_filter
+    private     :backend, :logger_params_filter
 
-    private_constant :SENSITIVE_ATTRIBUTES_KEYWORDS, :STRIPPED_VALUE
-
-    attr_reader :backend
-    private     :backend
-
-    def initialize(backend: Hutch.logger)
+    def initialize(backend: Hermes::DependenciesContainer["hutch_logger"],
+    logger_params_filter: Hermes::DependenciesContainer["logger_params_filter"])
       @backend = backend
+      @logger_params_filter = logger_params_filter
     end
 
     def log_enqueued(event_class, body, headers, timestamp)
@@ -23,13 +20,9 @@ module Hermes
     private
 
     def strip_sensitive_info(body)
-      body.stringify_keys.map do |attribute, value|
-        if SENSITIVE_ATTRIBUTES_KEYWORDS.any? { |sensitive_attribute| attribute.match(sensitive_attribute) }
-          [attribute, STRIPPED_VALUE]
-        else
-          [attribute, value]
-        end
-      end.to_h
+      body.deep_dup.tap do |body_copy|
+        body_copy.each { |key, value| logger_params_filter.call(key, value) }
+      end
     end
   end
 end

data/lib/hermes/logger/params_filter.rb

@@ -0,0 +1,25 @@
+module Hermes
+  class Logger
+    class ParamsFilter
+      SENSITIVE_ATTRIBUTES_KEYWORDS = %w(token password credit_card card_number security_code verification_value
+        private_key signature api_key secret_key publishable_key).freeze
+      STRIPPED_VALUE = "[STRIPPED]".freeze
+
+      private_constant :SENSITIVE_ATTRIBUTES_KEYWORDS, :STRIPPED_VALUE
+
+      attr_reader :sensitive_keywords, :stripped_value
+      private     :sensitive_keywords, :stripped_value
+
+      def initialize(sensitive_keywords: SENSITIVE_ATTRIBUTES_KEYWORDS, stripped_value: STRIPPED_VALUE)
+        @sensitive_keywords = sensitive_keywords.map(&:to_s)
+        @stripped_value = stripped_value
+      end
+
+      def call(attribute, value)
+        if sensitive_keywords.any? { |sensitive_attribute| attribute.to_s.match(sensitive_attribute) }
+          value.gsub!(value, stripped_value)
+        end
+      end
+    end
+  end
+end

data/lib/hermes/publisher.rb

@@ -1,19 +1,19 @@
 require "forwardable"
 require "singleton"
 
-
 module Hermes
   class Publisher
     include Singleton
     extend Forwardable
 
-    attr_reader :mutex
-    private     :mutex
+    attr_reader :configuration, :mutex
+    private     :configuration, :mutex
 
     def_delegators :current_adapter, :publish
 
-    def initialize
-      super
+    def initialize(configuration: Hermes::DependenciesContainer["config"])
+      super()
+      @configuration = configuration
       @mutex = Mutex.new
     end
 
@@ -34,15 +34,5 @@ module Hermes
         end
       end
     end
-
-    def connect
-      current_adapter.class.connect
-    end
-
-    private
-
-    def configuration
-      Hermes.configuration
-    end
   end
 end

data/lib/hermes/publisher/hutch_adapter.rb

@@ -3,10 +3,7 @@ require "hutch"
 module Hermes
   class Publisher::HutchAdapter
     def self.connect(configuration: Hermes::DependenciesContainer["hutch_config"])
-      Hutch::Config.set(:uri, configuration.uri)
-      Hutch::Config.set(:force_publisher_confirms, true)
-      Hutch::Config.set(:tracer, Hutch::Tracers::NewRelic) if Object.const_defined?("NewRelic")
-      Hutch.connect(enable_http_api_use: false)
+      Hutch.connect(enable_http_api_use: configuration.enable_http_api_use)
     end
 
     def initialize(configuration: Hermes::DependenciesContainer["hutch_config"])

data/lib/hermes/rb/version.rb

@@ -1,5 +1,5 @@
 module Hermes
   module Rb
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hermes-rb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-13 00:00:00.000000000 Z
+date: 2021-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: newrelic_rpm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A messenger of gods, delivering them via RabbitMQ with a little help
   from Hutch
 email:
@@ -207,11 +221,13 @@ files:
 - lib/hermes/database_error_handler.rb
 - lib/hermes/dependencies_container.rb
 - lib/hermes/distributed_trace.rb
+- lib/hermes/distributed_trace/mapper.rb
 - lib/hermes/distributed_trace_repository.rb
 - lib/hermes/event_handler.rb
 - lib/hermes/event_processor.rb
 - lib/hermes/event_producer.rb
 - lib/hermes/logger.rb
+- lib/hermes/logger/params_filter.rb
 - lib/hermes/null_error_notification_service.rb
 - lib/hermes/null_instrumenter.rb
 - lib/hermes/producer_error_handler.rb