kiev 4.4.0 → 4.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f25b13c7dfe2a466ce7f9b0f03f3b4078d70158d3241aa589568124ab8b5501
-  data.tar.gz: 3665a4859072ec37b624a89c218ec362e019a2994236901a2ce2dfc55a2a60ca
+  metadata.gz: 80613e919661165f768bd2a96b293c1908447e2cf936d7bf0f3b47e2338dec26
+  data.tar.gz: a49dffa5584e4a326ca21b10cdb6a82f091caf23a49d18241db3225343193ef8
 SHA512:
-  metadata.gz: 35a76a1baf47a1c857ef9c0a6b39f9adb513dd7366c372d6cbcb148980b25772759a529c82559af5779d61bf2eb51706016463fda47aaa85c8a0a3604d5667c0
-  data.tar.gz: b7830138907f5a65d062c3e5f2c7e2d3a9bced6595ac428b9e14ab47f11ebbef94abd35bd590a9dc5890e1045d8fdd86a85a643b1aa29c1dcb545d921045c2d6
+  metadata.gz: ddb0cc47d3f960a1e761ed9e6647a9f3048ef6a4ed85ddd46c4c21c1f78d4dd2975f3842172570fba2570f968a23107d1d28133c5cc0a82c171fcc8892fa16f3
+  data.tar.gz: c54860b71c585ef469c9f425601e55d282cedaafde4860bcf0f46474bab9b5b7fe4f6cef460729c11de7af61636c9b099f6c7865f5ac336d8b6d3301a2444d69

data/.github/workflows/push.yml

@@ -21,6 +21,7 @@ jobs:
       matrix:
         ruby: ['2.5.1', '2.7.2', '2.6.5']
         gemfile:
+          - gemfiles/ruby_kafka.gemfile
           - gemfiles/que_0.12.2.gemfile
           - gemfiles/que_0.12.3.gemfile
           - gemfiles/rails_5.2.gemfile

data/README.md

@@ -154,6 +154,36 @@ Kiev::Shoryuken.enable
 
 The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
 
+### AWS SNS
+
+To enhance messages published to SNS topics you can use the ContextInjector:
+
+```ruby
+sns_message = { topic_arn: "...",  message: "{...}" }
+Kiev::Kafka.inject_context(sns_message[:message_attributes])
+
+```
+
+After this operation the message attributes will also include required context for the Kiev logger.
+
+### Kafka
+
+To enhance messages published to Kafka topics you can use the ContextInjector:
+
+```ruby
+Kiev::Kafka.inject_context(headers)
+```
+
+After this operation the headers variable will also include required context for the Kiev logger.
+
+If you have a consumed `Kafka::FetchedMessage` you can extract logger context with:
+
+```ruby
+Kiev::Kafka.extract_context(message)
+```
+
+This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -219,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
 
 * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
+
+* `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`.
 
 * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 
@@ -285,6 +317,11 @@ Kiev.event(:my_event, { some_array: [1, 2, 3] })
 
 # Log other data types (will be available under the `message` key)
 Kiev.event(:my_event, "hello world")
+
+# Log with given severity [debug, info, warn, error, fatal]
+Kiev.info(:my_event)
+Kiev.info(:my_event, { some_array: [1, 2, 3] })
+Kiev.info(:my_event, "hello world")
 ```
 
 However, `Kiev.logger` implements the Ruby `Logger` class, so all the other methods are available as well:
@@ -407,6 +444,28 @@ get "/" do
 end
 ```
 
+### log_level
+You can specify log level.
+
+```ruby
+Kiev.configure do |config|
+  # One of: 0, 1, 2, 3, 4 (DEBUG, INFO, WARN, ERROR, FATAL)
+  config.log_level = 0
+end
+```
+
+### enable_filter_for_log_levels
+You can specify for which log levels personal identifying information filter will be applied.
+
+```ruby
+Kiev.configure do |config|
+  # [DEBUG, INFO, WARN, ERROR, FATAL]
+  config.enable_filter_for_log_levels = [0, 1, 2, 3, 4]
+end
+```
+
+**By default enabled for all suppported log levels.**
+
 ## nginx
 
 If you want to log 499 and 50x errors in nginx, which will not be captured by Ruby application, consider adding this to your nginx configuration:

data/gemfiles/que_0.12.3.gemfile

@@ -6,7 +6,7 @@ source "https://rubygems.org"
 gem "multi_json"
 gem "oj"
 
-gem "que", ">= 0.12.3"
+gem "que", "0.12.3"
 gem "pg"
 gem "sequel"
 

data/gemfiles/ruby_kafka.gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "ruby-kafka", "~> 0.7.10"
+
+gem "rack-test", require: false
+gem "rspec", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/sidekiq_4.2.gemfile

@@ -11,5 +11,8 @@ gem "rspec", require: false
 gem "minitest-reporters", require: false
 
 gem "her"
+# We need to do it, since her gem doesn't lock upper boundry
+# https://github.com/remi/her/blob/master/her.gemspec#L26
+gem "faraday", "~> 1.9.3"
 
 gemspec path: "../"

data/kiev.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack", ">= 1", "< 3"
   spec.add_dependency "request_store", ">= 1.0", "< 1.4"
   spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
-  spec.add_development_dependency "rake", '~> 0'
-  spec.add_development_dependency "rspec", '~> 0'
+  spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "rspec", "~> 3.10"
   spec.add_development_dependency "rubocop", "~> 0.54"
 end

data/lib/kiev/aws_sns/context_injector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module AwsSns
+    class ContextInjector
+      # @param [Hash] message_attributes Injects context headers
+      # @return [Hash]
+      def call(message_attributes = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          message_attributes[key] = {
+            data_type: "String",
+            string_value: value.to_s
+          }
+        end
+        message_attributes
+      end
+    end
+  end
+end

data/lib/kiev/aws_sns.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module AwsSns
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::AwsSns::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/base.rb

@@ -13,21 +13,38 @@ require_relative "config"
 require_relative "util"
 require_relative "subrequest_helper"
 require_relative "hanami"
+require "forwardable"
+require "logger"
 
 module Kiev
   class << self
+    extend Forwardable
+
+    def_delegators :config,
+                   :logger,
+                   :filtered_params,
+                   :ignored_params,
+                   :log_level,
+                   :enable_filter_for_log_levels
+
     EMPTY_OBJ = {}.freeze
 
     def configure
       yield(Config.instance)
     end
 
-    def logger
-      Config.instance.logger
+    def config
+      Config.instance
     end
 
-    def event(event_name, data = EMPTY_OBJ)
-      logger.log(::Logger::Severity::INFO, data, event_name)
+    def event(log_name, data = EMPTY_OBJ, severity = log_level)
+      logger.log(severity, logged_data(data), log_name)
+    end
+
+    Config.instance.supported_log_levels.each_pair do |key, value|
+      define_method(key) do |log_name, data = EMPTY_OBJ|
+        event(log_name, data, value)
+      end
     end
 
     def []=(name, value)
@@ -47,7 +64,17 @@ module Kiev
     end
 
     def request_id
-      RequestStore.store[:request_id]
+      RequestStore.store[:tracking_id]
+    end
+
+    alias_method :tracking_id, :request_id
+
+    private
+
+    def logged_data(data)
+      return data unless enable_filter_for_log_levels.include?(log_level)
+
+      ParamFilter.filter(data, filtered_params, ignored_params)
     end
   end
 end

data/lib/kiev/config.rb

@@ -75,6 +75,16 @@ module Kiev
       end
     end
 
+    SUPPORTED_LOG_LEVELS = {
+      debug: ::Logger::DEBUG,
+      info: ::Logger::INFO,
+      warn: ::Logger::WARN,
+      error: ::Logger::ERROR,
+      fatal: ::Logger::FATAL
+    }.freeze
+
+    private_constant :SUPPORTED_LOG_LEVELS
+
     attr_accessor :app,
                   :log_request_condition,
                   :log_request_error_condition,
@@ -89,10 +99,12 @@ module Kiev
 
     attr_reader :development_mode,
                 :logger,
+                :log_level,
                 :http_propagated_fields,
                 :jobs_propagated_fields,
                 :all_http_propagated_fields, # for internal use
-                :all_jobs_propagated_fields # for internal use
+                :all_jobs_propagated_fields, # for internal use
+                :enable_filter_for_log_levels
 
     def initialize
       @log_request_condition = DEFAULT_LOG_REQUEST_CONDITION
@@ -105,9 +117,10 @@ module Kiev
       @development_mode = false
       @ignored_rack_exceptions = DEFAULT_IGNORED_RACK_EXCEPTIONS.dup
       @logger = Kiev::Logger.new(STDOUT)
-      @log_level = nil
+      @log_level = default_log_level
       @persistent_log_fields = []
       @pre_rack_hook = DEFAULT_PRE_RACK_HOOK
+      @enable_filter_for_log_levels = supported_log_levels.values
       self.propagated_fields = {}
       update_logger_settings
     end
@@ -138,20 +151,32 @@ module Kiev
     end
 
     def log_level=(value)
+      raise ArgumentError, "Unsupported log level #{value}" unless supported_log_level?(value)
+
       @log_level = value
       update_logger_settings
     end
 
+    def enable_filter_for_log_levels=(log_levels)
+      raise ArgumentError, "Unsupported log levels" unless array_with_log_levels?(log_levels)
+
+      @enable_filter_for_log_levels = log_levels
+    end
+
     def development_mode=(value)
       @development_mode = value
       update_logger_settings
     end
 
+    def supported_log_levels
+      SUPPORTED_LOG_LEVELS
+    end
+
     private
 
     def update_logger_settings
       @logger.formatter = formatter
-      @logger.level = @log_level || default_log_level
+      @logger.level = @log_level
     end
 
     def formatter
@@ -161,5 +186,15 @@ module Kiev
     def default_log_level
       development_mode ? ::Logger::DEBUG : ::Logger::INFO
     end
+
+    def array_with_log_levels?(log_levels)
+      return false unless log_levels.is_a?(Array)
+
+      log_levels.all? { |level| supported_log_level?(level) }
+    end
+
+    def supported_log_level?(log_level)
+      supported_log_levels.value?(log_level)
+    end
   end
 end

data/lib/kiev/kafka/context_extractor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "message_context"
+require "kiev/request_id"
+require "kiev/context_reader"
+
+module Kiev
+  module Kafka
+    class ContextExtractor
+      include Kiev::RequestId::Mixin
+
+      # @param [Kafka::FetchedMessage] message
+      def call(message)
+        context = Kiev::Kafka::MessageContext.new(message)
+        context_reader = Kiev::ContextReader.new(context)
+        wrap_request_id(context_reader) {}
+
+        Kiev[:message_key] = message.key
+
+        Config.instance.jobs_propagated_fields.each do |key|
+          Kiev[key] = context_reader[key]
+        end
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_injector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module Kafka
+    class ContextInjector
+      # @param [Hash] headers Injects context headers
+      # @return [Hash]
+      def call(headers = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
+          headers[field_key] = value
+        end
+        headers
+      end
+    end
+  end
+end

data/lib/kiev/kafka/message_context.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Kafka
+    class MessageContext
+      # @param [Kafka::FetchedMessage] message
+      def initialize(message)
+        @headers = message.headers
+      end
+
+      def value(field)
+        headers[header_key(field)] || headers[field.to_s]
+      end
+
+      alias_method :[], :value
+
+      private
+
+      attr_reader :headers
+
+      # @param [String] field
+      def header_key(field)
+        "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
+      end
+    end
+  end
+end

data/lib/kiev/kafka.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Kafka
+    require_relative "kafka/context_extractor"
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Kafka::FetchedMessage] message
+      def extract_context(message)
+        Kiev::Kafka::ContextExtractor.new.call(message)
+      end
+
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::Kafka::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/logger.rb

@@ -12,17 +12,19 @@ module Kiev
     extend Forwardable
     def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
 
-    DEFAULT_EVENT_NAME = "log"
+    DEFAULT_LOG_NAME = "log"
+    DEFAULT_MESSAGE = "log"
     LOG_ERROR = "ERROR"
     ERROR_STATUS = 500
 
-    FORMATTER = proc do |severity, time, event_name, data|
+    FORMATTER = proc do |severity, time, log_name, data|
       entry =
         {
           application: Config.instance.app,
-          event: event_name || DEFAULT_EVENT_NAME,
+          log_name: log_name || DEFAULT_LOG_NAME,
           level: severity,
           timestamp: time.utc,
+          message: log_name || DEFAULT_MESSAGE,
           tracking_id: RequestStore.store[:tracking_id],
           request_id: RequestStore.store[:request_id],
           request_depth: RequestStore.store[:request_depth],
@@ -39,12 +41,12 @@ module Kiev
         entry[:jid] = RequestStore.store[:jid]
       end
 
-      if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(event_name)
+      if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(log_name)
         entry[:tree_leaf] = true
       end
 
       if RequestStore.store[:payload]
-        if %i(request_finished job_finished).include?(event_name)
+        if %i(request_finished job_finished).include?(log_name)
           entry.merge!(RequestStore.store[:payload])
         else
           Config.instance.persistent_log_fields.each do |field|
@@ -68,17 +70,17 @@ module Kiev
       JSON.logstash(entry)
     end
 
-    DEVELOPMENT_FORMATTER = proc do |severity, time, event_name, data|
+    DEVELOPMENT_FORMATTER = proc do |severity, time, log_name, data|
       entry = []
 
       entry << time.iso8601
-      entry << (event_name || severity).upcase
+      entry << (log_name || severity).upcase
 
       if data.is_a?(String)
         entry << "#{data}\n"
       end
 
-      if %i(request_finished job_finished).include?(event_name)
+      if %i(request_finished job_finished).include?(log_name)
         verb = RequestStore.store[:request_verb]
         path = RequestStore.store[:request_path]
         entry << "#{verb} #{path}" if verb && path

data/lib/kiev/param_filter.rb

@@ -1,12 +1,23 @@
 # frozen_string_literal: true
 
 module Kiev
-  module ParamFilter
+  class ParamFilter
     FILTERED = "[FILTERED]"
 
     def self.filter(params, filtered_params, ignored_params)
+      new(filtered_params, ignored_params).call(params)
+    end
+
+    def initialize(filtered_params, ignored_params)
+      @filtered_params = normalize(filtered_params)
+      @ignored_params = normalize(ignored_params)
+    end
+
+    def call(params)
+      return params unless filterable?(params)
+
       params.each_with_object({}) do |(key, value), acc|
-        next if ignored_params.include?(key)
+        next if ignored_params.include?(key.to_s)
 
         if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
           value = {
@@ -17,14 +28,26 @@ module Kiev
         end
 
         acc[key] =
-          if filtered_params.include?(key) && !value.is_a?(Hash)
+          if filtered_params.include?(key.to_s) && !value.is_a?(Hash)
             FILTERED
           elsif value.is_a?(Hash)
-            filter(value, filtered_params, ignored_params)
+            call(value)
           else
             value
           end
       end
     end
+
+    private
+
+    attr_reader :filtered_params, :ignored_params
+
+    def filterable?(params)
+      params.respond_to?(:each_with_object)
+    end
+
+    def normalize(params)
+      Set.new(params.map(&:to_s))
+    end
   end
 end

data/lib/kiev/rack/request_logger.rb

@@ -85,8 +85,6 @@ module Kiev
             request.params
           end
 
-        params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
-
         data = {
           host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
           params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
+require "kiev/aws_sns/context_injector"
+
 module Kiev
   module Shoryuken
     module Middleware
       class MessageTracer
         def call(options)
-          attrbutes = options[:message_attributes] ||= {}
-          SubrequestHelper.payload.each do |key, value|
-            attrbutes[key] = {
-              data_type: "String",
-              string_value: value.to_s
-            }
-          end
+          options[:message_attributes] ||= {}
+          Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
           yield
         end
       end

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "4.4.0"
+  VERSION = "4.8.0"
 end

data/lib/kiev.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "kiev/base"
+require_relative "kiev/aws_sns" if defined?(AWS::SNS)
+require_relative "kiev/kafka" if defined?(Kafka)
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.8.0
 platform: ruby
 authors:
 - Blacklane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-03 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oga
@@ -98,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -145,6 +145,7 @@ files:
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
 - gemfiles/rails_5.2.gemfile
+- gemfiles/ruby_kafka.gemfile
 - gemfiles/shoryuken_4.0.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
@@ -152,6 +153,8 @@ files:
 - kiev.gemspec
 - lib/ext/rack/common_logger.rb
 - lib/kiev.rb
+- lib/kiev/aws_sns.rb
+- lib/kiev/aws_sns/context_injector.rb
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
@@ -160,6 +163,10 @@ files:
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
+- lib/kiev/kafka.rb
+- lib/kiev/kafka/context_extractor.rb
+- lib/kiev/kafka/context_injector.rb
+- lib/kiev/kafka/message_context.rb
 - lib/kiev/logger.rb
 - lib/kiev/param_filter.rb
 - lib/kiev/que/job.rb
@@ -215,8 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.3.6
 signing_key: 
 specification_version: 4
 summary: Distributed logging to JSON integrated with various Ruby frameworks and tools